From 6d1147f8fecb0f8052a8fbff9f67959511a53af8 Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 17 Jun 2024 17:02:35 -0400 Subject: [PATCH 01/18] feat: use @chainsafe/blst directly (napi-rs) --- packages/beacon-node/package.json | 3 +- .../beacon-node/src/chain/bls/interface.ts | 2 +- .../beacon-node/src/chain/bls/maybeBatch.ts | 15 +- .../src/chain/bls/multithread/index.ts | 16 +- .../src/chain/bls/multithread/jobItem.ts | 13 +- .../src/chain/bls/multithread/types.ts | 1 - .../src/chain/bls/multithread/worker.ts | 5 +- .../beacon-node/src/chain/bls/singleThread.ts | 14 +- packages/beacon-node/src/chain/bls/utils.ts | 5 +- .../opPools/aggregatedAttestationPool.ts | 4 +- .../src/chain/opPools/attestationPool.ts | 10 +- .../chain/opPools/syncCommitteeMessagePool.ts | 7 +- .../opPools/syncContributionAndProofPool.ts | 5 +- .../beacon-node/src/chain/opPools/utils.ts | 5 +- .../signatureSets/aggregateAndProof.ts | 2 +- .../signatureSets/selectionProof.ts | 2 +- .../syncCommitteeContribution.ts | 2 +- .../e2e/api/impl/lightclient/endpoint.test.ts | 4 +- .../test/e2e/chain/bls/multithread.test.ts | 5 +- packages/beacon-node/test/mocks/mockedBls.ts | 2 +- .../perf/api/impl/validator/attester.test.ts | 3 +- .../beacon-node/test/perf/bls/bls.test.ts | 43 ++--- packages/beacon-node/test/spec/bls/bls.ts | 48 ++++-- packages/beacon-node/test/spec/general/bls.ts | 44 +++-- .../test/unit/chain/bls/bls.test.ts | 10 +- .../test/unit/chain/genesis/genesis.test.ts | 2 +- .../opPools/aggregatedAttestationPool.test.ts | 13 +- .../unit/chain/opPools/syncCommittee.test.ts | 6 +- .../opPools/syncCommitteeContribution.test.ts | 11 +- ...idateGossipAttestationsSameAttData.test.ts | 5 +- .../validation/blsToExecutionChange.test.ts | 15 +- .../chain/validation/voluntaryExit.test.ts | 7 +- packages/beacon-node/test/utils/cache.ts | 2 +- .../beacon-node/test/utils/node/validator.ts | 2 +- packages/beacon-node/test/utils/state.ts | 4 +- packages/cli/package.json | 3 +- .../cmds/validator/blsToExecutionChange.ts | 7 +- .../keymanager/decryptKeystoreDefinitions.ts | 4 +- .../cli/src/cmds/validator/keymanager/impl.ts | 4 +- .../validator/keymanager/keystoreCache.ts | 7 +- .../cli/src/cmds/validator/signers/index.ts | 7 +- .../cli/src/cmds/validator/voluntaryExit.ts | 6 +- packages/cli/src/util/format.ts | 5 +- .../keymanager/keystoreCache.test.ts | 4 +- .../crucible/assertions/nodeAssertion.ts | 5 +- .../utils/crucible/externalSignerServer.ts | 2 +- .../cli/test/utils/crucible/interfaces.ts | 2 +- packages/flare/package.json | 2 +- packages/flare/src/cmds/selfSlashAttester.ts | 5 +- packages/flare/src/cmds/selfSlashProposer.ts | 2 +- packages/flare/src/util/deriveSecretKeys.ts | 5 +- packages/state-transition/package.json | 3 +- .../src/block/processDeposit.ts | 9 +- .../state-transition/src/cache/epochCache.ts | 5 +- .../state-transition/src/cache/pubkeyCache.ts | 5 +- .../state-transition/src/cache/stateCache.ts | 5 +- .../src/epoch/processSyncCommitteeUpdates.ts | 4 +- .../src/signatureSets/blsToExecutionChange.ts | 5 +- packages/state-transition/src/util/interop.ts | 5 +- .../src/util/signatureSets.ts | 9 +- .../src/util/syncCommittee.ts | 4 +- .../state-transition/test/perf/block/util.ts | 4 +- packages/state-transition/test/perf/util.ts | 5 +- .../perf/util/loadState/loadState.test.ts | 5 +- .../test/unit/constants.test.ts | 2 +- .../unit/signatureSets/signatureSets.test.ts | 4 +- packages/test-utils/package.json | 2 +- packages/test-utils/src/keystores.ts | 4 +- packages/validator/package.json | 2 +- .../src/services/externalSignerSync.ts | 5 +- .../validator/src/services/validatorStore.ts | 2 +- packages/validator/src/types.ts | 2 +- .../test/unit/services/attestation.test.ts | 4 +- .../unit/services/attestationDuties.test.ts | 4 +- .../test/unit/services/block.test.ts | 4 +- .../test/unit/services/blockDuties.test.ts | 4 +- .../unit/services/externalSignerSync.test.ts | 5 +- .../test/unit/services/indicesService.test.ts | 6 +- .../unit/services/syncCommitteDuties.test.ts | 6 +- .../test/unit/services/syncCommittee.test.ts | 4 +- .../test/unit/validatorStore.test.ts | 4 +- .../validator/test/utils/validatorStore.ts | 2 +- yarn.lock | 158 ++---------------- 83 files changed, 275 insertions(+), 424 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index ee17ff0c0c08..aeb9550a2f97 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -95,8 +95,7 @@ }, "dependencies": { "@chainsafe/as-sha256": "^0.4.1", - "@chainsafe/bls": "7.1.3", - "@chainsafe/blst": "^0.2.11", + "@chainsafe/blst": "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz", "@chainsafe/discv5": "^9.0.0", "@chainsafe/enr": "^3.0.0", "@chainsafe/libp2p-gossipsub": "^13.0.0", diff --git a/packages/beacon-node/src/chain/bls/interface.ts b/packages/beacon-node/src/chain/bls/interface.ts index e9c98ba1920e..f37db9b34aeb 100644 --- a/packages/beacon-node/src/chain/bls/interface.ts +++ b/packages/beacon-node/src/chain/bls/interface.ts @@ -1,4 +1,4 @@ -import {PublicKey} from "@chainsafe/bls/types"; +import {PublicKey} from "@chainsafe/blst"; import {ISignatureSet} from "@lodestar/state-transition"; export type VerifySignatureOpts = { diff --git a/packages/beacon-node/src/chain/bls/maybeBatch.ts b/packages/beacon-node/src/chain/bls/maybeBatch.ts index 619ddf4d72ec..e300b8ff8b76 100644 --- a/packages/beacon-node/src/chain/bls/maybeBatch.ts +++ b/packages/beacon-node/src/chain/bls/maybeBatch.ts @@ -1,5 +1,4 @@ -import {CoordType, PublicKey} from "@chainsafe/bls/types"; -import bls from "@chainsafe/bls"; +import {PublicKey, Signature, verify, verifyMultipleAggregateSignatures} from "@chainsafe/blst"; const MIN_SET_COUNT_TO_BATCH = 2; @@ -16,12 +15,12 @@ export type SignatureSetDeserialized = { export function verifySignatureSetsMaybeBatch(sets: SignatureSetDeserialized[]): boolean { try { if (sets.length >= MIN_SET_COUNT_TO_BATCH) { - return bls.Signature.verifyMultipleSignatures( + return verifyMultipleAggregateSignatures( sets.map((s) => ({ - publicKey: s.publicKey, - message: s.message, + pk: s.publicKey, + msg: s.message, // true = validate signature - signature: bls.Signature.fromBytes(s.signature, CoordType.affine, true), + sig: Signature.fromBytes(s.signature, true), })) ); } @@ -34,8 +33,8 @@ export function verifySignatureSetsMaybeBatch(sets: SignatureSetDeserialized[]): // If too few signature sets verify them without batching return sets.every((set) => { // true = validate signature - const sig = bls.Signature.fromBytes(set.signature, CoordType.affine, true); - return sig.verify(set.publicKey, set.message); + const sig = Signature.fromBytes(set.signature, true); + return verify(set.message, set.publicKey, sig); }); } catch (_) { // A signature could be malformed, in that case fromBytes throws error diff --git a/packages/beacon-node/src/chain/bls/multithread/index.ts b/packages/beacon-node/src/chain/bls/multithread/index.ts index 23e6f1bb460b..3725fa4bcb1c 100644 --- a/packages/beacon-node/src/chain/bls/multithread/index.ts +++ b/packages/beacon-node/src/chain/bls/multithread/index.ts @@ -7,8 +7,7 @@ import {spawn, Worker} from "@chainsafe/threads"; // @ts-ignore // eslint-disable-next-line self = undefined; -import bls from "@chainsafe/bls"; -import {Implementation, PointFormat, PublicKey} from "@chainsafe/bls/types"; +import {PublicKey} from "@chainsafe/blst"; import {Logger} from "@lodestar/utils"; import {ISignatureSet} from "@lodestar/state-transition"; import {QueueError, QueueErrorCode} from "../../../util/queue/index.js"; @@ -116,7 +115,6 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { private readonly logger: Logger; private readonly metrics: Metrics | null; - private readonly format: PointFormat; private readonly workers: WorkerDescriptor[]; private readonly jobs = new LinkedList(); private bufferedJobs: { @@ -136,14 +134,10 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { this.metrics = metrics; this.blsVerifyAllMultiThread = options.blsVerifyAllMultiThread ?? false; - // TODO: Allow to customize implementation - const implementation = bls.implementation; - // Use compressed for herumi for now. // THe worker is not able to deserialize from uncompressed // `Error: err _wrapDeserialize` - this.format = implementation === "blst-native" ? PointFormat.uncompressed : PointFormat.compressed; - this.workers = this.createWorkers(implementation, blsPoolSize); + this.workers = this.createWorkers(blsPoolSize); if (metrics) { metrics.blsThreadPool.queueLength.addCollect(() => { @@ -265,11 +259,11 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { ); } - private createWorkers(implementation: Implementation, poolSize: number): WorkerDescriptor[] { + private createWorkers(poolSize: number): WorkerDescriptor[] { const workers: WorkerDescriptor[] = []; for (let i = 0; i < poolSize; i++) { - const workerData: WorkerData = {implementation, workerId: i}; + const workerData: WorkerData = {workerId: i}; const worker = new Worker(path.join(workerDir, "worker.js"), { workerData, } as ConstructorParameters[1]); @@ -400,7 +394,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { try { // Note: This can throw, must be handled per-job. // Pubkey and signature aggregation is defered here - workReq = jobItemWorkReq(job, this.format, this.metrics); + workReq = jobItemWorkReq(job, this.metrics); } catch (e) { this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({type: job.type}); diff --git a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts index 8b5c63df2eeb..8d051ddc411e 100644 --- a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts +++ b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts @@ -1,5 +1,4 @@ -import bls from "@chainsafe/bls"; -import {CoordType, PointFormat, PublicKey} from "@chainsafe/bls/types"; +import {PublicKey, Signature, aggregatePublicKeys, aggregateSignatures} from "@chainsafe/blst"; import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; import {VerifySignatureOpts} from "../interface.js"; import {getAggregatedPubkey} from "../utils.js"; @@ -49,14 +48,14 @@ export function jobItemSigSets(job: JobQueueItem): number { * Prepare BlsWorkReq from JobQueueItem * WARNING: May throw with untrusted user input */ -export function jobItemWorkReq(job: JobQueueItem, format: PointFormat, metrics: Metrics | null): BlsWorkReq { +export function jobItemWorkReq(job: JobQueueItem, metrics: Metrics | null): BlsWorkReq { switch (job.type) { case JobQueueItemType.default: return { opts: job.opts, sets: job.sets.map((set) => ({ // this can throw, handled in the consumer code - publicKey: getAggregatedPubkey(set, metrics).toBytes(format), + publicKey: getAggregatedPubkey(set, metrics).toBytes(), signature: set.signature, message: set.signingRoot, })), @@ -70,15 +69,15 @@ export function jobItemWorkReq(job: JobQueueItem, format: PointFormat, metrics: // and not a problem in the near future // this is monitored on v1.11.0 https://github.com/ChainSafe/lodestar/pull/5912#issuecomment-1700320307 const timer = metrics?.blsThreadPool.signatureDeserializationMainThreadDuration.startTimer(); - const signatures = job.sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, true)); + const signatures = job.sets.map((set) => Signature.fromBytes(set.signature, true)); timer?.(); return { opts: job.opts, sets: [ { - publicKey: bls.PublicKey.aggregate(job.sets.map((set) => set.publicKey)).toBytes(format), - signature: bls.Signature.aggregate(signatures).toBytes(format), + publicKey: aggregatePublicKeys(job.sets.map((set) => set.publicKey)).toBytes(), + signature: aggregateSignatures(signatures).toBytes(), message: job.message, }, ], diff --git a/packages/beacon-node/src/chain/bls/multithread/types.ts b/packages/beacon-node/src/chain/bls/multithread/types.ts index 3ebc979b559e..74d400a9aa62 100644 --- a/packages/beacon-node/src/chain/bls/multithread/types.ts +++ b/packages/beacon-node/src/chain/bls/multithread/types.ts @@ -1,7 +1,6 @@ import {VerifySignatureOpts} from "../interface.js"; export type WorkerData = { - implementation: "herumi" | "blst-native"; workerId: number; }; diff --git a/packages/beacon-node/src/chain/bls/multithread/worker.ts b/packages/beacon-node/src/chain/bls/multithread/worker.ts index 0db88dcfccd8..9073920df19c 100644 --- a/packages/beacon-node/src/chain/bls/multithread/worker.ts +++ b/packages/beacon-node/src/chain/bls/multithread/worker.ts @@ -1,8 +1,7 @@ /* eslint-disable @typescript-eslint/strict-boolean-expressions */ import worker from "node:worker_threads"; import {expose} from "@chainsafe/threads/worker"; -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/bls/types"; +import {PublicKey} from "@chainsafe/blst"; import {verifySignatureSetsMaybeBatch, SignatureSetDeserialized} from "../maybeBatch.js"; import {WorkerData, BlsWorkReq, WorkResult, WorkResultCode, SerializedSet, BlsWorkResult} from "./types.js"; import {chunkifyMaximizeChunkSize} from "./utils.js"; @@ -109,7 +108,7 @@ function verifyManySignatureSets(workReqArr: BlsWorkReq[]): BlsWorkResult { function deserializeSet(set: SerializedSet): SignatureSetDeserialized { return { - publicKey: bls.PublicKey.fromBytes(set.publicKey, CoordType.affine), + publicKey: PublicKey.fromBytes(set.publicKey), message: set.message, signature: set.signature, }; diff --git a/packages/beacon-node/src/chain/bls/singleThread.ts b/packages/beacon-node/src/chain/bls/singleThread.ts index 58ef6b6d9eec..4e2875b1cea9 100644 --- a/packages/beacon-node/src/chain/bls/singleThread.ts +++ b/packages/beacon-node/src/chain/bls/singleThread.ts @@ -1,6 +1,4 @@ -import {PublicKey, Signature} from "@chainsafe/bls/types"; -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/blst"; +import {PublicKey, Signature, aggregatePublicKeys, aggregateSignatures, verify} from "@chainsafe/blst"; import {ISignatureSet} from "@lodestar/state-transition"; import {Metrics} from "../../metrics/index.js"; import {IBlsVerifier} from "./interface.js"; @@ -40,12 +38,12 @@ export class BlsSingleThreadVerifier implements IBlsVerifier { message: Uint8Array ): Promise { const timer = this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.startTimer(); - const pubkey = bls.PublicKey.aggregate(sets.map((set) => set.publicKey)); + const pubkey = aggregatePublicKeys(sets.map((set) => set.publicKey)); let isAllValid = true; // validate signature = true const signatures = sets.map((set) => { try { - return bls.Signature.fromBytes(set.signature, CoordType.affine, true); + return Signature.fromBytes(set.signature, true); } catch (_) { // at least one set has malformed signature isAllValid = false; @@ -54,8 +52,8 @@ export class BlsSingleThreadVerifier implements IBlsVerifier { }); if (isAllValid) { - const signature = bls.Signature.aggregate(signatures as Signature[]); - isAllValid = signature.verify(pubkey, message); + const signature = aggregateSignatures(signatures as Signature[]); + isAllValid = verify(message, pubkey, signature); } let result: boolean[]; @@ -67,7 +65,7 @@ export class BlsSingleThreadVerifier implements IBlsVerifier { if (sig === null) { return false; } - return sig.verify(set.publicKey, message); + return verify(message, set.publicKey, sig); }); } diff --git a/packages/beacon-node/src/chain/bls/utils.ts b/packages/beacon-node/src/chain/bls/utils.ts index 4a3a027f31ac..63f2bdd80458 100644 --- a/packages/beacon-node/src/chain/bls/utils.ts +++ b/packages/beacon-node/src/chain/bls/utils.ts @@ -1,5 +1,4 @@ -import type {PublicKey} from "@chainsafe/bls/types"; -import bls from "@chainsafe/bls"; +import {PublicKey, aggregatePublicKeys} from "@chainsafe/blst"; import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; import {Metrics} from "../../metrics/metrics.js"; @@ -10,7 +9,7 @@ export function getAggregatedPubkey(signatureSet: ISignatureSet, metrics: Metric case SignatureSetType.aggregate: { const timer = metrics?.blsThreadPool.pubkeysAggregationMainThreadDuration.startTimer(); - const pubkeys = bls.PublicKey.aggregate(signatureSet.pubkeys); + const pubkeys = aggregatePublicKeys(signatureSet.pubkeys); timer?.(); return pubkeys; } diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index c94e5d81e823..556e1f397d60 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -1,5 +1,5 @@ -import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; +import {aggregateSignatures} from "@chainsafe/blst"; import {ForkName, ForkSeq, MAX_ATTESTATIONS, MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params"; import {phase0, Epoch, Slot, ssz, ValidatorIndex, RootHex} from "@lodestar/types"; import { @@ -383,7 +383,7 @@ export function aggregateInto(attestation1: AttestationWithIndex, attestation2: const signature1 = signatureFromBytesNoCheck(attestation1.attestation.signature); const signature2 = signatureFromBytesNoCheck(attestation2.attestation.signature); - attestation1.attestation.signature = bls.Signature.aggregate([signature1, signature2]).toBytes(); + attestation1.attestation.signature = aggregateSignatures([signature1, signature2]).toBytes(); } /** diff --git a/packages/beacon-node/src/chain/opPools/attestationPool.ts b/packages/beacon-node/src/chain/opPools/attestationPool.ts index 804d8798cbc2..2b511598f9a4 100644 --- a/packages/beacon-node/src/chain/opPools/attestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/attestationPool.ts @@ -1,6 +1,5 @@ -import {PointFormat, Signature} from "@chainsafe/bls/types"; -import bls from "@chainsafe/bls"; import {BitArray} from "@chainsafe/ssz"; +import {Signature, aggregateSignatures} from "@chainsafe/blst"; import {phase0, Slot, RootHex} from "@lodestar/types"; import {MapDef} from "@lodestar/utils"; import {IClock} from "../../util/clock.js"; @@ -191,10 +190,7 @@ function aggregateAttestationInto(aggregate: AggregateFast, attestation: phase0. } aggregate.aggregationBits.set(bitIndex, true); - aggregate.signature = bls.Signature.aggregate([ - aggregate.signature, - signatureFromBytesNoCheck(attestation.signature), - ]); + aggregate.signature = aggregateSignatures([aggregate.signature, signatureFromBytesNoCheck(attestation.signature)]); return InsertOutcome.Aggregated; } @@ -217,6 +213,6 @@ function fastToAttestation(aggFast: AggregateFast): phase0.Attestation { return { data: aggFast.data, aggregationBits: aggFast.aggregationBits, - signature: aggFast.signature.toBytes(PointFormat.compressed), + signature: aggFast.signature.toBytes(), }; } diff --git a/packages/beacon-node/src/chain/opPools/syncCommitteeMessagePool.ts b/packages/beacon-node/src/chain/opPools/syncCommitteeMessagePool.ts index 03552992a72a..90a310841f01 100644 --- a/packages/beacon-node/src/chain/opPools/syncCommitteeMessagePool.ts +++ b/packages/beacon-node/src/chain/opPools/syncCommitteeMessagePool.ts @@ -1,6 +1,5 @@ -import {PointFormat, Signature} from "@chainsafe/bls/types"; -import bls from "@chainsafe/bls"; import {BitArray, toHexString} from "@chainsafe/ssz"; +import {Signature, aggregateSignatures} from "@chainsafe/blst"; import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; import {altair, Root, Slot, SubcommitteeIndex} from "@lodestar/types"; import {MapDef} from "@lodestar/utils"; @@ -108,7 +107,7 @@ export class SyncCommitteeMessagePool { return { ...contribution, aggregationBits: contribution.aggregationBits, - signature: contribution.signature.toBytes(PointFormat.compressed), + signature: contribution.signature.toBytes(), }; } @@ -136,7 +135,7 @@ function aggregateSignatureInto( } contribution.aggregationBits.set(indexInSubcommittee, true); - contribution.signature = bls.Signature.aggregate([ + contribution.signature = aggregateSignatures([ contribution.signature, signatureFromBytesNoCheck(signature.signature), ]); diff --git a/packages/beacon-node/src/chain/opPools/syncContributionAndProofPool.ts b/packages/beacon-node/src/chain/opPools/syncContributionAndProofPool.ts index 51b433fd6e50..7834ae534501 100644 --- a/packages/beacon-node/src/chain/opPools/syncContributionAndProofPool.ts +++ b/packages/beacon-node/src/chain/opPools/syncContributionAndProofPool.ts @@ -1,6 +1,5 @@ -import type {Signature} from "@chainsafe/bls/types"; -import bls from "@chainsafe/bls"; import {BitArray, toHexString} from "@chainsafe/ssz"; +import {Signature, aggregateSignatures} from "@chainsafe/blst"; import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params"; import {altair, Slot, Root, ssz} from "@lodestar/types"; import {G2_POINT_AT_INFINITY} from "@lodestar/state-transition"; @@ -182,6 +181,6 @@ export function aggregate(bestContributionBySubnet: Map, slot: Slot, slotsRetained: * No need to verify Signature is valid, already run sig-verify = false */ export function signatureFromBytesNoCheck(signature: Uint8Array): Signature { - return bls.Signature.fromBytes(signature, CoordType.affine, false); + return Signature.fromBytes(signature); } /** diff --git a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts index 2bc2e62c861f..59787341cfb9 100644 --- a/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/signatureSets/aggregateAndProof.ts @@ -1,4 +1,4 @@ -import type {PublicKey} from "@chainsafe/bls/types"; +import {PublicKey} from "@chainsafe/blst"; import {DOMAIN_AGGREGATE_AND_PROOF} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {Epoch, phase0} from "@lodestar/types"; diff --git a/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts b/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts index 09e0a5ef12be..5e129a88aa20 100644 --- a/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts +++ b/packages/beacon-node/src/chain/validation/signatureSets/selectionProof.ts @@ -1,4 +1,4 @@ -import type {PublicKey} from "@chainsafe/bls/types"; +import {PublicKey} from "@chainsafe/blst"; import {DOMAIN_SELECTION_PROOF} from "@lodestar/params"; import {phase0, Slot, ssz} from "@lodestar/types"; import {computeSigningRoot, createSingleSignatureSetFromComponents, ISignatureSet} from "@lodestar/state-transition"; diff --git a/packages/beacon-node/src/chain/validation/signatureSets/syncCommitteeContribution.ts b/packages/beacon-node/src/chain/validation/signatureSets/syncCommitteeContribution.ts index 71b7970716e9..0e1eaefef7c6 100644 --- a/packages/beacon-node/src/chain/validation/signatureSets/syncCommitteeContribution.ts +++ b/packages/beacon-node/src/chain/validation/signatureSets/syncCommitteeContribution.ts @@ -1,4 +1,4 @@ -import type {PublicKey} from "@chainsafe/bls/types"; +import {PublicKey} from "@chainsafe/blst"; import {altair, ssz} from "@lodestar/types"; import {DOMAIN_SYNC_COMMITTEE} from "@lodestar/params"; import {CachedBeaconStateAltair, computeSigningRoot, ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; diff --git a/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts b/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts index 0f13575a5ec4..af6bb5561948 100644 --- a/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/lightclient/endpoint.test.ts @@ -1,5 +1,5 @@ import {describe, it, beforeEach, afterEach, expect} from "vitest"; -import bls from "@chainsafe/bls"; +import {aggregateSerializedPublicKeys} from "@chainsafe/blst"; import {createBeaconConfig, ChainConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; import {getClient, routes} from "@lodestar/api"; @@ -127,7 +127,7 @@ describe("lightclient api", function () { const committeePubkeys = Array.from({length: SYNC_COMMITTEE_SIZE}, (_, i) => i % 2 === 0 ? pubkeys[0] : pubkeys[1] ); - const aggregatePubkey = bls.aggregatePublicKeys(committeePubkeys); + const aggregatePubkey = aggregateSerializedPublicKeys(committeePubkeys).toBytes(); // single committee hash since we requested for the first period expect(committeeRes.value()).toEqual([ ssz.altair.SyncCommittee.hashTreeRoot({ diff --git a/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts b/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts index bf1e73469433..25f0d5133bcd 100644 --- a/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts +++ b/packages/beacon-node/test/e2e/chain/bls/multithread.test.ts @@ -1,6 +1,5 @@ import {describe, it, beforeAll, expect, beforeEach, afterEach} from "vitest"; -import bls from "@chainsafe/bls"; -import {PublicKey} from "@chainsafe/bls/types"; +import {PublicKey, SecretKey} from "@chainsafe/blst"; import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; import {BlsMultiThreadWorkerPool} from "../../../../src/chain/bls/multithread/index.js"; import {testLogger} from "../../../utils/logger.js"; @@ -29,7 +28,7 @@ describe("chain / bls / multithread queue", function () { beforeAll(() => { for (let i = 0; i < 3; i++) { - const sk = bls.SecretKey.fromBytes(Buffer.alloc(32, i + 1)); + const sk = SecretKey.fromBytes(Buffer.alloc(32, i + 1)); const msg = Buffer.alloc(32, i + 1); const pk = sk.toPublicKey(); const sig = sk.sign(msg); diff --git a/packages/beacon-node/test/mocks/mockedBls.ts b/packages/beacon-node/test/mocks/mockedBls.ts index 0ecec5f13bde..bc6bbf361de5 100644 --- a/packages/beacon-node/test/mocks/mockedBls.ts +++ b/packages/beacon-node/test/mocks/mockedBls.ts @@ -1,4 +1,4 @@ -import {PublicKey} from "@chainsafe/bls/types"; +import {PublicKey} from "@chainsafe/blst"; import {IBlsVerifier} from "../../src/chain/bls/index.js"; export class BlsVerifierMock implements IBlsVerifier { diff --git a/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts b/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts index 10527fdf94b5..803d6c84c408 100644 --- a/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts +++ b/packages/beacon-node/test/perf/api/impl/validator/attester.test.ts @@ -1,5 +1,4 @@ import {itBench} from "@dapplion/benchmark"; -import {PointFormat} from "@chainsafe/bls/types"; // eslint-disable-next-line import/no-relative-packages import {generatePerfTestCachedStatePhase0, numValidators} from "../../../../../../state-transition/test/perf/util.js"; import {getPubkeysForIndices} from "../../../../../src/api/impl/validator/utils.js"; @@ -36,7 +35,7 @@ describe("api / impl / validator", () => { fn: () => { for (let i = 0; i < reqCount; i++) { const pubkey = state.epochCtx.index2pubkey[i]; - pubkey.toBytes(PointFormat.compressed); + pubkey.toBytes(); } }, }); diff --git a/packages/beacon-node/test/perf/bls/bls.test.ts b/packages/beacon-node/test/perf/bls/bls.test.ts index a982cc55e499..2adebba38213 100644 --- a/packages/beacon-node/test/perf/bls/bls.test.ts +++ b/packages/beacon-node/test/perf/bls/bls.test.ts @@ -1,7 +1,14 @@ import crypto from "node:crypto"; import {itBench} from "@dapplion/benchmark"; -import bls from "@chainsafe/bls"; -import {CoordType, type PublicKey, type SecretKey} from "@chainsafe/bls/types"; +import { + PublicKey, + SecretKey, + Signature, + aggregatePublicKeys, + aggregateSignatures, + verify, + verifyMultipleAggregateSignatures, +} from "@chainsafe/blst"; import {linspace} from "../../../src/util/numpy.js"; describe("BLS ops", function () { @@ -20,7 +27,7 @@ describe("BLS ops", function () { const bytes = new Uint8Array(32); const dataView = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); dataView.setUint32(0, i + 1, true); - const secretKey = bls.SecretKey.fromBytes(bytes); + const secretKey = SecretKey.fromBytes(bytes); const publicKey = secretKey.toPublicKey(); keypair = {secretKey, publicKey}; keypairs.set(i, keypair); @@ -53,8 +60,8 @@ describe("BLS ops", function () { } // Note: getSet() caches the value, does not re-compute every time - itBench({id: `BLS verify - ${bls.implementation}`, beforeEach: () => getSet(0)}, (set) => { - const isValid = bls.Signature.fromBytes(set.signature).verify(set.publicKey, set.message); + itBench({id: "BLS verify - blst", beforeEach: () => getSet(0)}, (set) => { + const isValid = verify(set.message, set.publicKey, Signature.fromBytes(set.signature)); if (!isValid) throw Error("Invalid"); }); @@ -62,14 +69,14 @@ describe("BLS ops", function () { // We may want to bundle up to 32 sets in a single batch. for (const count of [3, 8, 32, 64, 128]) { itBench({ - id: `BLS verifyMultipleSignatures ${count} - ${bls.implementation}`, + id: `BLS verifyMultipleSignatures ${count} - blst`, beforeEach: () => linspace(0, count - 1).map((i) => getSet(i)), fn: (sets) => { - const isValid = bls.Signature.verifyMultipleSignatures( + const isValid = verifyMultipleAggregateSignatures( sets.map((set) => ({ - publicKey: set.publicKey, - message: set.message, - signature: bls.Signature.fromBytes(set.signature), + pk: set.publicKey, + msg: set.message, + sig: Signature.fromBytes(set.signature), })) ); if (!isValid) throw Error("Invalid"); @@ -86,7 +93,7 @@ describe("BLS ops", function () { fn: () => { for (const signature of signatures) { // true = validate signature - bls.Signature.fromBytes(signature, CoordType.affine, true); + Signature.fromBytes(signature, true); } }, }); @@ -97,15 +104,13 @@ describe("BLS ops", function () { // TODO: figure out why it does not work with 256 or more for (const count of [3, 8, 32, 64, 128]) { itBench({ - id: `BLS verifyMultipleSignatures - same message - ${count} - ${bls.implementation}`, + id: `BLS verifyMultipleSignatures - same message - ${count} - blst`, beforeEach: () => linspace(0, count - 1).map((i) => getSetSameMessage(i)), fn: (sets) => { // aggregate and verify aggregated signatures - const aggregatedPubkey = bls.PublicKey.aggregate(sets.map((set) => set.publicKey)); - const aggregatedSignature = bls.Signature.aggregate( - sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, false)) - ); - const isValid = aggregatedSignature.verify(aggregatedPubkey, sets[0].message); + const aggregatedPubkey = aggregatePublicKeys(sets.map((set) => set.publicKey)); + const aggregatedSignature = aggregateSignatures(sets.map((set) => Signature.fromBytes(set.signature))); + const isValid = verify(sets[0].message, aggregatedPubkey, aggregatedSignature); if (!isValid) throw Error("Invalid"); }, }); @@ -114,10 +119,10 @@ describe("BLS ops", function () { // Attestations in Mainnet contain 128 max on average for (const count of [32, 128]) { itBench({ - id: `BLS aggregatePubkeys ${count} - ${bls.implementation}`, + id: `BLS aggregatePubkeys ${count} - blst`, beforeEach: () => linspace(0, count - 1).map((i) => getKeypair(i).publicKey), fn: (pubkeys) => { - bls.PublicKey.aggregate(pubkeys); + aggregatePublicKeys(pubkeys); }, }); } diff --git a/packages/beacon-node/test/spec/bls/bls.ts b/packages/beacon-node/test/spec/bls/bls.ts index c8d42ad84c5f..6432ca7de3cc 100644 --- a/packages/beacon-node/test/spec/bls/bls.ts +++ b/packages/beacon-node/test/spec/bls/bls.ts @@ -1,7 +1,14 @@ -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/bls/types"; +import { + PublicKey, + SecretKey, + Signature, + aggregateSignatures, + aggregateVerify, + fastAggregateVerify, + verifyMultipleAggregateSignatures, + verify as _verify, +} from "@chainsafe/blst"; import {fromHexString} from "@chainsafe/ssz"; -import {toHexString} from "@lodestar/utils"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -31,7 +38,11 @@ export const testFnByType: Record any)> = { */ function aggregate_verify(input: {pubkeys: string[]; messages: string[]; signature: string}): boolean { const {pubkeys, messages, signature} = input; - return bls.verifyMultiple(pubkeys.map(fromHexString), messages.map(fromHexString), fromHexString(signature)); + return aggregateVerify( + messages.map(fromHexString), + pubkeys.map((pk) => PublicKey.fromHex(pk)), + Signature.fromHex(signature) + ); } /** @@ -41,8 +52,8 @@ function aggregate_verify(input: {pubkeys: string[]; messages: string[]; signatu * ``` */ function aggregate(input: string[]): string { - const pks = input.map((pkHex) => bls.Signature.fromHex(pkHex)); - const agg = bls.Signature.aggregate(pks); + const pks = input.map((pkHex) => Signature.fromHex(pkHex)); + const agg = aggregateSignatures(pks); return agg.toHex(); } @@ -58,9 +69,10 @@ function aggregate(input: string[]): string { function fast_aggregate_verify(input: {pubkeys: string[]; message: string; signature: string}): boolean | null { const {pubkeys, message, signature} = input; try { - return bls.Signature.fromBytes(fromHexString(signature), undefined, true).verifyAggregate( - pubkeys.map((hex) => bls.PublicKey.fromBytes(fromHexString(hex), CoordType.jacobian, true)), - fromHexString(message) + return fastAggregateVerify( + fromHexString(message), + pubkeys.map((hex) => PublicKey.fromHex(hex, true)), + Signature.fromHex(signature, true) ); } catch (e) { return false; @@ -80,11 +92,11 @@ function fast_aggregate_verify(input: {pubkeys: string[]; message: string; signa function batch_verify(input: {pubkeys: string[]; messages: string[]; signatures: string[]}): boolean | null { const {pubkeys, messages, signatures} = input; try { - return bls.Signature.verifyMultipleSignatures( + return verifyMultipleAggregateSignatures( pubkeys.map((pubkey, i) => ({ - publicKey: bls.PublicKey.fromBytes(fromHexString(pubkey), CoordType.jacobian, true), - message: fromHexString(messages[i]), - signature: bls.Signature.fromBytes(fromHexString(signatures[i]), undefined, true), + pk: PublicKey.fromHex(pubkey, true), + msg: fromHexString(messages[i]), + sig: Signature.fromHex(signatures[i], true), })) ); } catch (e) { @@ -103,8 +115,8 @@ function batch_verify(input: {pubkeys: string[]; messages: string[]; signatures: */ function sign(input: {privkey: string; message: string}): string | null { const {privkey, message} = input; - const signature = bls.sign(fromHexString(privkey), fromHexString(message)); - return toHexString(signature); + const signature = SecretKey.fromHex(privkey).sign(fromHexString(message)); + return signature.toHex(); } /** @@ -119,7 +131,7 @@ function sign(input: {privkey: string; message: string}): string | null { */ function verify(input: {pubkey: string; message: string; signature: string}): boolean { const {pubkey, message, signature} = input; - return bls.verify(fromHexString(pubkey), fromHexString(message), fromHexString(signature)); + return _verify(fromHexString(message), PublicKey.fromHex(pubkey), Signature.fromHex(signature)); } /** @@ -131,7 +143,7 @@ function verify(input: {pubkey: string; message: string; signature: string}): bo */ function deserialization_G1(input: {pubkey: string}): boolean { try { - bls.PublicKey.fromBytes(fromHexString(input.pubkey), CoordType.jacobian, true); + PublicKey.fromHex(input.pubkey, true); return true; } catch (e) { return false; @@ -147,7 +159,7 @@ function deserialization_G1(input: {pubkey: string}): boolean { */ function deserialization_G2(input: {signature: string}): boolean { try { - bls.Signature.fromBytes(fromHexString(input.signature), undefined, true); + Signature.fromHex(input.signature, true); return true; } catch (e) { return false; diff --git a/packages/beacon-node/test/spec/general/bls.ts b/packages/beacon-node/test/spec/general/bls.ts index 88c1d79bcb79..5d1e3b472417 100644 --- a/packages/beacon-node/test/spec/general/bls.ts +++ b/packages/beacon-node/test/spec/general/bls.ts @@ -1,8 +1,15 @@ -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/bls/types"; import {fromHexString} from "@chainsafe/ssz"; +import { + PublicKey, + SecretKey, + Signature, + aggregateSerializedPublicKeys, + aggregateSignatures, + aggregateVerify, + fastAggregateVerify, + verify as _verify, +} from "@chainsafe/blst"; import {InputType} from "@lodestar/spec-test-util"; -import {toHexString} from "@lodestar/utils"; import {TestRunnerFn} from "../utils/types.js"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -64,8 +71,8 @@ type BlsTestCase = { * ``` */ function aggregate(input: string[]): string { - const pks = input.map((pkHex) => bls.Signature.fromHex(pkHex)); - const agg = bls.Signature.aggregate(pks); + const pks = input.map((pkHex) => Signature.fromHex(pkHex)); + const agg = aggregateSignatures(pks); return agg.toHex(); } @@ -80,7 +87,11 @@ function aggregate(input: string[]): string { */ function aggregate_verify(input: {pubkeys: string[]; messages: string[]; signature: string}): boolean { const {pubkeys, messages, signature} = input; - return bls.verifyMultiple(pubkeys.map(fromHexString), messages.map(fromHexString), fromHexString(signature)); + return aggregateVerify( + messages.map(fromHexString), + pubkeys.map((pk) => PublicKey.fromHex(pk)), + Signature.fromHex(signature) + ); } /** @@ -95,8 +106,7 @@ function eth_aggregate_pubkeys(input: string[]): string | null { if (pk === G1_POINT_AT_INFINITY) return null; } - const agg = bls.aggregatePublicKeys(input.map((hex) => fromHexString(hex))); - return toHexString(agg); + return aggregateSerializedPublicKeys(input.map((hex) => fromHexString(hex))).toHex(); } /** @@ -120,10 +130,10 @@ function eth_fast_aggregate_verify(input: {pubkeys: string[]; message: string; s if (pk === G1_POINT_AT_INFINITY) return false; } - return bls.verifyAggregate( - pubkeys.map((hex) => fromHexString(hex)), + return fastAggregateVerify( fromHexString(message), - fromHexString(signature) + pubkeys.map((hex) => PublicKey.fromHex(hex)), + Signature.fromHex(signature) ); } @@ -139,9 +149,10 @@ function eth_fast_aggregate_verify(input: {pubkeys: string[]; message: string; s function fast_aggregate_verify(input: {pubkeys: string[]; message: string; signature: string}): boolean | null { const {pubkeys, message, signature} = input; try { - return bls.Signature.fromBytes(fromHexString(signature), undefined, true).verifyAggregate( - pubkeys.map((hex) => bls.PublicKey.fromBytes(fromHexString(hex), CoordType.jacobian, true)), - fromHexString(message) + return fastAggregateVerify( + fromHexString(message), + pubkeys.map((hex) => PublicKey.fromHex(hex, true)), + Signature.fromHex(signature, true) ); } catch (e) { return false; @@ -156,8 +167,7 @@ function fast_aggregate_verify(input: {pubkeys: string[]; message: string; signa */ function sign(input: {privkey: string; message: string}): string | null { const {privkey, message} = input; - const signature = bls.sign(fromHexString(privkey), fromHexString(message)); - return toHexString(signature); + return SecretKey.fromHex(privkey).sign(fromHexString(message)).toHex(); } /** @@ -169,5 +179,5 @@ function sign(input: {privkey: string; message: string}): string | null { */ function verify(input: {pubkey: string; message: string; signature: string}): boolean { const {pubkey, message, signature} = input; - return bls.verify(fromHexString(pubkey), fromHexString(message), fromHexString(signature)); + return _verify(fromHexString(message), PublicKey.fromHex(pubkey), Signature.fromHex(signature)); } diff --git a/packages/beacon-node/test/unit/chain/bls/bls.test.ts b/packages/beacon-node/test/unit/chain/bls/bls.test.ts index e5b844262632..4203d7f9768c 100644 --- a/packages/beacon-node/test/unit/chain/bls/bls.test.ts +++ b/packages/beacon-node/test/unit/chain/bls/bls.test.ts @@ -1,7 +1,5 @@ -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/blst"; -import {PublicKey} from "@chainsafe/bls/types"; import {describe, it, expect, beforeEach} from "vitest"; +import {PublicKey, SecretKey, Signature} from "@chainsafe/blst"; import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; import {BlsSingleThreadVerifier} from "../../../../src/chain/bls/singleThread.js"; import {BlsMultiThreadWorkerPool} from "../../../../src/chain/bls/multithread/index.js"; @@ -10,7 +8,7 @@ import {testLogger} from "../../../utils/logger.js"; describe("BlsVerifier ", function () { // take time for creating thread pool const numKeys = 3; - const secretKeys = Array.from({length: numKeys}, (_, i) => bls.SecretKey.fromKeygen(Buffer.alloc(32, i))); + const secretKeys = Array.from({length: numKeys}, (_, i) => SecretKey.fromKeygen(Buffer.alloc(32, i))); const verifiers = [ new BlsSingleThreadVerifier({metrics: null}), new BlsMultiThreadWorkerPool({}, {metrics: null, logger: testLogger()}), @@ -46,7 +44,7 @@ describe("BlsVerifier ", function () { it("should return false if at least one signature is malformed", async () => { // signature is malformed const malformedSignature = Buffer.alloc(96, 10); - expect(() => bls.Signature.fromBytes(malformedSignature, CoordType.affine, true)).toThrow(); + expect(() => Signature.fromBytes(malformedSignature, true, true)).toThrow(); sets[1].signature = malformedSignature; expect(await verifier.verifySignatureSets(sets)).toBe(false); }); @@ -79,7 +77,7 @@ describe("BlsVerifier ", function () { it("should return false for malformed signature", async () => { // signature is malformed const malformedSignature = Buffer.alloc(96, 10); - expect(() => bls.Signature.fromBytes(malformedSignature, CoordType.affine, true)).toThrow(); + expect(() => Signature.fromBytes(malformedSignature, true, true)).toThrow(); sets[1].signature = malformedSignature; expect(await verifier.verifySignatureSetsSameMessage(sets, signingRoot)).toEqual([true, false, true]); }); diff --git a/packages/beacon-node/test/unit/chain/genesis/genesis.test.ts b/packages/beacon-node/test/unit/chain/genesis/genesis.test.ts index 986ca074242f..f836e3621cc9 100644 --- a/packages/beacon-node/test/unit/chain/genesis/genesis.test.ts +++ b/packages/beacon-node/test/unit/chain/genesis/genesis.test.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import type {SecretKey, PublicKey} from "@chainsafe/bls/types"; import {toHexString} from "@chainsafe/ssz"; import {describe, it, expect} from "vitest"; +import {PublicKey, SecretKey} from "@chainsafe/blst"; import {DOMAIN_DEPOSIT, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; import {config} from "@lodestar/config/default"; import {computeDomain, computeSigningRoot, interopSecretKey, ZERO_HASH} from "@lodestar/state-transition"; diff --git a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index 3c248ad4d194..800984fa84bc 100644 --- a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -1,7 +1,6 @@ -import type {SecretKey} from "@chainsafe/bls/types"; -import bls from "@chainsafe/bls"; import {BitArray, fromHexString, toHexString} from "@chainsafe/ssz"; import {describe, it, expect, beforeEach, beforeAll, afterEach, vi} from "vitest"; +import {SecretKey, Signature, fastAggregateVerify} from "@chainsafe/blst"; import {CachedBeaconStateAllForks, newFilledArray} from "@lodestar/state-transition"; import {FAR_FUTURE_EPOCH, ForkName, MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH} from "@lodestar/params"; import {ssz, phase0} from "@lodestar/types"; @@ -303,8 +302,8 @@ describe("MatchingDataAttestationGroup aggregateInto", function () { let sk2: SecretKey; beforeAll(async () => { - sk1 = bls.SecretKey.fromBytes(Buffer.alloc(32, 1)); - sk2 = bls.SecretKey.fromBytes(Buffer.alloc(32, 2)); + sk1 = SecretKey.fromBytes(Buffer.alloc(32, 1)); + sk2 = SecretKey.fromBytes(Buffer.alloc(32, 2)); attestation1.signature = sk1.sign(attestationDataRoot).toBytes(); attestation2.signature = sk2.sign(attestationDataRoot).toBytes(); }); @@ -315,7 +314,9 @@ describe("MatchingDataAttestationGroup aggregateInto", function () { aggregateInto(attWithIndex1, attWithIndex2); expect(renderBitArray(attWithIndex1.attestation.aggregationBits)).toEqual(renderBitArray(mergedBitArray)); - const aggregatedSignature = bls.Signature.fromBytes(attWithIndex1.attestation.signature, undefined, true); - expect(aggregatedSignature.verifyAggregate([sk1.toPublicKey(), sk2.toPublicKey()], attestationDataRoot)).toBe(true); + const aggregatedSignature = Signature.fromBytes(attWithIndex1.attestation.signature, true, true); + expect(fastAggregateVerify(attestationDataRoot, [sk1.toPublicKey(), sk2.toPublicKey()], aggregatedSignature)).toBe( + true + ); }); }); diff --git a/packages/beacon-node/test/unit/chain/opPools/syncCommittee.test.ts b/packages/beacon-node/test/unit/chain/opPools/syncCommittee.test.ts index 3cb5d496bf9b..507c78d560b6 100644 --- a/packages/beacon-node/test/unit/chain/opPools/syncCommittee.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/syncCommittee.test.ts @@ -1,6 +1,6 @@ -import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; import {describe, it, expect, beforeEach, beforeAll, afterEach, vi, MockedObject} from "vitest"; +import {SecretKey} from "@chainsafe/blst"; import {altair} from "@lodestar/types"; import {SyncCommitteeMessagePool} from "../../../../src/chain/opPools/index.js"; import {Clock} from "../../../../src/util/clock.js"; @@ -18,7 +18,7 @@ describe("chain / opPools / SyncCommitteeMessagePool", function () { const cutOffTime = 1; beforeAll(async () => { - const sk = bls.SecretKey.fromBytes(Buffer.alloc(32, 1)); + const sk = SecretKey.fromBytes(Buffer.alloc(32, 1)); syncCommittee = { slot, beaconBlockRoot, @@ -42,7 +42,7 @@ describe("chain / opPools / SyncCommitteeMessagePool", function () { clockStub.secFromSlot.mockReturnValue(0); let contribution = cache.getContribution(subcommitteeIndex, syncCommittee.slot, syncCommittee.beaconBlockRoot); expect(contribution).not.toBeNull(); - const newSecretKey = bls.SecretKey.fromBytes(Buffer.alloc(32, 2)); + const newSecretKey = SecretKey.fromBytes(Buffer.alloc(32, 2)); const newSyncCommittee: altair.SyncCommitteeMessage = { slot: syncCommittee.slot, beaconBlockRoot, diff --git a/packages/beacon-node/test/unit/chain/opPools/syncCommitteeContribution.test.ts b/packages/beacon-node/test/unit/chain/opPools/syncCommitteeContribution.test.ts index dd303673f61e..e34a5d006272 100644 --- a/packages/beacon-node/test/unit/chain/opPools/syncCommitteeContribution.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/syncCommitteeContribution.test.ts @@ -1,7 +1,6 @@ -import type {SecretKey} from "@chainsafe/bls/types"; -import bls from "@chainsafe/bls"; import {BitArray} from "@chainsafe/ssz"; import {describe, it, expect, beforeEach, beforeAll} from "vitest"; +import {SecretKey, Signature, fastAggregateVerify} from "@chainsafe/blst"; import {newFilledArray} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import {SYNC_COMMITTEE_SIZE, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; @@ -83,7 +82,7 @@ describe("aggregate", function () { let bestContributionBySubnet: Map; beforeAll(async () => { for (let i = 0; i < SYNC_COMMITTEE_SUBNET_COUNT; i++) { - sks.push(bls.SecretKey.fromBytes(Buffer.alloc(32, i + 1))); + sks.push(SecretKey.fromBytes(Buffer.alloc(32, i + 1))); } bestContributionBySubnet = new Map(); }); @@ -112,10 +111,10 @@ describe("aggregate", function () { renderBitArray(BitArray.fromBoolArray(expectSyncCommittees)) ); expect( - bls.verifyAggregate( - testSks.map((sk) => sk.toPublicKey().toBytes()), + fastAggregateVerify( blockRoot, - syncAggregate.syncCommitteeSignature + testSks.map((sk) => sk.toPublicKey()), + Signature.fromBytes(syncAggregate.syncCommitteeSignature) ) ).toBe(true); }); diff --git a/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts index cc6dc26f1a36..3ec95d1dcb55 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts @@ -1,6 +1,5 @@ -import bls from "@chainsafe/bls"; -import type {PublicKey, SecretKey} from "@chainsafe/bls/types"; import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; +import {PublicKey, SecretKey} from "@chainsafe/blst"; import {ForkName} from "@lodestar/params"; import {SignatureSetType} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; @@ -49,7 +48,7 @@ describe("validateGossipAttestationsSameAttData", () => { const bytes = new Uint8Array(32); const dataView = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); dataView.setUint32(0, i + 1, true); - const secretKey = bls.SecretKey.fromBytes(bytes); + const secretKey = SecretKey.fromBytes(bytes); const publicKey = secretKey.toPublicKey(); keypair = {secretKey, publicKey}; keypairs.set(i, keypair); diff --git a/packages/beacon-node/test/unit/chain/validation/blsToExecutionChange.test.ts b/packages/beacon-node/test/unit/chain/validation/blsToExecutionChange.test.ts index f83b900beb69..8d5032ed3ec7 100644 --- a/packages/beacon-node/test/unit/chain/validation/blsToExecutionChange.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/blsToExecutionChange.test.ts @@ -1,7 +1,6 @@ import {digest} from "@chainsafe/as-sha256"; -import bls from "@chainsafe/bls"; -import {PointFormat} from "@chainsafe/bls/types"; import {describe, it, beforeEach, afterEach, vi} from "vitest"; +import {SecretKey} from "@chainsafe/blst"; import {config as defaultConfig} from "@lodestar/config/default"; import {computeSigningRoot} from "@lodestar/state-transition"; import {capella, ssz} from "@lodestar/types"; @@ -29,12 +28,12 @@ describe("validate bls to execution change", () => { // Validator has to be active for long enough stateEmpty.slot = defaultConfig.SHARD_COMMITTEE_PERIOD * SLOTS_PER_EPOCH; // A withdrawal key which we will keep same on the two vals we generate - const wsk = bls.SecretKey.fromKeygen(); + const wsk = SecretKey.fromKeygen(Buffer.alloc(32)); // Generate and add first val - const sk1 = bls.SecretKey.fromKeygen(); - const pubkey1 = sk1.toPublicKey().toBytes(PointFormat.compressed); - const fromBlsPubkey = wsk.toPublicKey().toBytes(PointFormat.compressed); + const sk1 = SecretKey.fromKeygen(Buffer.alloc(32, 1)); + const pubkey1 = sk1.toPublicKey().toBytes(); + const fromBlsPubkey = wsk.toPublicKey().toBytes(); const withdrawalCredentials = digest(fromBlsPubkey); withdrawalCredentials[0] = BLS_WITHDRAWAL_PREFIX; const validator = ssz.phase0.Validator.toViewDU({ @@ -50,8 +49,8 @@ describe("validate bls to execution change", () => { stateEmpty.validators[0] = validator; // Gen and add second val - const sk2 = bls.SecretKey.fromKeygen(); - const pubkey2 = sk2.toPublicKey().toBytes(PointFormat.compressed); + const sk2 = SecretKey.fromKeygen(Buffer.alloc(32, 2)); + const pubkey2 = sk2.toPublicKey().toBytes(); // Set the next validator to already eth1 credential const withdrawalCredentialsTwo = digest(fromBlsPubkey); withdrawalCredentialsTwo[0] = ETH1_ADDRESS_WITHDRAWAL_PREFIX; diff --git a/packages/beacon-node/test/unit/chain/validation/voluntaryExit.test.ts b/packages/beacon-node/test/unit/chain/validation/voluntaryExit.test.ts index 3966815c28ce..d556ff610961 100644 --- a/packages/beacon-node/test/unit/chain/validation/voluntaryExit.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/voluntaryExit.test.ts @@ -1,6 +1,5 @@ -import bls from "@chainsafe/bls"; -import {PointFormat} from "@chainsafe/bls/types"; import {describe, it, beforeEach, beforeAll, vi, afterEach} from "vitest"; +import {SecretKey} from "@chainsafe/blst"; import {config} from "@lodestar/config/default"; import { CachedBeaconStateAllForks, @@ -25,7 +24,7 @@ describe("validate voluntary exit", () => { let opPool: MockedBeaconChain["opPool"]; beforeAll(() => { - const sk = bls.SecretKey.fromKeygen(); + const sk = SecretKey.fromKeygen(Buffer.alloc(32)); const stateEmpty = ssz.phase0.BeaconState.defaultValue(); @@ -34,7 +33,7 @@ describe("validate voluntary exit", () => { // Add a validator that's active since genesis and ready to exit const validator = ssz.phase0.Validator.toViewDU({ - pubkey: sk.toPublicKey().toBytes(PointFormat.compressed), + pubkey: sk.toPublicKey().toBytes(), withdrawalCredentials: Buffer.alloc(32, 0), effectiveBalance: 32e9, slashed: false, diff --git a/packages/beacon-node/test/utils/cache.ts b/packages/beacon-node/test/utils/cache.ts index 767956226776..9d5d3566c99d 100644 --- a/packages/beacon-node/test/utils/cache.ts +++ b/packages/beacon-node/test/utils/cache.ts @@ -1,4 +1,4 @@ -import type {SecretKey} from "@chainsafe/bls/types"; +import {SecretKey} from "@chainsafe/blst"; import {toHexString} from "@chainsafe/ssz"; export function memoOnce(fn: () => R): () => R { diff --git a/packages/beacon-node/test/utils/node/validator.ts b/packages/beacon-node/test/utils/node/validator.ts index c686449a29f2..1c17f0c5ed46 100644 --- a/packages/beacon-node/test/utils/node/validator.ts +++ b/packages/beacon-node/test/utils/node/validator.ts @@ -1,5 +1,5 @@ import tmp from "tmp"; -import type {SecretKey} from "@chainsafe/bls/types"; +import {SecretKey} from "@chainsafe/blst"; import {LevelDbController} from "@lodestar/db"; import {interopSecretKey} from "@lodestar/state-transition"; import {SlashingProtection, Validator, Signer, SignerType, ValidatorProposerConfig} from "@lodestar/validator"; diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index af9ebd479340..0d9ea92ed0f4 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -1,4 +1,4 @@ -import bls from "@chainsafe/bls"; +import {SecretKey} from "@chainsafe/blst"; import {config as minimalConfig} from "@lodestar/config/default"; import { BeaconStateAllForks, @@ -55,7 +55,7 @@ export function generateState( opts.validators ?? (withPubkey ? Array.from({length: numValidators}, (_, i) => { - const sk = bls.SecretKey.fromBytes(Buffer.alloc(32, i + 1)); + const sk = SecretKey.fromBytes(Buffer.alloc(32, i + 1)); return generateValidator({ ...validatorOpts, pubkey: sk.toPublicKey().toBytes(), diff --git a/packages/cli/package.json b/packages/cli/package.json index 6a2f9d04644d..44db995d3a62 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -51,10 +51,9 @@ "blockchain" ], "dependencies": { - "@chainsafe/bls": "7.1.3", "@chainsafe/bls-keygen": "^0.4.0", "@chainsafe/bls-keystore": "^3.0.1", - "@chainsafe/blst": "^0.2.11", + "@chainsafe/blst": "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz", "@chainsafe/discv5": "^9.0.0", "@chainsafe/enr": "^3.0.0", "@chainsafe/persistent-merkle-tree": "^0.7.1", diff --git a/packages/cli/src/cmds/validator/blsToExecutionChange.ts b/packages/cli/src/cmds/validator/blsToExecutionChange.ts index 960662b108ea..3a2fabcc37eb 100644 --- a/packages/cli/src/cmds/validator/blsToExecutionChange.ts +++ b/packages/cli/src/cmds/validator/blsToExecutionChange.ts @@ -1,6 +1,5 @@ import {fromHexString} from "@chainsafe/ssz"; -import bls from "@chainsafe/bls"; -import {PointFormat} from "@chainsafe/bls/types"; +import {SecretKey} from "@chainsafe/blst"; import {computeSigningRoot} from "@lodestar/state-transition"; import {DOMAIN_BLS_TO_EXECUTION_CHANGE, ForkName} from "@lodestar/params"; import {createBeaconConfig} from "@lodestar/config"; @@ -68,8 +67,8 @@ like to choose for BLS To Execution Change.", throw new Error(`Validator pubkey ${publicKey} not found in state`); } - const blsPrivkey = bls.SecretKey.fromBytes(fromHexString(args.fromBlsPrivkey)); - const fromBlsPubkey = blsPrivkey.toPublicKey().toBytes(PointFormat.compressed); + const blsPrivkey = SecretKey.fromBytes(fromHexString(args.fromBlsPrivkey)); + const fromBlsPubkey = blsPrivkey.toPublicKey().toBytes(); const blsToExecutionChange: capella.BLSToExecutionChange = { validatorIndex: validator.index, diff --git a/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts b/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts index 2901fd6cdfb5..07f6cd5f9dbd 100644 --- a/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts +++ b/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import bls from "@chainsafe/bls"; +import {SecretKey} from "@chainsafe/blst"; import {SignerLocal, SignerType} from "@lodestar/validator"; import {LogLevel, Logger} from "@lodestar/utils"; import {lockFilepath, unlockFilepath} from "../../../util/lockfile.js"; @@ -67,7 +67,7 @@ export async function decryptKeystoreDefinitions( (secretKeyBytes: Uint8Array) => { const signer: SignerLocal = { type: SignerType.Local, - secretKey: bls.SecretKey.fromBytes(secretKeyBytes), + secretKey: SecretKey.fromBytes(secretKeyBytes), }; signers[index] = signer; diff --git a/packages/cli/src/cmds/validator/keymanager/impl.ts b/packages/cli/src/cmds/validator/keymanager/impl.ts index 36ded66976b1..b3b7c9f458bc 100644 --- a/packages/cli/src/cmds/validator/keymanager/impl.ts +++ b/packages/cli/src/cmds/validator/keymanager/impl.ts @@ -1,6 +1,6 @@ -import bls from "@chainsafe/bls"; import {Keystore} from "@chainsafe/bls-keystore"; import {fromHexString} from "@chainsafe/ssz"; +import {SecretKey} from "@chainsafe/blst"; import { DeleteRemoteKeyStatus, DeletionStatus, @@ -149,7 +149,7 @@ export class KeymanagerApi implements Api { decryptKeystores.queue( {keystoreStr, password}, async (secretKeyBytes: Uint8Array) => { - const secretKey = bls.SecretKey.fromBytes(secretKeyBytes); + const secretKey = SecretKey.fromBytes(secretKeyBytes); // Persist the key to disk for restarts, before adding to in-memory store // If the keystore exist and has a lock it will throw diff --git a/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts b/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts index 1f1c6cd6e79f..85b1702892ee 100644 --- a/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts +++ b/packages/cli/src/cmds/validator/keymanager/keystoreCache.ts @@ -1,8 +1,7 @@ import fs from "node:fs"; import path from "node:path"; -import bls from "@chainsafe/bls"; import {Keystore} from "@chainsafe/bls-keystore"; -import {PointFormat} from "@chainsafe/bls/types"; +import {SecretKey} from "@chainsafe/blst"; import {SignerLocal, SignerType} from "@lodestar/validator"; import {fromHex, toHex} from "@lodestar/utils"; import {writeFile600Perm} from "../../../util/file.js"; @@ -40,8 +39,8 @@ export async function loadKeystoreCache( const result: SignerLocal[] = []; for (const [index, k] of keystores.entries()) { const secretKeyBytes = Uint8Array.prototype.slice.call(secretKeyConcatenatedBytes, index * 32, (index + 1) * 32); - const secretKey = bls.SecretKey.fromBytes(secretKeyBytes); - const publicKey = secretKey.toPublicKey().toBytes(PointFormat.compressed); + const secretKey = SecretKey.fromBytes(secretKeyBytes); + const publicKey = secretKey.toPublicKey().toBytes(); if (toHex(publicKey) !== toHex(fromHex(k.pubkey))) { throw new Error( diff --git a/packages/cli/src/cmds/validator/signers/index.ts b/packages/cli/src/cmds/validator/signers/index.ts index be028461c0ee..cfd774b13a63 100644 --- a/packages/cli/src/cmds/validator/signers/index.ts +++ b/packages/cli/src/cmds/validator/signers/index.ts @@ -1,7 +1,6 @@ import path from "node:path"; -import bls from "@chainsafe/bls"; import {deriveEth2ValidatorKeys, deriveKeyFromMnemonic} from "@chainsafe/bls-keygen"; -import {toHexString} from "@chainsafe/ssz"; +import {SecretKey} from "@chainsafe/blst"; import {interopSecretKey} from "@lodestar/state-transition"; import {externalSignerGetKeys, Signer, SignerType} from "@lodestar/validator"; import {LogLevel, Logger, isValidHttpUrl} from "@lodestar/utils"; @@ -72,7 +71,7 @@ export async function getSignersFromArgs( const indexes = parseRange(args.mnemonicIndexes); return indexes.map((index) => ({ type: SignerType.Local, - secretKey: bls.SecretKey.fromBytes(deriveEth2ValidatorKeys(masterSK, index).signing), + secretKey: SecretKey.fromBytes(deriveEth2ValidatorKeys(masterSK, index).signing), })); } @@ -148,7 +147,7 @@ export async function getSignersFromArgs( export function getSignerPubkeyHex(signer: Signer): string { switch (signer.type) { case SignerType.Local: - return toHexString(signer.secretKey.toPublicKey().toBytes()); + return signer.secretKey.toPublicKey().toHex(); case SignerType.Remote: return signer.pubkey; diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index a08f33eb6bad..279076b619f2 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -1,5 +1,5 @@ import inquirer from "inquirer"; -import bls from "@chainsafe/bls"; +import {Signature} from "@chainsafe/blst"; import { computeEpochAtSlot, computeSigningRoot, @@ -8,7 +8,7 @@ import { } from "@lodestar/state-transition"; import {createBeaconConfig, BeaconConfig} from "@lodestar/config"; import {phase0, ssz, ValidatorIndex, Epoch} from "@lodestar/types"; -import {CliCommand, toHex} from "@lodestar/utils"; +import {CliCommand, fromHex, toHex} from "@lodestar/utils"; import {externalSignerPostSignature, SignableMessageType, Signer, SignerType} from "@lodestar/validator"; import {ApiClient, getClient} from "@lodestar/api"; import {ensure0xPrefix, YargsError, wrapError} from "../../util/index.js"; @@ -161,7 +161,7 @@ async function processVoluntaryExit( data: voluntaryExit, type: SignableMessageType.VOLUNTARY_EXIT, }); - signature = bls.Signature.fromHex(signatureHex); + signature = Signature.fromBytes(fromHex(signatureHex)); break; } default: diff --git a/packages/cli/src/util/format.ts b/packages/cli/src/util/format.ts index e673fa4d4408..01c2753193a4 100644 --- a/packages/cli/src/util/format.ts +++ b/packages/cli/src/util/format.ts @@ -1,5 +1,4 @@ -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/blst"; +import {PublicKey} from "@chainsafe/blst"; import {fromHexString} from "@chainsafe/ssz"; /** @@ -52,7 +51,7 @@ export function parseRange(range: string): number[] { export function assertValidPubkeysHex(pubkeysHex: string[]): void { for (const pubkeyHex of pubkeysHex) { const pubkeyBytes = fromHexString(pubkeyHex); - bls.PublicKey.fromBytes(pubkeyBytes, CoordType.jacobian, true); + PublicKey.fromBytes(pubkeyBytes, true); } } diff --git a/packages/cli/test/unit/cmds/validator/keymanager/keystoreCache.test.ts b/packages/cli/test/unit/cmds/validator/keymanager/keystoreCache.test.ts index 0f4d0126bb20..8d3dbb989b5d 100644 --- a/packages/cli/test/unit/cmds/validator/keymanager/keystoreCache.test.ts +++ b/packages/cli/test/unit/cmds/validator/keymanager/keystoreCache.test.ts @@ -3,7 +3,7 @@ import {randomBytes} from "node:crypto"; import {describe, it, expect, beforeEach, vi} from "vitest"; import tmp from "tmp"; import {Keystore} from "@chainsafe/bls-keystore"; -import bls from "@chainsafe/bls"; +import {SecretKey} from "@chainsafe/blst"; import {interopSecretKey} from "@lodestar/state-transition"; import {SignerLocal, SignerType} from "@lodestar/validator"; import {loadKeystoreCache, writeKeystoreCache} from "../../../../../src/cmds/validator/keymanager/keystoreCache.js"; @@ -27,7 +27,7 @@ describe("keystoreCache", () => { keystoreCacheFile = tmp.tmpNameSync({postfix: ".cache"}); for (let i = 0; i < numberOfSigners; i++) { - const secretKey = bls.SecretKey.fromBytes(interopSecretKey(i).toBytes()); + const secretKey = SecretKey.fromBytes(interopSecretKey(i).toBytes()); const keystorePath = tmp.tmpNameSync({postfix: ".json"}); const password = secretKey.toHex(); const keystore = await Keystore.create( diff --git a/packages/cli/test/utils/crucible/assertions/nodeAssertion.ts b/packages/cli/test/utils/crucible/assertions/nodeAssertion.ts index 69039accb182..9659cc34e7bf 100644 --- a/packages/cli/test/utils/crucible/assertions/nodeAssertion.ts +++ b/packages/cli/test/utils/crucible/assertions/nodeAssertion.ts @@ -1,5 +1,6 @@ -import type {SecretKey} from "@chainsafe/bls/types"; +import {SecretKey} from "@chainsafe/blst"; import {routes} from "@lodestar/api/beacon"; +import {toHex} from "@lodestar/utils"; import {AssertionResult, ValidatorClientKeys, Assertion, ValidatorClient} from "../interfaces.js"; import {arrayEquals} from "../utils/index.js"; import {neverMatcher} from "./matchers.js"; @@ -38,7 +39,7 @@ export const nodeAssertion: Assertion<"node", {health: number; keyManagerKeys: s } const expectedPublicKeys = node.validator - ? getAllKeys(node.validator.keys).map((k) => k.toPublicKey().toHex()) + ? getAllKeys(node.validator.keys).map((k) => toHex(k.toPublicKey().toBytes())) : []; if (!arrayEquals(keyManagerKeys.sort(), expectedPublicKeys.sort())) { diff --git a/packages/cli/test/utils/crucible/externalSignerServer.ts b/packages/cli/test/utils/crucible/externalSignerServer.ts index 25071d1ee545..bd8576d214c2 100644 --- a/packages/cli/test/utils/crucible/externalSignerServer.ts +++ b/packages/cli/test/utils/crucible/externalSignerServer.ts @@ -1,6 +1,6 @@ -import type {SecretKey} from "@chainsafe/bls/types"; import {fromHexString} from "@chainsafe/ssz"; import {fastify, FastifyInstance} from "fastify"; +import {SecretKey} from "@chainsafe/blst"; import {EXTERNAL_SIGNER_BASE_PORT} from "./constants.js"; /* eslint-disable no-console */ diff --git a/packages/cli/test/utils/crucible/interfaces.ts b/packages/cli/test/utils/crucible/interfaces.ts index c406d5a72432..c36f5db2d8f9 100644 --- a/packages/cli/test/utils/crucible/interfaces.ts +++ b/packages/cli/test/utils/crucible/interfaces.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import {ChildProcess} from "node:child_process"; -import type {SecretKey} from "@chainsafe/bls/types"; import {Web3} from "web3"; +import {SecretKey} from "@chainsafe/blst"; import {ApiClient} from "@lodestar/api"; import {ApiClient as KeyManagerApi} from "@lodestar/api/keymanager"; import {ChainForkConfig} from "@lodestar/config"; diff --git a/packages/flare/package.json b/packages/flare/package.json index 54a0dbec51f5..2ade4da3ddd1 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -58,7 +58,7 @@ "blockchain" ], "dependencies": { - "@chainsafe/bls": "7.1.3", + "@chainsafe/blst": "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz", "@chainsafe/bls-keygen": "^0.4.0", "@lodestar/api": "^1.19.0", "@lodestar/config": "^1.19.0", diff --git a/packages/flare/src/cmds/selfSlashAttester.ts b/packages/flare/src/cmds/selfSlashAttester.ts index 6c42372f45c7..8b43e6a92cb0 100644 --- a/packages/flare/src/cmds/selfSlashAttester.ts +++ b/packages/flare/src/cmds/selfSlashAttester.ts @@ -1,5 +1,4 @@ -import bls from "@chainsafe/bls"; -import type {SecretKey} from "@chainsafe/bls/types"; +import {SecretKey, aggregateSignatures} from "@chainsafe/blst"; import {getClient} from "@lodestar/api"; import {phase0, ssz} from "@lodestar/types"; import {config as chainConfig} from "@lodestar/config/default"; @@ -149,5 +148,5 @@ function signAttestationDataBigint( const signingRoot = computeSigningRoot(ssz.phase0.AttestationDataBigint, data, proposerDomain); const sigs = sks.map((sk) => sk.sign(signingRoot)); - return bls.Signature.aggregate(sigs).toBytes(); + return aggregateSignatures(sigs).toBytes(); } diff --git a/packages/flare/src/cmds/selfSlashProposer.ts b/packages/flare/src/cmds/selfSlashProposer.ts index 0b71d5558af8..8b11ff2ce4dc 100644 --- a/packages/flare/src/cmds/selfSlashProposer.ts +++ b/packages/flare/src/cmds/selfSlashProposer.ts @@ -1,4 +1,4 @@ -import type {SecretKey} from "@chainsafe/bls/types"; +import {SecretKey} from "@chainsafe/blst"; import {getClient} from "@lodestar/api"; import {phase0, ssz} from "@lodestar/types"; import {config as chainConfig} from "@lodestar/config/default"; diff --git a/packages/flare/src/util/deriveSecretKeys.ts b/packages/flare/src/util/deriveSecretKeys.ts index 272cf87c09c4..5017fc6e7508 100644 --- a/packages/flare/src/util/deriveSecretKeys.ts +++ b/packages/flare/src/util/deriveSecretKeys.ts @@ -1,6 +1,5 @@ -import bls from "@chainsafe/bls"; -import type {SecretKey} from "@chainsafe/bls/types"; import {deriveEth2ValidatorKeys, deriveKeyFromMnemonic} from "@chainsafe/bls-keygen"; +import {SecretKey} from "@chainsafe/blst"; import {interopSecretKey} from "@lodestar/state-transition"; import {CliCommandOptions} from "@lodestar/utils"; import {YargsError} from "./errors.js"; @@ -42,7 +41,7 @@ export function deriveSecretKeys(args: SecretKeysArgs): SecretKey[] { return indexes.map((index) => { const {signing} = deriveEth2ValidatorKeys(masterSK, index); - return bls.SecretKey.fromBytes(signing); + return SecretKey.fromBytes(signing); }); } diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 76b7f52325a3..d9ce361998ee 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -59,8 +59,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/as-sha256": "^0.4.1", - "@chainsafe/bls": "7.1.3", - "@chainsafe/blst": "^0.2.11", + "@chainsafe/blst": "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz", "@chainsafe/persistent-merkle-tree": "^0.7.1", "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.15.1", diff --git a/packages/state-transition/src/block/processDeposit.ts b/packages/state-transition/src/block/processDeposit.ts index dae37d5d0afe..7ade79fd7739 100644 --- a/packages/state-transition/src/block/processDeposit.ts +++ b/packages/state-transition/src/block/processDeposit.ts @@ -1,5 +1,4 @@ -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/bls/types"; +import {PublicKey, Signature, verify} from "@chainsafe/blst"; import {phase0, ssz} from "@lodestar/types"; import {verifyMerkleBranch} from "@lodestar/utils"; @@ -56,9 +55,9 @@ export function processDeposit(fork: ForkSeq, state: CachedBeaconStateAllForks, const signingRoot = computeSigningRoot(ssz.phase0.DepositMessage, depositMessage, domain); try { // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed - const publicKey = bls.PublicKey.fromBytes(pubkey, CoordType.affine, true); - const signature = bls.Signature.fromBytes(deposit.data.signature, CoordType.affine, true); - if (!signature.verify(publicKey, signingRoot)) { + const publicKey = PublicKey.fromBytes(pubkey, true); + const signature = Signature.fromBytes(deposit.data.signature, true); + if (!verify(signingRoot, publicKey, signature)) { return; } } catch (e) { diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 9565898eb09d..ae2e5786ef28 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -1,5 +1,4 @@ -import {CoordType} from "@chainsafe/bls/types"; -import bls from "@chainsafe/bls"; +import {PublicKey} from "@chainsafe/blst"; import {BLSSignature, CommitteeIndex, Epoch, Slot, ValidatorIndex, phase0, SyncPeriod} from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainConfig} from "@lodestar/config"; import { @@ -762,7 +761,7 @@ export class EpochCache { addPubkey(index: ValidatorIndex, pubkey: Uint8Array): void { this.pubkey2index.set(pubkey, index); - this.index2pubkey[index] = bls.PublicKey.fromBytes(pubkey, CoordType.jacobian); // Optimize for aggregation + this.index2pubkey[index] = PublicKey.fromBytes(pubkey); // Optimize for aggregation } getShufflingAtSlot(slot: Slot): EpochShuffling { diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index fdc343ff4ec7..0fd7a80fe990 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -1,5 +1,4 @@ -import {CoordType, PublicKey} from "@chainsafe/bls/types"; -import bls from "@chainsafe/bls"; +import {PublicKey} from "@chainsafe/blst"; import {ValidatorIndex} from "@lodestar/types"; import {BeaconStateAllForks} from "./types.js"; @@ -72,6 +71,6 @@ export function syncPubkeys( // Pubkeys must be checked for group + inf. This must be done only once when the validator deposit is processed. // Afterwards any public key is the state consider validated. // > Do not do any validation here - index2pubkey.push(bls.PublicKey.fromBytes(pubkey, CoordType.jacobian)); // Optimize for aggregation + index2pubkey.push(PublicKey.fromBytes(pubkey)); // Optimize for aggregation } } diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index 8b45152a3646..f4e637e5d665 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -1,5 +1,4 @@ -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/bls/types"; +import {PublicKey} from "@chainsafe/blst"; import {BeaconConfig} from "@lodestar/config"; import {loadState} from "../util/loadState/loadState.js"; import {EpochCache, EpochCacheImmutableData, EpochCacheOpts} from "./epochCache.js"; @@ -180,7 +179,7 @@ export function loadCachedBeaconState bls.PublicKey.fromBytes(pk, CoordType.jacobian)); + const pubkeysModObj = pubkeysMod.map((pk) => PublicKey.fromBytes(pk)); const pubkeys = Array.from({length: vc}, (_, i) => pubkeysMod[i % keypairsMod]); return {pubkeysMod, pubkeysModObj, pubkeys}; } diff --git a/packages/state-transition/test/perf/util/loadState/loadState.test.ts b/packages/state-transition/test/perf/util/loadState/loadState.test.ts index 5d40c64f6ab4..a8a1b1399dc5 100644 --- a/packages/state-transition/test/perf/util/loadState/loadState.test.ts +++ b/packages/state-transition/test/perf/util/loadState/loadState.test.ts @@ -1,6 +1,5 @@ -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/bls/types"; import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {PublicKey} from "@chainsafe/blst"; import {loadState} from "../../../../src/util/loadState/loadState.js"; import {createCachedBeaconState} from "../../../../src/cache/stateCache.js"; import {Index2PubkeyCache, PubkeyIndexMap} from "../../../../src/cache/pubkeyCache.js"; @@ -78,7 +77,7 @@ describe("loadState", function () { const validator = validators.getReadonly(validatorIndex); const pubkey = validator.pubkey; pubkey2index.set(pubkey, validatorIndex); - index2pubkey[validatorIndex] = bls.PublicKey.fromBytes(pubkey, CoordType.jacobian); + index2pubkey[validatorIndex] = PublicKey.fromBytes(pubkey); } // skip computimg shuffling in performance test because in reality we have a ShufflingCache // eslint-disable-next-line @typescript-eslint/explicit-function-return-type diff --git a/packages/state-transition/test/unit/constants.test.ts b/packages/state-transition/test/unit/constants.test.ts index 5b8cc66da73a..1cb17472218b 100644 --- a/packages/state-transition/test/unit/constants.test.ts +++ b/packages/state-transition/test/unit/constants.test.ts @@ -5,6 +5,6 @@ import {G2_POINT_AT_INFINITY} from "../../src/index.js"; describe("constants", () => { it("G2_POINT_AT_INFINITY", () => { const p2 = blst.Signature.fromBytes(G2_POINT_AT_INFINITY); - expect(p2.value.is_inf()).toBe(true); + expect(() => p2.sigValidate(true)).toThrow(); }); }); diff --git a/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts b/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts index 1a5b15e1b041..353b68195636 100644 --- a/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts +++ b/packages/state-transition/test/unit/signatureSets/signatureSets.test.ts @@ -1,7 +1,7 @@ import crypto from "node:crypto"; import {describe, it, expect} from "vitest"; -import bls from "@chainsafe/bls"; import {BitArray} from "@chainsafe/ssz"; +import {SecretKey} from "@chainsafe/blst"; import {config} from "@lodestar/config/default"; import {phase0, capella, ValidatorIndex, BLSSignature, ssz} from "@lodestar/types"; import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE} from "@lodestar/params"; @@ -61,7 +61,7 @@ describe("signatureSets", () => { exit: FAR_FUTURE_EPOCH, }); for (const validator of validators) { - validator.pubkey = bls.SecretKey.fromKeygen().toPublicKey().toBytes(); + validator.pubkey = SecretKey.fromKeygen(Buffer.alloc(32)).toPublicKey().toBytes(); } const state = generateCachedState(config, {validators}); diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 1a39f4d6c7ca..a9d483271680 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -57,7 +57,7 @@ "blockchain" ], "dependencies": { - "@chainsafe/bls": "7.1.3", + "@chainsafe/blst": "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz", "@chainsafe/bls-keystore": "^3.0.1", "@lodestar/params": "^1.19.0", "@lodestar/utils": "^1.19.0", diff --git a/packages/test-utils/src/keystores.ts b/packages/test-utils/src/keystores.ts index 2321fde30bd4..0014ac6c761f 100644 --- a/packages/test-utils/src/keystores.ts +++ b/packages/test-utils/src/keystores.ts @@ -1,5 +1,5 @@ -import bls from "@chainsafe/bls"; import {Keystore} from "@chainsafe/bls-keystore"; +import {SecretKey} from "@chainsafe/blst"; import {fromHex} from "@lodestar/utils"; /** @@ -10,7 +10,7 @@ export async function getKeystoresStr(password: string, secretKeys: string[]): P for (const secretKey of secretKeys) { const sk = fromHex(secretKey); - const pk = bls.SecretKey.fromBytes(sk).toPublicKey().toBytes(); + const pk = SecretKey.fromBytes(sk).toPublicKey().toBytes(); const keystore = await Keystore.create(password, sk, pk, ""); keystoresStr.push(keystore.stringify()); } diff --git a/packages/validator/package.json b/packages/validator/package.json index 7bd1cdae0ac7..c5edd8a566bb 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -45,7 +45,7 @@ "blockchain" ], "dependencies": { - "@chainsafe/bls": "7.1.3", + "@chainsafe/blst": "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz", "@chainsafe/ssz": "^0.15.1", "@lodestar/api": "^1.19.0", "@lodestar/config": "^1.19.0", diff --git a/packages/validator/src/services/externalSignerSync.ts b/packages/validator/src/services/externalSignerSync.ts index edfc0f5bc350..03db612249b4 100644 --- a/packages/validator/src/services/externalSignerSync.ts +++ b/packages/validator/src/services/externalSignerSync.ts @@ -1,6 +1,5 @@ -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/bls/types"; import {fromHexString} from "@chainsafe/ssz"; +import {PublicKey} from "@chainsafe/blst"; import {ChainForkConfig} from "@lodestar/config"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {toSafePrintableUrl} from "@lodestar/utils"; @@ -79,6 +78,6 @@ export function pollExternalSignerPubkeys( function assertValidPubkeysHex(pubkeysHex: string[]): void { for (const pubkeyHex of pubkeysHex) { const pubkeyBytes = fromHexString(pubkeyHex); - bls.PublicKey.fromBytes(pubkeyBytes, CoordType.jacobian, true); + PublicKey.fromBytes(pubkeyBytes, true); } } diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index 6cd9ed8dc065..42e4a0041c10 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -1,5 +1,5 @@ -import type {SecretKey} from "@chainsafe/bls/types"; import {BitArray, fromHexString, toHexString} from "@chainsafe/ssz"; +import {SecretKey} from "@chainsafe/blst"; import { computeEpochAtSlot, computeSigningRoot, diff --git a/packages/validator/src/types.ts b/packages/validator/src/types.ts index 9a2314269234..dd44fd49c61e 100644 --- a/packages/validator/src/types.ts +++ b/packages/validator/src/types.ts @@ -1,4 +1,4 @@ -import type {SecretKey} from "@chainsafe/bls/types"; +import {SecretKey} from "@chainsafe/blst"; import {BLSPubkey} from "@lodestar/types"; import {DatabaseController} from "@lodestar/db"; diff --git a/packages/validator/test/unit/services/attestation.test.ts b/packages/validator/test/unit/services/attestation.test.ts index e1254d1c6a52..0ffec323ee30 100644 --- a/packages/validator/test/unit/services/attestation.test.ts +++ b/packages/validator/test/unit/services/attestation.test.ts @@ -1,6 +1,6 @@ import {describe, it, expect, beforeAll, beforeEach, afterEach, vi} from "vitest"; -import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; +import {SecretKey} from "@chainsafe/blst"; import {ssz} from "@lodestar/types"; import {routes} from "@lodestar/api"; import {AttestationService, AttestationServiceOpts} from "../../../src/services/attestation.js"; @@ -28,7 +28,7 @@ describe("AttestationService", function () { let pubkeys: Uint8Array[]; // Initialize pubkeys in before() so bls is already initialized beforeAll(() => { - const secretKeys = Array.from({length: 1}, (_, i) => bls.SecretKey.fromBytes(Buffer.alloc(32, i + 1))); + const secretKeys = Array.from({length: 1}, (_, i) => SecretKey.fromBytes(Buffer.alloc(32, i + 1))); pubkeys = secretKeys.map((sk) => sk.toPublicKey().toBytes()); validatorStore.votingPubkeys.mockReturnValue(pubkeys.map(toHexString)); validatorStore.hasVotingPubkey.mockReturnValue(true); diff --git a/packages/validator/test/unit/services/attestationDuties.test.ts b/packages/validator/test/unit/services/attestationDuties.test.ts index f7154a3a174e..ad54c0735eea 100644 --- a/packages/validator/test/unit/services/attestationDuties.test.ts +++ b/packages/validator/test/unit/services/attestationDuties.test.ts @@ -1,7 +1,7 @@ import {describe, it, expect, beforeAll, vi, Mocked, beforeEach, afterEach} from "vitest"; import {toBufferBE} from "bigint-buffer"; -import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; +import {SecretKey} from "@chainsafe/blst"; import {chainConfig} from "@lodestar/config/default"; import {routes} from "@lodestar/api"; import {ssz} from "@lodestar/types"; @@ -37,7 +37,7 @@ describe("AttestationDutiesService", function () { }; beforeAll(async () => { - const secretKeys = [bls.SecretKey.fromBytes(toBufferBE(BigInt(98), 32))]; + const secretKeys = [SecretKey.fromBytes(toBufferBE(BigInt(98), 32))]; pubkeys = secretKeys.map((sk) => sk.toPublicKey().toBytes()); validatorStore = await initValidatorStore(secretKeys, api, chainConfig); }); diff --git a/packages/validator/test/unit/services/block.test.ts b/packages/validator/test/unit/services/block.test.ts index 6864e62906d1..641ca620a1b5 100644 --- a/packages/validator/test/unit/services/block.test.ts +++ b/packages/validator/test/unit/services/block.test.ts @@ -1,6 +1,6 @@ import {describe, it, expect, beforeAll, beforeEach, afterEach, vi} from "vitest"; -import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; +import {SecretKey} from "@chainsafe/blst"; import {createChainForkConfig} from "@lodestar/config"; import {config as mainnetConfig} from "@lodestar/config/default"; import {sleep} from "@lodestar/utils"; @@ -25,7 +25,7 @@ describe("BlockDutiesService", function () { const config = createChainForkConfig(mainnetConfig); beforeAll(() => { - const secretKeys = Array.from({length: 2}, (_, i) => bls.SecretKey.fromBytes(Buffer.alloc(32, i + 1))); + const secretKeys = Array.from({length: 2}, (_, i) => SecretKey.fromBytes(Buffer.alloc(32, i + 1))); pubkeys = secretKeys.map((sk) => sk.toPublicKey().toBytes()); validatorStore.votingPubkeys.mockReturnValue(pubkeys.map(toHexString)); }); diff --git a/packages/validator/test/unit/services/blockDuties.test.ts b/packages/validator/test/unit/services/blockDuties.test.ts index c1edcc955b2d..bc4053b704c8 100644 --- a/packages/validator/test/unit/services/blockDuties.test.ts +++ b/packages/validator/test/unit/services/blockDuties.test.ts @@ -1,7 +1,7 @@ import {describe, it, expect, beforeAll, beforeEach, afterEach, vi} from "vitest"; import {toBufferBE} from "bigint-buffer"; -import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; +import {SecretKey} from "@chainsafe/blst"; import {routes} from "@lodestar/api"; import {chainConfig} from "@lodestar/config/default"; import {toHex} from "@lodestar/utils"; @@ -19,7 +19,7 @@ describe("BlockDutiesService", function () { let pubkeys: Uint8Array[]; // Initialize pubkeys in before() so bls is already initialized beforeAll(async () => { - const secretKeys = Array.from({length: 3}, (_, i) => bls.SecretKey.fromBytes(toBufferBE(BigInt(i + 1), 32))); + const secretKeys = Array.from({length: 3}, (_, i) => SecretKey.fromBytes(toBufferBE(BigInt(i + 1), 32))); pubkeys = secretKeys.map((sk) => sk.toPublicKey().toBytes()); validatorStore = await initValidatorStore(secretKeys, api); }); diff --git a/packages/validator/test/unit/services/externalSignerSync.test.ts b/packages/validator/test/unit/services/externalSignerSync.test.ts index 5fdf7d5ae5b2..8dc7bff7e469 100644 --- a/packages/validator/test/unit/services/externalSignerSync.test.ts +++ b/packages/validator/test/unit/services/externalSignerSync.test.ts @@ -1,7 +1,6 @@ import {MockedFunction, afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi} from "vitest"; import {toBufferBE} from "bigint-buffer"; -import bls from "@chainsafe/bls"; -import {SecretKey} from "@chainsafe/bls/types"; +import {SecretKey} from "@chainsafe/blst"; import {createChainForkConfig} from "@lodestar/config"; import {chainConfig} from "@lodestar/config/default"; import {ExternalSignerOptions, pollExternalSignerPubkeys} from "../../../src/services/externalSignerSync.js"; @@ -32,7 +31,7 @@ describe("External signer sync", () => { beforeAll(() => { vi.useFakeTimers(); - secretKeys = Array.from({length: 3}, (_, i) => bls.SecretKey.fromBytes(toBufferBE(BigInt(i + 1), 32))); + secretKeys = Array.from({length: 3}, (_, i) => SecretKey.fromBytes(toBufferBE(BigInt(i + 1), 32))); pubkeys = secretKeys.map((sk) => sk.toPublicKey().toHex()); externalSignerGetKeysStub = vi.mocked(externalSignerGetKeys); }); diff --git a/packages/validator/test/unit/services/indicesService.test.ts b/packages/validator/test/unit/services/indicesService.test.ts index b94ec6fa398a..8332bac7cfd6 100644 --- a/packages/validator/test/unit/services/indicesService.test.ts +++ b/packages/validator/test/unit/services/indicesService.test.ts @@ -1,7 +1,7 @@ import {describe, it, expect, beforeAll} from "vitest"; import {toBufferBE} from "bigint-buffer"; -import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; +import {SecretKey} from "@chainsafe/blst"; import {getApiClientStub} from "../../utils/apiStub.js"; import {testLogger} from "../../utils/logger.js"; import {IndicesService} from "../../../src/services/indices.js"; @@ -14,8 +14,8 @@ describe("IndicesService", function () { beforeAll(() => { const secretKeys = [ - bls.SecretKey.fromBytes(toBufferBE(BigInt(98), 32)), - bls.SecretKey.fromBytes(toBufferBE(BigInt(99), 32)), + SecretKey.fromBytes(toBufferBE(BigInt(98), 32)), + SecretKey.fromBytes(toBufferBE(BigInt(99), 32)), ]; pubkeys = secretKeys.map((sk) => sk.toPublicKey().toBytes()); }); diff --git a/packages/validator/test/unit/services/syncCommitteDuties.test.ts b/packages/validator/test/unit/services/syncCommitteDuties.test.ts index c44360485d09..52f2071f9102 100644 --- a/packages/validator/test/unit/services/syncCommitteDuties.test.ts +++ b/packages/validator/test/unit/services/syncCommitteDuties.test.ts @@ -1,8 +1,8 @@ import {describe, it, expect, beforeAll, beforeEach, afterEach} from "vitest"; import {when} from "vitest-when"; import {toBufferBE} from "bigint-buffer"; -import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; +import {SecretKey} from "@chainsafe/blst"; import {createChainForkConfig} from "@lodestar/config"; import {config as mainnetConfig} from "@lodestar/config/default"; import {routes} from "@lodestar/api"; @@ -43,8 +43,8 @@ describe("SyncCommitteeDutiesService", function () { beforeAll(async () => { const secretKeys = [ - bls.SecretKey.fromBytes(toBufferBE(BigInt(98), 32)), - bls.SecretKey.fromBytes(toBufferBE(BigInt(99), 32)), + SecretKey.fromBytes(toBufferBE(BigInt(98), 32)), + SecretKey.fromBytes(toBufferBE(BigInt(99), 32)), ]; pubkeys = secretKeys.map((sk) => sk.toPublicKey().toBytes()); validatorStore = await initValidatorStore(secretKeys, api, altair0Config); diff --git a/packages/validator/test/unit/services/syncCommittee.test.ts b/packages/validator/test/unit/services/syncCommittee.test.ts index b6cba32fc96b..c65912f12e9a 100644 --- a/packages/validator/test/unit/services/syncCommittee.test.ts +++ b/packages/validator/test/unit/services/syncCommittee.test.ts @@ -1,6 +1,6 @@ import {describe, it, expect, beforeAll, beforeEach, afterEach, vi} from "vitest"; -import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; +import {SecretKey} from "@chainsafe/blst"; import {createChainForkConfig} from "@lodestar/config"; import {config as mainnetConfig} from "@lodestar/config/default"; import {ssz} from "@lodestar/types"; @@ -37,7 +37,7 @@ describe("SyncCommitteeService", function () { }); beforeAll(() => { - const secretKeys = Array.from({length: 1}, (_, i) => bls.SecretKey.fromBytes(Buffer.alloc(32, i + 1))); + const secretKeys = Array.from({length: 1}, (_, i) => SecretKey.fromBytes(Buffer.alloc(32, i + 1))); pubkeys = secretKeys.map((sk) => sk.toPublicKey().toBytes()); validatorStore.votingPubkeys.mockReturnValue(pubkeys.map(toHexString)); validatorStore.hasVotingPubkey.mockReturnValue(true); diff --git a/packages/validator/test/unit/validatorStore.test.ts b/packages/validator/test/unit/validatorStore.test.ts index 3f7f0792f378..3d8d88ee3e64 100644 --- a/packages/validator/test/unit/validatorStore.test.ts +++ b/packages/validator/test/unit/validatorStore.test.ts @@ -1,7 +1,7 @@ import {describe, it, expect, beforeEach, afterEach, vi} from "vitest"; import {toBufferBE} from "bigint-buffer"; -import bls from "@chainsafe/bls"; import {toHexString, fromHexString} from "@chainsafe/ssz"; +import {SecretKey} from "@chainsafe/blst"; import {chainConfig} from "@lodestar/config/default"; import {bellatrix} from "@lodestar/types"; import {routes} from "@lodestar/api"; @@ -92,7 +92,7 @@ describe("ValidatorStore", function () { }); }); -const secretKeys = Array.from({length: 3}, (_, i) => bls.SecretKey.fromBytes(toBufferBE(BigInt(i + 1), 32))); +const secretKeys = Array.from({length: 3}, (_, i) => SecretKey.fromBytes(toBufferBE(BigInt(i + 1), 32))); const pubkeys = secretKeys.map((sk) => sk.toPublicKey().toBytes()); const valRegF00G100 = { diff --git a/packages/validator/test/utils/validatorStore.ts b/packages/validator/test/utils/validatorStore.ts index 5fe530ea0cfe..d3037ce522b9 100644 --- a/packages/validator/test/utils/validatorStore.ts +++ b/packages/validator/test/utils/validatorStore.ts @@ -1,4 +1,4 @@ -import {SecretKey} from "@chainsafe/bls/types"; +import {SecretKey} from "@chainsafe/blst"; import {ApiClient} from "@lodestar/api"; import {chainConfig} from "@lodestar/config/default"; import {createBeaconConfig, ChainConfig} from "@lodestar/config"; diff --git a/yarn.lock b/yarn.lock index 19f39c58ea4f..da406925d8ab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -306,14 +306,10 @@ "@chainsafe/bls-keygen" "^0.4.0" bls-eth-wasm "^0.4.8" -"@chainsafe/blst@^0.2.11": - version "0.2.11" - resolved "https://registry.yarnpkg.com/@chainsafe/blst/-/blst-0.2.11.tgz#5ec85cd663592819d1dc51127e75dfd834250e3d" - integrity sha512-URyOLq5GtxBoxibOnd2pgLydCy0UZzbiIIBcsRAvGxAsRzjZL04TsQfwRkz5aphU3a1ebeRoMmI/HHyMCiFSQg== - dependencies: - "@types/tar" "^6.1.4" - node-fetch "^2.6.1" - node-gyp "^8.4.0" +"@chainsafe/blst@https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz": + version "0.0.0" + uid "3aa6b47e083234520767968b1dc7f28453b70e04" + resolved "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz#3aa6b47e083234520767968b1dc7f28453b70e04" "@chainsafe/discv5@^9.0.0": version "9.0.0" @@ -1271,7 +1267,7 @@ rfdc "^1.3.0" yaml "^2.2.2" -"@gar/promisify@^1.0.1", "@gar/promisify@^1.1.3": +"@gar/promisify@^1.1.3": version "1.1.3" resolved "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz" integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw== @@ -2013,14 +2009,6 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" -"@npmcli/fs@^1.0.0": - version "1.1.1" - resolved "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz" - integrity sha512-8KG5RD0GVP4ydEzRn/I4BNDuxDtqVbOdm8675T49OIG/NGhaK0pjPX7ZcDlvKYbA+ulvVK3ztfcF4uBdOxuJbQ== - dependencies: - "@gar/promisify" "^1.0.1" - semver "^7.3.5" - "@npmcli/fs@^2.1.0": version "2.1.0" resolved "https://registry.yarnpkg.com/@npmcli/fs/-/fs-2.1.0.tgz#f2a21c28386e299d1a9fae8051d35ad180e33109" @@ -2058,14 +2046,6 @@ npm-bundled "^3.0.0" npm-normalize-package-bin "^3.0.0" -"@npmcli/move-file@^1.0.1": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@npmcli/move-file/-/move-file-1.1.2.tgz#1a82c3e372f7cae9253eb66d72543d6b8685c674" - integrity sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg== - dependencies: - mkdirp "^1.0.4" - rimraf "^3.0.2" - "@npmcli/move-file@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@npmcli/move-file/-/move-file-2.0.0.tgz#417f585016081a0184cef3e38902cd917a9bbd02" @@ -2788,11 +2768,6 @@ dependencies: defer-to-connect "^2.0.1" -"@tootallnate/once@1": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" - integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== - "@tootallnate/once@2": version "2.0.0" resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf" @@ -3587,7 +3562,7 @@ JSONStream@^1.3.5: jsonparse "^1.2.0" through ">=2.2.7 <3" -abbrev@1, abbrev@^1.0.0: +abbrev@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== @@ -3671,7 +3646,7 @@ agent-base@^7.0.2, agent-base@^7.1.0: dependencies: debug "^4.3.4" -agentkeepalive@^4.1.3, agentkeepalive@^4.2.1: +agentkeepalive@^4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.2.1.tgz#a7975cbb9f83b367f06c90cc51ff28fe7d499717" integrity sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA== @@ -4507,30 +4482,6 @@ cac@^6.7.14: resolved "https://registry.yarnpkg.com/cac/-/cac-6.7.14.tgz#804e1e6f506ee363cb0e3ccbb09cad5dd9870959" integrity sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ== -cacache@^15.2.0: - version "15.3.0" - resolved "https://registry.yarnpkg.com/cacache/-/cacache-15.3.0.tgz#dc85380fb2f556fe3dda4c719bfa0ec875a7f1eb" - integrity sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ== - dependencies: - "@npmcli/fs" "^1.0.0" - "@npmcli/move-file" "^1.0.1" - chownr "^2.0.0" - fs-minipass "^2.0.0" - glob "^7.1.4" - infer-owner "^1.0.4" - lru-cache "^6.0.0" - minipass "^3.1.1" - minipass-collect "^1.0.2" - minipass-flush "^1.0.5" - minipass-pipeline "^1.2.2" - mkdirp "^1.0.3" - p-map "^4.0.0" - promise-inflight "^1.0.1" - rimraf "^3.0.2" - ssri "^8.0.1" - tar "^6.0.2" - unique-filename "^1.1.1" - cacache@^16.1.0: version "16.1.1" resolved "https://registry.yarnpkg.com/cacache/-/cacache-16.1.1.tgz#4e79fb91d3efffe0630d5ad32db55cc1b870669c" @@ -5745,7 +5696,7 @@ enabled@2.0.x: resolved "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz" integrity sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ== -encoding@^0.1.12, encoding@^0.1.13: +encoding@^0.1.13: version "0.1.13" resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.13.tgz#56574afdd791f54a8e9b2785c0582a2d26210fa9" integrity sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A== @@ -7328,15 +7279,6 @@ http-errors@2.0.0: statuses "2.0.1" toidentifier "1.0.1" -http-proxy-agent@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a" - integrity sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg== - dependencies: - "@tootallnate/once" "1" - agent-base "6" - debug "4" - http-proxy-agent@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz#5129800203520d434f142bc78ff3c170800f2b43" @@ -8917,28 +8859,6 @@ make-fetch-happen@^11.0.0, make-fetch-happen@^11.0.1, make-fetch-happen@^11.1.1: socks-proxy-agent "^7.0.0" ssri "^10.0.0" -make-fetch-happen@^9.1.0: - version "9.1.0" - resolved "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-9.1.0.tgz" - integrity sha512-+zopwDy7DNknmwPQplem5lAZX/eCOzSvSNNcSKm5eVwTkOBzoktEfXsa9L23J/GIRhxRsaxzkPEhrJEpE2F4Gg== - dependencies: - agentkeepalive "^4.1.3" - cacache "^15.2.0" - http-cache-semantics "^4.1.0" - http-proxy-agent "^4.0.1" - https-proxy-agent "^5.0.0" - is-lambda "^1.0.1" - lru-cache "^6.0.0" - minipass "^3.1.3" - minipass-collect "^1.0.2" - minipass-fetch "^1.3.2" - minipass-flush "^1.0.5" - minipass-pipeline "^1.2.4" - negotiator "^0.6.2" - promise-retry "^2.0.1" - socks-proxy-agent "^6.0.0" - ssri "^8.0.0" - map-obj@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d" @@ -9190,17 +9110,6 @@ minipass-collect@^1.0.2: dependencies: minipass "^3.0.0" -minipass-fetch@^1.3.2: - version "1.4.1" - resolved "https://registry.yarnpkg.com/minipass-fetch/-/minipass-fetch-1.4.1.tgz#d75e0091daac1b0ffd7e9d41629faff7d0c1f1b6" - integrity sha512-CGH1eblLq26Y15+Azk7ey4xh0J/XfJfrCox5LDJiKqI2Q2iwOLOKrlmIaODiSQS8d18jalF6y2K2ePUm0CmShw== - dependencies: - minipass "^3.1.0" - minipass-sized "^1.0.3" - minizlib "^2.0.0" - optionalDependencies: - encoding "^0.1.12" - minipass-fetch@^2.0.3: version "2.1.0" resolved "https://registry.yarnpkg.com/minipass-fetch/-/minipass-fetch-2.1.0.tgz#ca1754a5f857a3be99a9271277246ac0b44c3ff8" @@ -9238,7 +9147,7 @@ minipass-json-stream@^1.0.1: jsonparse "^1.3.1" minipass "^3.0.0" -minipass-pipeline@^1.2.2, minipass-pipeline@^1.2.4: +minipass-pipeline@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz#68472f79711c084657c067c5c6ad93cddea8214c" integrity sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A== @@ -9252,7 +9161,7 @@ minipass-sized@^1.0.3: dependencies: minipass "^3.0.0" -minipass@^3.0.0, minipass@^3.1.0, minipass@^3.1.1, minipass@^3.1.3, minipass@^3.1.6: +minipass@^3.0.0, minipass@^3.1.1, minipass@^3.1.6: version "3.3.6" resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.6.tgz#7bba384db3a1520d18c9c0e5251c3444e95dd94a" integrity sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw== @@ -9274,7 +9183,7 @@ minipass@^5.0.0: resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.0.4.tgz#dbce03740f50a4786ba994c1fb908844d27b038c" integrity sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ== -minizlib@^2.0.0, minizlib@^2.1.1, minizlib@^2.1.2: +minizlib@^2.1.1, minizlib@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" integrity sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg== @@ -9466,7 +9375,7 @@ natural-compare@^1.4.0: resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= -negotiator@^0.6.2, negotiator@^0.6.3: +negotiator@^0.6.3: version "0.6.3" resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz" integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== @@ -9529,22 +9438,6 @@ node-gyp-build@^4.3.0: resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.5.0.tgz#7a64eefa0b21112f89f58379da128ac177f20e40" integrity sha512-2iGbaQBV+ITgCz76ZEjmhUKAKVf7xfY1sRl4UiKQspfZMH2h06SyhNsnSVy50cwkFQDGLyif6m/6uFXHkOZ6rg== -node-gyp@^8.4.0: - version "8.4.1" - resolved "https://registry.npmjs.org/node-gyp/-/node-gyp-8.4.1.tgz" - integrity sha512-olTJRgUtAb/hOXG0E93wZDs5YiJlgbXxTwQAFHyNlRsXQnYzUaF2aGgujZbw+hR8aF4ZG/rST57bWMWD16jr9w== - dependencies: - env-paths "^2.2.0" - glob "^7.1.4" - graceful-fs "^4.2.6" - make-fetch-happen "^9.1.0" - nopt "^5.0.0" - npmlog "^6.0.0" - rimraf "^3.0.2" - semver "^7.3.5" - tar "^6.1.2" - which "^2.0.2" - node-gyp@^9.0.0, node-gyp@^9.4.0: version "9.4.1" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-9.4.1.tgz#8a1023e0d6766ecb52764cc3a734b36ff275e185" @@ -9629,13 +9522,6 @@ node-stdlib-browser@^1.2.0: util "^0.12.4" vm-browserify "^1.0.1" -nopt@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/nopt/-/nopt-5.0.0.tgz#530942bb58a512fccafe53fe210f13a25355dc88" - integrity sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ== - dependencies: - abbrev "1" - nopt@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/nopt/-/nopt-6.0.0.tgz#245801d8ebf409c6df22ab9d95b65e1309cdb16d" @@ -11513,15 +11399,6 @@ snappyjs@^0.7.0: resolved "https://registry.yarnpkg.com/snappyjs/-/snappyjs-0.7.0.tgz#6096eac06382700ae7fdefa579dea5e2aa20f51c" integrity sha512-u5iEEXkMe2EInQio6Wv9LWHOQYRDbD2O9hzS27GpT/lwfIQhTCnHCTqedqHIHe9ZcvQo+9au6vngQayipz1NYw== -socks-proxy-agent@^6.0.0: - version "6.1.1" - resolved "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.1.1.tgz" - integrity sha512-t8J0kG3csjA4g6FTbsMOWws+7R7vuRC8aQ/wy3/1OWmsgwA68zs/+cExQ0koSitUDXqhufF/YJr9wtNMZHw5Ew== - dependencies: - agent-base "^6.0.2" - debug "^4.3.1" - socks "^2.6.1" - socks-proxy-agent@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-7.0.0.tgz#dc069ecf34436621acb41e3efa66ca1b5fed15b6" @@ -11540,7 +11417,7 @@ socks-proxy-agent@^8.0.2: debug "^4.3.4" socks "^2.7.1" -socks@^2.6.1, socks@^2.6.2, socks@^2.7.1: +socks@^2.6.2, socks@^2.7.1: version "2.7.1" resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.1.tgz#d8e651247178fde79c0663043e07240196857d55" integrity sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ== @@ -11676,13 +11553,6 @@ ssri@^10.0.0, ssri@^10.0.1: dependencies: minipass "^4.0.0" -ssri@^8.0.0, ssri@^8.0.1: - version "8.0.1" - resolved "https://registry.yarnpkg.com/ssri/-/ssri-8.0.1.tgz#638e4e439e2ffbd2cd289776d5ca457c4f51a2af" - integrity sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ== - dependencies: - minipass "^3.1.1" - ssri@^9.0.0, ssri@^9.0.1: version "9.0.1" resolved "https://registry.yarnpkg.com/ssri/-/ssri-9.0.1.tgz#544d4c357a8d7b71a19700074b6883fcb4eae057" @@ -12089,7 +11959,7 @@ tar@6.1.11: mkdirp "^1.0.3" yallist "^4.0.0" -tar@^6.0.2, tar@^6.1.11, tar@^6.1.13, tar@^6.1.2: +tar@^6.1.11, tar@^6.1.13, tar@^6.1.2: version "6.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a" integrity sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A== From 0b31779ed96eebaf0b4cdf06a8d9b5f53ee04153 Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 17 Jun 2024 18:53:25 -0400 Subject: [PATCH 02/18] chore: update blst --- yarn.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index da406925d8ab..e417bbd07e00 100644 --- a/yarn.lock +++ b/yarn.lock @@ -308,8 +308,8 @@ "@chainsafe/blst@https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz": version "0.0.0" - uid "3aa6b47e083234520767968b1dc7f28453b70e04" - resolved "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz#3aa6b47e083234520767968b1dc7f28453b70e04" + uid "4d6fce598429e888ad4fd49d7f4b7234537be5b6" + resolved "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz#4d6fce598429e888ad4fd49d7f4b7234537be5b6" "@chainsafe/discv5@^9.0.0": version "9.0.0" From ec1259bb8cab8c4aca5a19c6d4703a0475a248c4 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 18 Jun 2024 14:19:02 -0400 Subject: [PATCH 03/18] chore: try async blst verifier --- .../beacon-node/src/chain/bls/asyncBlst.ts | 91 +++++++++++++++++++ packages/beacon-node/src/chain/bls/index.ts | 1 + .../beacon-node/src/chain/bls/maybeBatch.ts | 40 +++++++- packages/beacon-node/src/chain/bls/utils.ts | 22 ++++- packages/beacon-node/src/chain/chain.ts | 12 ++- yarn.lock | 4 +- 6 files changed, 161 insertions(+), 9 deletions(-) create mode 100644 packages/beacon-node/src/chain/bls/asyncBlst.ts diff --git a/packages/beacon-node/src/chain/bls/asyncBlst.ts b/packages/beacon-node/src/chain/bls/asyncBlst.ts new file mode 100644 index 000000000000..892fd5a80162 --- /dev/null +++ b/packages/beacon-node/src/chain/bls/asyncBlst.ts @@ -0,0 +1,91 @@ +import {PublicKey, Signature, aggregatePublicKeysAsync, aggregateSignaturesAsync, verifyAsync} from "@chainsafe/blst"; +import {ISignatureSet} from "@lodestar/state-transition"; +import {Metrics} from "../../metrics/index.js"; +import {IBlsVerifier} from "./interface.js"; +import {verifySignatureSetsMaybeBatchAsync} from "./maybeBatch.js"; +import {getAggregatedPubkeyAsync, getAggregatedPubkeysCount} from "./utils.js"; + +export class BlsAsyncBlstVerifier implements IBlsVerifier { + private readonly metrics: Metrics | null; + + constructor({metrics = null}: {metrics: Metrics | null}) { + this.metrics = metrics; + } + + async verifySignatureSets(sets: ISignatureSet[]): Promise { + this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets)); + + const setsAggregated = await Promise.all( + sets.map(async (set) => ({ + publicKey: await getAggregatedPubkeyAsync(set), + message: set.signingRoot, + signature: set.signature, + })) + ); + + // Count time after aggregating + const timer = this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.startTimer(); + const isValid = await verifySignatureSetsMaybeBatchAsync(setsAggregated); + + // Don't use a try/catch, only count run without exceptions + if (timer) { + timer(); + } + + return isValid; + } + + async verifySignatureSetsSameMessage( + sets: {publicKey: PublicKey; signature: Uint8Array}[], + message: Uint8Array + ): Promise { + const timer = this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.startTimer(); + const pubkey = await aggregatePublicKeysAsync(sets.map((set) => set.publicKey)); + let isAllValid = true; + // validate signature = true + const signatures = sets.map((set) => { + try { + return Signature.fromBytes(set.signature, true); + } catch (_) { + // at least one set has malformed signature + isAllValid = false; + return null; + } + }); + + if (isAllValid) { + const signature = await aggregateSignaturesAsync(signatures as Signature[]); + isAllValid = await verifyAsync(message, pubkey, signature); + } + + let result: boolean[]; + if (isAllValid) { + result = sets.map(() => true); + } else { + result = await Promise.all( + sets.map(async (set, i) => { + const sig = signatures[i]; + if (sig === null) { + return false; + } + return verifyAsync(message, set.publicKey, sig); + }) + ); + } + + if (timer) { + timer(); + } + + return result; + } + + async close(): Promise { + // nothing to do + } + + canAcceptWork(): boolean { + // Since sigs are verified blocking the main thread, there's no mechanism to throttle + return true; + } +} diff --git a/packages/beacon-node/src/chain/bls/index.ts b/packages/beacon-node/src/chain/bls/index.ts index f9898b13776b..62ed6c257c25 100644 --- a/packages/beacon-node/src/chain/bls/index.ts +++ b/packages/beacon-node/src/chain/bls/index.ts @@ -2,3 +2,4 @@ export type {IBlsVerifier} from "./interface.js"; export type {BlsMultiThreadWorkerPoolModules, JobQueueItemType} from "./multithread/index.js"; export {BlsMultiThreadWorkerPool} from "./multithread/index.js"; export {BlsSingleThreadVerifier} from "./singleThread.js"; +export {BlsAsyncBlstVerifier} from "./asyncBlst.js"; diff --git a/packages/beacon-node/src/chain/bls/maybeBatch.ts b/packages/beacon-node/src/chain/bls/maybeBatch.ts index e300b8ff8b76..76dd35720b78 100644 --- a/packages/beacon-node/src/chain/bls/maybeBatch.ts +++ b/packages/beacon-node/src/chain/bls/maybeBatch.ts @@ -1,4 +1,10 @@ -import {PublicKey, Signature, verify, verifyMultipleAggregateSignatures} from "@chainsafe/blst"; +import { + PublicKey, + Signature, + verify, + verifyMultipleAggregateSignatures, + verifyMultipleAggregateSignaturesAsync, +} from "@chainsafe/blst"; const MIN_SET_COUNT_TO_BATCH = 2; @@ -43,3 +49,35 @@ export function verifySignatureSetsMaybeBatch(sets: SignatureSetDeserialized[]): return false; } } + +export async function verifySignatureSetsMaybeBatchAsync(sets: SignatureSetDeserialized[]): Promise { + try { + if (sets.length >= MIN_SET_COUNT_TO_BATCH) { + return await verifyMultipleAggregateSignaturesAsync( + sets.map((s) => ({ + pk: s.publicKey, + msg: s.message, + // true = validate signature + sig: Signature.fromBytes(s.signature, true), + })) + ); + } + + // .every on an empty array returns true + if (sets.length === 0) { + throw Error("Empty signature set"); + } + + // If too few signature sets verify them without batching + return sets.every((set) => { + // true = validate signature + const sig = Signature.fromBytes(set.signature, true); + return verify(set.message, set.publicKey, sig); + }); + } catch (_) { + // A signature could be malformed, in that case fromBytes throws error + // blst-ts `verifyMultipleSignatures` is also a fallible operation if mul_n_aggregate fails + // see https://github.com/ChainSafe/blst-ts/blob/b1ba6333f664b08e5c50b2b0d18c4f079203962b/src/lib.ts#L291 + return false; + } +} diff --git a/packages/beacon-node/src/chain/bls/utils.ts b/packages/beacon-node/src/chain/bls/utils.ts index 63f2bdd80458..52a72508858d 100644 --- a/packages/beacon-node/src/chain/bls/utils.ts +++ b/packages/beacon-node/src/chain/bls/utils.ts @@ -1,4 +1,4 @@ -import {PublicKey, aggregatePublicKeys} from "@chainsafe/blst"; +import {PublicKey, aggregatePublicKeys, aggregatePublicKeysAsync} from "@chainsafe/blst"; import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; import {Metrics} from "../../metrics/metrics.js"; @@ -19,6 +19,26 @@ export function getAggregatedPubkey(signatureSet: ISignatureSet, metrics: Metric } } +export async function getAggregatedPubkeyAsync( + signatureSet: ISignatureSet, + metrics: Metrics | null = null +): Promise { + switch (signatureSet.type) { + case SignatureSetType.single: + return signatureSet.pubkey; + + case SignatureSetType.aggregate: { + const timer = metrics?.blsThreadPool.pubkeysAggregationMainThreadDuration.startTimer(); + const pubkeys = await aggregatePublicKeysAsync(signatureSet.pubkeys); + timer?.(); + return pubkeys; + } + + default: + throw Error("Unknown signature set type"); + } +} + export function getAggregatedPubkeysCount(signatureSets: ISignatureSet[]): number { let pubkeysConut = 0; for (const set of signatureSets) { diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 2f58962f3cc5..846cea8e1b3f 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -57,7 +57,7 @@ import {IChainOptions} from "./options.js"; import {QueuedStateRegenerator, RegenCaller} from "./regen/index.js"; import {initializeForkChoice} from "./forkChoice/index.js"; import {computeAnchorCheckpoint} from "./initState.js"; -import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool} from "./bls/index.js"; +import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool, BlsAsyncBlstVerifier} from "./bls/index.js"; import { SeenAttesters, SeenAggregators, @@ -209,10 +209,12 @@ export class BeaconChain implements IBeaconChain { this.executionBuilder = executionBuilder; const signal = this.abortController.signal; const emitter = new ChainEventEmitter(); - // by default, verify signatures on both main threads and worker threads - const bls = opts.blsVerifyAllMainThread - ? new BlsSingleThreadVerifier({metrics}) - : new BlsMultiThreadWorkerPool(opts, {logger, metrics}); + // hard-code async blst verifier + const bls = + // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions + new BlsAsyncBlstVerifier({metrics}) || opts.blsVerifyAllMainThread + ? new BlsSingleThreadVerifier({metrics}) + : new BlsMultiThreadWorkerPool(opts, {logger, metrics}); if (!clock) clock = new Clock({config, genesisTime: this.genesisTime, signal}); diff --git a/yarn.lock b/yarn.lock index e417bbd07e00..99d683193d75 100644 --- a/yarn.lock +++ b/yarn.lock @@ -308,8 +308,8 @@ "@chainsafe/blst@https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz": version "0.0.0" - uid "4d6fce598429e888ad4fd49d7f4b7234537be5b6" - resolved "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz#4d6fce598429e888ad4fd49d7f4b7234537be5b6" + uid ffa9a00ebba2cdf7c3aa11f874cf5175aa434cb0 + resolved "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz#ffa9a00ebba2cdf7c3aa11f874cf5175aa434cb0" "@chainsafe/discv5@^9.0.0": version "9.0.0" From d827bd868335bbd9e20901ac06a6dac7498216ba Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 18 Jun 2024 15:25:24 -0400 Subject: [PATCH 04/18] chore: remove hardcoded blst async verifier use --- packages/beacon-node/src/chain/chain.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 846cea8e1b3f..2f58962f3cc5 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -57,7 +57,7 @@ import {IChainOptions} from "./options.js"; import {QueuedStateRegenerator, RegenCaller} from "./regen/index.js"; import {initializeForkChoice} from "./forkChoice/index.js"; import {computeAnchorCheckpoint} from "./initState.js"; -import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool, BlsAsyncBlstVerifier} from "./bls/index.js"; +import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool} from "./bls/index.js"; import { SeenAttesters, SeenAggregators, @@ -209,12 +209,10 @@ export class BeaconChain implements IBeaconChain { this.executionBuilder = executionBuilder; const signal = this.abortController.signal; const emitter = new ChainEventEmitter(); - // hard-code async blst verifier - const bls = - // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions - new BlsAsyncBlstVerifier({metrics}) || opts.blsVerifyAllMainThread - ? new BlsSingleThreadVerifier({metrics}) - : new BlsMultiThreadWorkerPool(opts, {logger, metrics}); + // by default, verify signatures on both main threads and worker threads + const bls = opts.blsVerifyAllMainThread + ? new BlsSingleThreadVerifier({metrics}) + : new BlsMultiThreadWorkerPool(opts, {logger, metrics}); if (!clock) clock = new Clock({config, genesisTime: this.genesisTime, signal}); From de255f8c7dbb45f440f697fa21fa5a04a72f0fe5 Mon Sep 17 00:00:00 2001 From: Cayman Date: Wed, 19 Jun 2024 15:38:03 -0400 Subject: [PATCH 05/18] feat: add async blst verifier with queueing --- .../beacon-node/src/chain/bls/asyncBlst.ts | 91 --- .../src/chain/bls/asyncBlst/index.ts | 538 ++++++++++++++++++ .../src/chain/bls/asyncBlst/jobItem.ts | 127 +++++ .../src/chain/bls/asyncBlst/types.ts | 33 ++ .../chain/bls/asyncBlst/verifySignatures.ts | 40 ++ packages/beacon-node/src/chain/bls/index.ts | 2 +- packages/beacon-node/src/chain/chain.ts | 10 +- packages/beacon-node/src/chain/options.ts | 2 + .../src/options/beaconNodeOptions/chain.ts | 9 + yarn.lock | 4 +- 10 files changed, 758 insertions(+), 98 deletions(-) delete mode 100644 packages/beacon-node/src/chain/bls/asyncBlst.ts create mode 100644 packages/beacon-node/src/chain/bls/asyncBlst/index.ts create mode 100644 packages/beacon-node/src/chain/bls/asyncBlst/jobItem.ts create mode 100644 packages/beacon-node/src/chain/bls/asyncBlst/types.ts create mode 100644 packages/beacon-node/src/chain/bls/asyncBlst/verifySignatures.ts diff --git a/packages/beacon-node/src/chain/bls/asyncBlst.ts b/packages/beacon-node/src/chain/bls/asyncBlst.ts deleted file mode 100644 index 892fd5a80162..000000000000 --- a/packages/beacon-node/src/chain/bls/asyncBlst.ts +++ /dev/null @@ -1,91 +0,0 @@ -import {PublicKey, Signature, aggregatePublicKeysAsync, aggregateSignaturesAsync, verifyAsync} from "@chainsafe/blst"; -import {ISignatureSet} from "@lodestar/state-transition"; -import {Metrics} from "../../metrics/index.js"; -import {IBlsVerifier} from "./interface.js"; -import {verifySignatureSetsMaybeBatchAsync} from "./maybeBatch.js"; -import {getAggregatedPubkeyAsync, getAggregatedPubkeysCount} from "./utils.js"; - -export class BlsAsyncBlstVerifier implements IBlsVerifier { - private readonly metrics: Metrics | null; - - constructor({metrics = null}: {metrics: Metrics | null}) { - this.metrics = metrics; - } - - async verifySignatureSets(sets: ISignatureSet[]): Promise { - this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets)); - - const setsAggregated = await Promise.all( - sets.map(async (set) => ({ - publicKey: await getAggregatedPubkeyAsync(set), - message: set.signingRoot, - signature: set.signature, - })) - ); - - // Count time after aggregating - const timer = this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.startTimer(); - const isValid = await verifySignatureSetsMaybeBatchAsync(setsAggregated); - - // Don't use a try/catch, only count run without exceptions - if (timer) { - timer(); - } - - return isValid; - } - - async verifySignatureSetsSameMessage( - sets: {publicKey: PublicKey; signature: Uint8Array}[], - message: Uint8Array - ): Promise { - const timer = this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.startTimer(); - const pubkey = await aggregatePublicKeysAsync(sets.map((set) => set.publicKey)); - let isAllValid = true; - // validate signature = true - const signatures = sets.map((set) => { - try { - return Signature.fromBytes(set.signature, true); - } catch (_) { - // at least one set has malformed signature - isAllValid = false; - return null; - } - }); - - if (isAllValid) { - const signature = await aggregateSignaturesAsync(signatures as Signature[]); - isAllValid = await verifyAsync(message, pubkey, signature); - } - - let result: boolean[]; - if (isAllValid) { - result = sets.map(() => true); - } else { - result = await Promise.all( - sets.map(async (set, i) => { - const sig = signatures[i]; - if (sig === null) { - return false; - } - return verifyAsync(message, set.publicKey, sig); - }) - ); - } - - if (timer) { - timer(); - } - - return result; - } - - async close(): Promise { - // nothing to do - } - - canAcceptWork(): boolean { - // Since sigs are verified blocking the main thread, there's no mechanism to throttle - return true; - } -} diff --git a/packages/beacon-node/src/chain/bls/asyncBlst/index.ts b/packages/beacon-node/src/chain/bls/asyncBlst/index.ts new file mode 100644 index 000000000000..6a3cd7308ce5 --- /dev/null +++ b/packages/beacon-node/src/chain/bls/asyncBlst/index.ts @@ -0,0 +1,538 @@ +/* eslint-disable @typescript-eslint/strict-boolean-expressions */ +import {PublicKey, Signature} from "@chainsafe/blst"; +import {Logger} from "@lodestar/utils"; +import {ISignatureSet} from "@lodestar/state-transition"; +import {QueueError, QueueErrorCode} from "../../../util/queue/index.js"; +import {Metrics} from "../../../metrics/index.js"; +import {callInNextEventLoop} from "../../../util/eventLoop.js"; +import {LinkedList} from "../../../util/array.js"; +import {IBlsVerifier, VerifySignatureOpts} from "../interface.js"; +import {getAggregatedPubkey, getAggregatedPubkeysCount} from "../utils.js"; +import {chunkifyMaximizeChunkSize} from "../multithread/utils.js"; +import {defaultPoolSize} from "../multithread/poolSize.js"; +import {BlsWorkReq, BlsWorkResult, SignatureSet, WorkResult, WorkResultCode, WorkResultError} from "./types.js"; +import { + JobQueueItem, + JobQueueItemSameMessage, + JobQueueItemType, + jobItemSameMessageToMultiSet, + jobItemSigSets, + jobItemWorkReq, +} from "./jobItem.js"; +import {verifySignatureSetsMaybeBatch} from "./verifySignatures.js"; + +export type BlsMultiThreadWorkerPoolModules = { + logger: Logger; + metrics: Metrics | null; +}; + +export type {JobQueueItemType}; + +/** + * Split big signature sets into smaller sets so they can be sent to multiple workers. + * + * The biggest sets happen during sync, on mainnet batches of 64 blocks have around ~8000 signatures. + * The latency cost of sending the job to and from the worker is approx a single sig verification. + * If you split a big signature into 2, the extra time cost is `(2+2N)/(1+2N)`. + * For 128, the extra time cost is about 0.3%. No specific reasoning for `128`, it's just good enough. + */ +const MAX_SIGNATURE_SETS_PER_JOB = 128; + +/** + * If there are more than `MAX_BUFFERED_SIGS` buffered sigs, verify them immediately without waiting `MAX_BUFFER_WAIT_MS`. + * + * The efficiency improvement of batching sets asymptotically reaches x2. However, for batching large sets + * has more risk in case a signature is invalid, requiring to revalidate all sets in the batch. 32 is sweet + * point for this tradeoff. + */ +const MAX_BUFFERED_SIGS = 32; +/** + * Gossip objects usually come in bursts. Buffering them for a short period of time allows to increase batching + * efficiency, at the cost of delaying validation. Unless running in production shows otherwise, it's not critical + * to hold attestations and aggregates for 100ms. Lodestar existing queues may hold those objects for much more anyway. + * + * There's no exact reasoning for the `100` milliseconds number. The metric `batchSigsSuccess` should indicate if this + * value needs revision + */ +const MAX_BUFFER_WAIT_MS = 100; + +/** + * Max concurrent jobs on `canAcceptWork` status + */ +const MAX_JOBS_CAN_ACCEPT_WORK = 512; + +/** + * Split batchable sets in chunks of minimum size 16. + * Batch verify 16 has an aprox cost of 16+1. For 32 it's 32+1. After ~16 the additional savings are not significant. + * However, if a sig is invalid the whole batch has to be re-verified. So it's important to keep this number low. + * In normal network conditions almost all signatures received by the node are correct. + * After observing metrics this number can be reviewed + */ +const BATCHABLE_MIN_PER_CHUNK = 16; + +/** + */ +export class BlsAsyncBlstVerifier implements IBlsVerifier { + private readonly logger: Logger; + private readonly metrics: Metrics | null; + private readonly poolSize = defaultPoolSize; + private workerId = 0; + + private readonly jobs = new LinkedList(); + private bufferedJobs: { + jobs: LinkedList; + prioritizedJobs: LinkedList; + sigCount: number; + firstPush: number; + timeout: NodeJS.Timeout; + } | null = null; + private closed = false; + private workersBusy = 0; + + constructor(modules: BlsMultiThreadWorkerPoolModules) { + const {logger, metrics} = modules; + this.logger = logger; + this.metrics = metrics; + + if (metrics) { + metrics.blsThreadPool.queueLength.addCollect(() => { + metrics.blsThreadPool.queueLength.set(this.jobs.length); + metrics.blsThreadPool.workersBusy.set(this.workersBusy); + }); + } + } + + canAcceptWork(): boolean { + return ( + this.workersBusy < this.poolSize && + // TODO: Should also bound the jobs queue? + this.jobs.length < MAX_JOBS_CAN_ACCEPT_WORK + ); + } + + async verifySignatureSets(sets: ISignatureSet[], opts: VerifySignatureOpts = {}): Promise { + // Pubkeys are aggregated in the main thread regardless if verified in workers or in main thread + this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets)); + this.metrics?.blsThreadPool.totalSigSets.inc(sets.length); + if (opts.priority) { + this.metrics?.blsThreadPool.prioritizedSigSets.inc(sets.length); + } + if (opts.batchable) { + this.metrics?.blsThreadPool.batchableSigSets.inc(sets.length); + } + + if (opts.verifyOnMainThread) { + const timer = this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.startTimer(); + try { + return await verifySignatureSetsMaybeBatch( + sets.map((set) => ({ + publicKey: getAggregatedPubkey(set), + message: set.signingRoot.valueOf(), + signature: Signature.fromBytes(set.signature, true), + })) + ); + } finally { + if (timer) timer(); + } + } + + // Split large array of sets into smaller. + // Very helpful when syncing finalized, sync may submit +1000 sets so chunkify allows to distribute to many workers + const results = await Promise.all( + chunkifyMaximizeChunkSize(sets, MAX_SIGNATURE_SETS_PER_JOB).map( + (setsChunk) => + new Promise((resolve, reject) => { + return this.queueBlsWork({ + type: JobQueueItemType.default, + resolve, + reject, + addedTimeMs: Date.now(), + opts, + sets: setsChunk, + }); + }) + ) + ); + + // .every on an empty array returns true + if (results.length === 0) { + throw Error("Empty results array"); + } + + return results.every((isValid) => isValid === true); + } + + /** + * Verify signature sets of the same message, only supports worker verification. + */ + async verifySignatureSetsSameMessage( + sets: {publicKey: PublicKey; signature: Uint8Array}[], + message: Uint8Array, + opts: Omit = {} + ): Promise { + // chunkify so that it reduce the risk of retrying when there is at least one invalid signature + const results = await Promise.all( + chunkifyMaximizeChunkSize(sets, MAX_SIGNATURE_SETS_PER_JOB).map( + (setsChunk) => + new Promise((resolve, reject) => { + this.queueBlsWork({ + type: JobQueueItemType.sameMessage, + resolve, + reject, + addedTimeMs: Date.now(), + opts, + sets: setsChunk, + message, + }); + }) + ) + ); + + return results.flat(); + } + + async close(): Promise { + if (this.bufferedJobs) { + clearTimeout(this.bufferedJobs.timeout); + } + + // Abort all jobs + for (const job of this.jobs) { + job.reject(new QueueError({code: QueueErrorCode.QUEUE_ABORTED})); + } + this.jobs.clear(); + } + + /** + * Register BLS work to be done eventually in a worker + */ + private queueBlsWork(job: JobQueueItem): void { + if (this.closed) { + throw new QueueError({code: QueueErrorCode.QUEUE_ABORTED}); + } + + // TODO: Consider if limiting queue size is necessary here. + // It would be bad to reject signatures because the node is slow. + // However, if the worker communication broke jobs won't ever finish + + // Append batchable sets to `bufferedJobs`, starting a timeout to push them into `jobs`. + // Do not call `runJob()`, it is called from `runBufferedJobs()` + if (job.opts.batchable) { + if (!this.bufferedJobs) { + this.bufferedJobs = { + jobs: new LinkedList(), + prioritizedJobs: new LinkedList(), + sigCount: 0, + firstPush: Date.now(), + timeout: setTimeout(this.runBufferedJobs, MAX_BUFFER_WAIT_MS), + }; + } + const jobs = job.opts.priority ? this.bufferedJobs.prioritizedJobs : this.bufferedJobs.jobs; + jobs.push(job); + this.bufferedJobs.sigCount += jobItemSigSets(job); + if (this.bufferedJobs.sigCount > MAX_BUFFERED_SIGS) { + clearTimeout(this.bufferedJobs.timeout); + this.runBufferedJobs(); + } + } + + // Push job and schedule to call `runJob` in the next macro event loop cycle. + // This is useful to allow batching job submitted from a synchronous for loop, + // and to prevent large stacks since runJob may be called recursively. + else { + if (job.opts.priority) { + this.jobs.unshift(job); + } else { + this.jobs.push(job); + } + callInNextEventLoop(this.runJob); + } + } + + /** + * Potentially submit jobs to an idle worker, only if there's a worker and jobs + */ + private runJob = async (): Promise => { + if (this.closed) { + return; + } + + // Prepare work package + const jobsInput = this.prepareWork(); + if (jobsInput.length === 0) { + return; + } + + this.workersBusy++; + + try { + let startedJobsDefault = 0; + let startedJobsSameMessage = 0; + let startedSetsDefault = 0; + let startedSetsSameMessage = 0; + const workReqs: BlsWorkReq[] = []; + const jobsStarted: JobQueueItem[] = []; + + for (const job of jobsInput) { + this.metrics?.blsThreadPool.jobWaitTime.observe((Date.now() - job.addedTimeMs) / 1000); + + let workReq: BlsWorkReq; + try { + // Note: This can throw, must be handled per-job. + // Pubkey and signature aggregation is defered here + workReq = await jobItemWorkReq(job, this.metrics); + } catch (e) { + this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({type: job.type}); + + switch (job.type) { + case JobQueueItemType.default: + job.reject(e as Error); + break; + + case JobQueueItemType.sameMessage: + // there could be an invalid pubkey/signature, retry each individually + this.retryJobItemSameMessage(job); + break; + } + + continue; + } + // Re-push all jobs with matching workReq for easier accounting of results + workReqs.push(workReq); + jobsStarted.push(job); + + if (job.type === JobQueueItemType.sameMessage) { + startedJobsSameMessage += 1; + startedSetsSameMessage += job.sets.length; + } else { + startedJobsDefault += 1; + startedSetsDefault += job.sets.length; + } + } + + const startedSigSets = startedSetsDefault + startedSetsSameMessage; + this.metrics?.blsThreadPool.totalJobsGroupsStarted.inc(1); + this.metrics?.blsThreadPool.totalJobsStarted.inc({type: JobQueueItemType.default}, startedJobsDefault); + this.metrics?.blsThreadPool.totalJobsStarted.inc({type: JobQueueItemType.sameMessage}, startedJobsSameMessage); + this.metrics?.blsThreadPool.totalSigSetsStarted.inc({type: JobQueueItemType.default}, startedSetsDefault); + this.metrics?.blsThreadPool.totalSigSetsStarted.inc({type: JobQueueItemType.sameMessage}, startedSetsSameMessage); + + // Send work package to the worker + // If the job, metrics or any code below throws: the job will reject never going stale. + // Only downside is the job promise may be resolved twice, but that's not an issue + + const [jobStartSec, jobStartNs] = process.hrtime(); + const workResult = await verifyManySignatureSets(workReqs); + const [jobEndSec, jobEndNs] = process.hrtime(); + const {batchRetries, batchSigsSuccess, workerStartTime, workerEndTime, results} = workResult; + + const [workerStartSec, workerStartNs] = workerStartTime; + const [workerEndSec, workerEndNs] = workerEndTime; + + let successCount = 0; + let errorCount = 0; + + // Un-wrap work package + for (let i = 0; i < jobsStarted.length; i++) { + const job = jobsStarted[i]; + const jobResult = results[i]; + const sigSetCount = jobItemSigSets(job); + + // TODO: enable exhaustive switch case checks lint rule + switch (job.type) { + case JobQueueItemType.default: + if (!jobResult || jobResult.code !== WorkResultCode.success) { + job.reject(getJobResultError(jobResult, i)); + errorCount += sigSetCount; + } else { + job.resolve(jobResult.result); + successCount += sigSetCount; + } + break; + + // handle result of the verification of aggregated signature against aggregated pubkeys + case JobQueueItemType.sameMessage: + if (!jobResult || jobResult.code !== WorkResultCode.success) { + job.reject(getJobResultError(jobResult, i)); + errorCount += 1; + } else { + if (jobResult.result) { + // All are valid, most of the time it goes here + job.resolve(job.sets.map(() => true)); + } else { + // Retry each individually + this.retryJobItemSameMessage(job); + } + successCount += 1; + } + break; + } + } + + const workerId = this.workerId++ % this.poolSize; + const workerJobTimeSec = workerEndSec - workerStartSec + (workerEndNs - workerStartNs) / 1e9; + const latencyToWorkerSec = workerStartSec - jobStartSec + (workerStartNs - jobStartNs) / 1e9; + const latencyFromWorkerSec = jobEndSec - workerEndSec + Number(jobEndNs - workerEndNs) / 1e9; + + this.metrics?.blsThreadPool.timePerSigSet.observe(workerJobTimeSec / startedSigSets); + this.metrics?.blsThreadPool.jobsWorkerTime.inc({workerId}, workerJobTimeSec); + this.metrics?.blsThreadPool.latencyToWorker.observe(latencyToWorkerSec); + this.metrics?.blsThreadPool.latencyFromWorker.observe(latencyFromWorkerSec); + this.metrics?.blsThreadPool.successJobsSignatureSetsCount.inc(successCount); + this.metrics?.blsThreadPool.errorJobsSignatureSetsCount.inc(errorCount); + this.metrics?.blsThreadPool.batchRetries.inc(batchRetries); + this.metrics?.blsThreadPool.batchSigsSuccess.inc(batchSigsSuccess); + } catch (e) { + // Worker communications should never reject + if (!this.closed) { + this.logger.error("BlsMultiThreadWorkerPool error", {}, e as Error); + } + // Reject all + for (const job of jobsInput) { + job.reject(e as Error); + } + } + + this.workersBusy--; + + // Potentially run a new job + callInNextEventLoop(this.runJob); + }; + + /** + * Grab pending work up to a max number of signatures + */ + private prepareWork(): JobQueueItem[] { + const jobs: JobQueueItem[] = []; + let totalSigs = 0; + + while (totalSigs < MAX_SIGNATURE_SETS_PER_JOB) { + const job = this.jobs.shift(); + if (!job) { + break; + } + + jobs.push(job); + totalSigs += jobItemSigSets(job); + } + + return jobs; + } + + /** + * Add all buffered jobs to the job queue and potentially run them immediately + */ + private runBufferedJobs = (): void => { + if (this.bufferedJobs) { + for (const job of this.bufferedJobs.jobs) { + this.jobs.push(job); + } + for (const job of this.bufferedJobs.prioritizedJobs) { + this.jobs.unshift(job); + } + this.bufferedJobs = null; + callInNextEventLoop(this.runJob); + } + }; + + private retryJobItemSameMessage(job: JobQueueItemSameMessage): void { + // Create new jobs for each pubkey set, and Promise.all all the results + for (const j of jobItemSameMessageToMultiSet(job)) { + if (j.opts.priority) { + this.jobs.unshift(j); + } else { + this.jobs.push(j); + } + } + this.metrics?.blsThreadPool.sameMessageRetryJobs.inc(1); + this.metrics?.blsThreadPool.sameMessageRetrySets.inc(job.sets.length); + } + + /** For testing */ + private async waitTillInitialized(): Promise {} +} + +function getJobResultError(jobResult: WorkResultError | null, i: number): Error { + const workerError = jobResult ? Error(jobResult.error.message) : Error(`No jobResult for index ${i}`); + if (jobResult?.error?.stack) workerError.stack = jobResult.error.stack; + return workerError; +} + +async function verifyManySignatureSets(workReqArr: BlsWorkReq[]): Promise { + const [startSec, startNs] = process.hrtime(); + const results: WorkResult[] = []; + let batchRetries = 0; + let batchSigsSuccess = 0; + + // If there are multiple batchable sets attempt batch verification with them + const batchableSets: {idx: number; sets: SignatureSet[]}[] = []; + const nonBatchableSets: {idx: number; sets: SignatureSet[]}[] = []; + + // Split sets between batchable and non-batchable preserving their original index in the req array + for (let i = 0; i < workReqArr.length; i++) { + const workReq = workReqArr[i]; + const sets = workReq.sets; + + if (workReq.opts.batchable) { + batchableSets.push({idx: i, sets}); + } else { + nonBatchableSets.push({idx: i, sets}); + } + } + + if (batchableSets.length > 0) { + // Split batchable into chunks of max size ~ 32 to minimize cost if a sig is wrong + const batchableChunks = chunkifyMaximizeChunkSize(batchableSets, BATCHABLE_MIN_PER_CHUNK); + + for (const batchableChunk of batchableChunks) { + const allSets: SignatureSet[] = []; + for (const {sets} of batchableChunk) { + for (const set of sets) { + allSets.push(set); + } + } + + try { + // Attempt to verify multiple sets at once + const isValid = await verifySignatureSetsMaybeBatch(allSets); + + if (isValid) { + // The entire batch is valid, return success to all + for (const {idx, sets} of batchableChunk) { + batchSigsSuccess += sets.length; + results[idx] = {code: WorkResultCode.success, result: isValid}; + } + } else { + batchRetries++; + // Re-verify all sigs + nonBatchableSets.push(...batchableChunk); + } + } catch (e) { + // TODO: Ignore this error expecting that the same error will happen when re-verifying the set individually + // It's not ideal but '@chainsafe/blst' may throw errors on some conditions + batchRetries++; + // Re-verify all sigs + nonBatchableSets.push(...batchableChunk); + } + } + } + + for (const {idx, sets} of nonBatchableSets) { + try { + const isValid = await verifySignatureSetsMaybeBatch(sets); + results[idx] = {code: WorkResultCode.success, result: isValid}; + } catch (e) { + results[idx] = {code: WorkResultCode.error, error: e as Error}; + } + } + + const [workerEndSec, workerEndNs] = process.hrtime(); + + return { + batchRetries, + batchSigsSuccess, + workerStartTime: [startSec, startNs], + workerEndTime: [workerEndSec, workerEndNs], + results, + }; +} diff --git a/packages/beacon-node/src/chain/bls/asyncBlst/jobItem.ts b/packages/beacon-node/src/chain/bls/asyncBlst/jobItem.ts new file mode 100644 index 000000000000..e4bbabf948d9 --- /dev/null +++ b/packages/beacon-node/src/chain/bls/asyncBlst/jobItem.ts @@ -0,0 +1,127 @@ +import {PublicKey, Signature, aggregatePublicKeys, aggregateSerializedSignaturesAsync} from "@chainsafe/blst"; +import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; +import {VerifySignatureOpts} from "../interface.js"; +import {getAggregatedPubkey} from "../utils.js"; +import {LinkedList} from "../../../util/array.js"; +import {Metrics} from "../../../metrics/metrics.js"; +import {BlsWorkReq} from "./types.js"; + +export type JobQueueItem = JobQueueItemDefault | JobQueueItemSameMessage; + +export type JobQueueItemDefault = { + type: JobQueueItemType.default; + resolve: (result: boolean) => void; + reject: (error?: Error) => void; + addedTimeMs: number; + opts: VerifySignatureOpts; + sets: ISignatureSet[]; +}; + +export type JobQueueItemSameMessage = { + type: JobQueueItemType.sameMessage; + resolve: (result: boolean[]) => void; + reject: (error?: Error) => void; + addedTimeMs: number; + opts: VerifySignatureOpts; + sets: {publicKey: PublicKey; signature: Uint8Array}[]; + message: Uint8Array; +}; + +export enum JobQueueItemType { + default = "default", + sameMessage = "same_message", +} + +/** + * Return count of signature sets from a JobQueueItem + */ +export function jobItemSigSets(job: JobQueueItem): number { + switch (job.type) { + case JobQueueItemType.default: + return job.sets.length; + case JobQueueItemType.sameMessage: + return 1; + } +} + +/** + * Prepare BlsWorkReq from JobQueueItem + * WARNING: May throw with untrusted user input + */ +export async function jobItemWorkReq(job: JobQueueItem, metrics: Metrics | null): Promise { + switch (job.type) { + case JobQueueItemType.default: + return { + opts: job.opts, + sets: job.sets.map((set) => ({ + // this can throw, handled in the consumer code + publicKey: getAggregatedPubkey(set, metrics), + signature: Signature.fromBytes(set.signature, true), + message: set.signingRoot, + })), + }; + case JobQueueItemType.sameMessage: { + // validate signature = true, this is slow code on main thread so should only run with network thread mode (useWorker=true) + // For a node subscribing to all subnets, with 1 signature per validator per epoch it takes around 80s + // to deserialize 750_000 signatures per epoch + // cpu profile on main thread has 250s idle so this only works until we reach 3M validators + // However, for normal node with only 2 to 7 subnet subscriptions per epoch this works until 27M validators + // and not a problem in the near future + // this is monitored on v1.11.0 https://github.com/ChainSafe/lodestar/pull/5912#issuecomment-1700320307 + const timer = metrics?.blsThreadPool.signatureDeserializationMainThreadDuration.startTimer(); + const signature = await aggregateSerializedSignaturesAsync( + job.sets.map((set) => set.signature), + true + ); + timer?.(); + + return { + opts: job.opts, + sets: [ + { + publicKey: aggregatePublicKeys(job.sets.map((set) => set.publicKey)), + signature, + message: job.message, + }, + ], + }; + } + } +} + +/** + * Convert a JobQueueItemSameMessage into multiple JobQueueItemDefault linked to the original promise + */ +export function jobItemSameMessageToMultiSet(job: JobQueueItemSameMessage): LinkedList { + // Retry each individually + // Create new jobs for each pubkey set, and Promise.all all the results + const promises: Promise[] = []; + const jobs = new LinkedList(); + + for (const set of job.sets) { + promises.push( + new Promise((resolve, reject) => { + jobs.push({ + type: JobQueueItemType.default, + resolve, + reject, + addedTimeMs: job.addedTimeMs, + opts: {batchable: false, priority: job.opts.priority}, + sets: [ + { + type: SignatureSetType.single, + pubkey: set.publicKey, + signature: set.signature, + signingRoot: job.message, + }, + ], + }); + }) + ); + } + + // Connect jobs to main job + Promise.all(promises).then(job.resolve, job.reject); + + return jobs; +} diff --git a/packages/beacon-node/src/chain/bls/asyncBlst/types.ts b/packages/beacon-node/src/chain/bls/asyncBlst/types.ts new file mode 100644 index 000000000000..55470c76d658 --- /dev/null +++ b/packages/beacon-node/src/chain/bls/asyncBlst/types.ts @@ -0,0 +1,33 @@ +import {PublicKey, Signature} from "@chainsafe/blst"; +import {VerifySignatureOpts} from "../interface.js"; + +export type SignatureSet = { + publicKey: PublicKey; + message: Uint8Array; + signature: Signature; +}; + +export type BlsWorkReq = { + opts: VerifySignatureOpts; + sets: SignatureSet[]; +}; + +export enum WorkResultCode { + success = "success", + error = "error", +} + +export type WorkResultError = {code: WorkResultCode.error; error: Error}; +export type WorkResult = {code: WorkResultCode.success; result: R} | WorkResultError; + +export type BlsWorkResult = { + /** Total num of batches that had to be retried */ + batchRetries: number; + /** Total num of sigs that have been successfully verified with batching */ + batchSigsSuccess: number; + /** Time worker function starts - UNIX timestamp in seconds and nanoseconds */ + workerStartTime: [number, number]; + /** Time worker function ends - UNIX timestamp in seconds and nanoseconds */ + workerEndTime: [number, number]; + results: WorkResult[]; +}; diff --git a/packages/beacon-node/src/chain/bls/asyncBlst/verifySignatures.ts b/packages/beacon-node/src/chain/bls/asyncBlst/verifySignatures.ts new file mode 100644 index 000000000000..14583ff99aa3 --- /dev/null +++ b/packages/beacon-node/src/chain/bls/asyncBlst/verifySignatures.ts @@ -0,0 +1,40 @@ +import {verifyAsync, verifyMultipleAggregateSignaturesAsync} from "@chainsafe/blst"; +import {SignatureSet} from "./types.js"; + +const MIN_SET_COUNT_TO_BATCH = 2; + +/** + * Verify signatures sets with batch verification or regular core verify depending on the set count. + * Abstracted in a separate file to be consumed by the threaded pool and the main thread implementation. + */ +export async function verifySignatureSetsMaybeBatch(sets: SignatureSet[]): Promise { + try { + if (sets.length >= MIN_SET_COUNT_TO_BATCH) { + return await verifyMultipleAggregateSignaturesAsync( + sets.map((s) => ({ + pk: s.publicKey, + msg: s.message, + sig: s.signature, + })) + ); + } + + // .every on an empty array returns true + if (sets.length === 0) { + throw Error("Empty signature set"); + } + + // If too few signature sets verify them without batching + for (const set of sets) { + if (!(await verifyAsync(set.message, set.publicKey, set.signature))) { + return false; + } + } + return true; + } catch (_) { + // A signature could be malformed, in that case fromBytes throws error + // blst-ts `verifyMultipleSignatures` is also a fallible operation if mul_n_aggregate fails + // see https://github.com/ChainSafe/blst-ts/blob/b1ba6333f664b08e5c50b2b0d18c4f079203962b/src/lib.ts#L291 + return false; + } +} diff --git a/packages/beacon-node/src/chain/bls/index.ts b/packages/beacon-node/src/chain/bls/index.ts index 62ed6c257c25..347b65e7f6a2 100644 --- a/packages/beacon-node/src/chain/bls/index.ts +++ b/packages/beacon-node/src/chain/bls/index.ts @@ -2,4 +2,4 @@ export type {IBlsVerifier} from "./interface.js"; export type {BlsMultiThreadWorkerPoolModules, JobQueueItemType} from "./multithread/index.js"; export {BlsMultiThreadWorkerPool} from "./multithread/index.js"; export {BlsSingleThreadVerifier} from "./singleThread.js"; -export {BlsAsyncBlstVerifier} from "./asyncBlst.js"; +export {BlsAsyncBlstVerifier} from "./asyncBlst/index.js"; diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 2f58962f3cc5..2c4cc5676837 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -57,7 +57,7 @@ import {IChainOptions} from "./options.js"; import {QueuedStateRegenerator, RegenCaller} from "./regen/index.js"; import {initializeForkChoice} from "./forkChoice/index.js"; import {computeAnchorCheckpoint} from "./initState.js"; -import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool} from "./bls/index.js"; +import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool, BlsAsyncBlstVerifier} from "./bls/index.js"; import { SeenAttesters, SeenAggregators, @@ -210,9 +210,11 @@ export class BeaconChain implements IBeaconChain { const signal = this.abortController.signal; const emitter = new ChainEventEmitter(); // by default, verify signatures on both main threads and worker threads - const bls = opts.blsVerifyAllMainThread - ? new BlsSingleThreadVerifier({metrics}) - : new BlsMultiThreadWorkerPool(opts, {logger, metrics}); + const bls = opts.blsVerifyAllAsyncBlst + ? new BlsAsyncBlstVerifier({logger, metrics}) + : opts.blsVerifyAllMainThread + ? new BlsSingleThreadVerifier({metrics}) + : new BlsMultiThreadWorkerPool(opts, {logger, metrics}); if (!clock) clock = new Clock({config, genesisTime: this.genesisTime, signal}); diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index 7c7cfcdde75b..b50e7ab1cbe0 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -19,6 +19,7 @@ export type IChainOptions = BlockProcessOpts & LightClientServerOpts & { blsVerifyAllMainThread?: boolean; blsVerifyAllMultiThread?: boolean; + blsVerifyAllAsyncBlst?: boolean; persistProducedBlocks?: boolean; persistInvalidSszObjects?: boolean; persistInvalidSszObjectsDir?: string; @@ -94,6 +95,7 @@ export type SeenCacheOpts = { export const defaultChainOptions: IChainOptions = { blsVerifyAllMainThread: false, blsVerifyAllMultiThread: false, + blsVerifyAllAsyncBlst: false, disableBlsBatchVerify: false, proposerBoost: true, proposerBoostReorg: false, diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts index aae97b6db68f..3b7ad1e42186 100644 --- a/packages/cli/src/options/beaconNodeOptions/chain.ts +++ b/packages/cli/src/options/beaconNodeOptions/chain.ts @@ -6,6 +6,7 @@ export type ChainArgs = { suggestedFeeRecipient: string; "chain.blsVerifyAllMultiThread"?: boolean; "chain.blsVerifyAllMainThread"?: boolean; + "chain.blsVerifyAllAsyncBlst"?: boolean; "chain.disableBlsBatchVerify"?: boolean; "chain.persistProducedBlocks"?: boolean; "chain.persistInvalidSszObjects"?: boolean; @@ -101,6 +102,14 @@ export const options: CliCommandOptions = { group: "chain", }, + "chain.blsVerifyAllAsyncBlst": { + hidden: true, + type: "boolean", + description: "Always use async blst for BLS verification", + defaultDescription: String(defaultOptions.chain.blsVerifyAllAsyncBlst), + group: "chain", + }, + "chain.disableBlsBatchVerify": { hidden: true, type: "boolean", diff --git a/yarn.lock b/yarn.lock index 99d683193d75..dc075f8dfc26 100644 --- a/yarn.lock +++ b/yarn.lock @@ -308,8 +308,8 @@ "@chainsafe/blst@https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz": version "0.0.0" - uid ffa9a00ebba2cdf7c3aa11f874cf5175aa434cb0 - resolved "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz#ffa9a00ebba2cdf7c3aa11f874cf5175aa434cb0" + uid "58f57171e362d3a52bf33007a657062ca537bd5e" + resolved "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz#58f57171e362d3a52bf33007a657062ca537bd5e" "@chainsafe/discv5@^9.0.0": version "9.0.0" From a19de434cd076c2542ddd82c5cdf6c83128f73f2 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 2 Jul 2024 22:50:34 -0400 Subject: [PATCH 06/18] chore: add aggregateWithRandomness --- packages/beacon-node/src/chain/bls/multithread/jobItem.ts | 8 ++++---- yarn.lock | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts index 8d051ddc411e..b6390b5a45ef 100644 --- a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts +++ b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts @@ -1,4 +1,4 @@ -import {PublicKey, Signature, aggregatePublicKeys, aggregateSignatures} from "@chainsafe/blst"; +import {PublicKey, aggregateWithRandomness} from "@chainsafe/blst"; import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; import {VerifySignatureOpts} from "../interface.js"; import {getAggregatedPubkey} from "../utils.js"; @@ -69,15 +69,15 @@ export function jobItemWorkReq(job: JobQueueItem, metrics: Metrics | null): BlsW // and not a problem in the near future // this is monitored on v1.11.0 https://github.com/ChainSafe/lodestar/pull/5912#issuecomment-1700320307 const timer = metrics?.blsThreadPool.signatureDeserializationMainThreadDuration.startTimer(); - const signatures = job.sets.map((set) => Signature.fromBytes(set.signature, true)); + const {pk, sig} = aggregateWithRandomness(job.sets.map((set) => ({pk: set.publicKey, sig: set.signature}))); timer?.(); return { opts: job.opts, sets: [ { - publicKey: aggregatePublicKeys(job.sets.map((set) => set.publicKey)).toBytes(), - signature: aggregateSignatures(signatures).toBytes(), + publicKey: pk.toBytes(), + signature: sig.toBytes(), message: job.message, }, ], diff --git a/yarn.lock b/yarn.lock index dc075f8dfc26..4db2b56729d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -308,8 +308,8 @@ "@chainsafe/blst@https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz": version "0.0.0" - uid "58f57171e362d3a52bf33007a657062ca537bd5e" - resolved "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz#58f57171e362d3a52bf33007a657062ca537bd5e" + uid f9a163d697d253279e4aef6fb644e3418ebc7a03 + resolved "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz#f9a163d697d253279e4aef6fb644e3418ebc7a03" "@chainsafe/discv5@^9.0.0": version "9.0.0" From 3c59fce1e5b253d9f08bfab7af8ad00077627072 Mon Sep 17 00:00:00 2001 From: Cayman Date: Wed, 3 Jul 2024 00:10:53 -0400 Subject: [PATCH 07/18] chore: bump blst --- yarn.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index 4db2b56729d0..ccd43a4ad57e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -308,8 +308,8 @@ "@chainsafe/blst@https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz": version "0.0.0" - uid f9a163d697d253279e4aef6fb644e3418ebc7a03 - resolved "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz#f9a163d697d253279e4aef6fb644e3418ebc7a03" + uid efe8ceb54b30a8fb76f3ebd65ed33c94a19a1a07 + resolved "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz#efe8ceb54b30a8fb76f3ebd65ed33c94a19a1a07" "@chainsafe/discv5@^9.0.0": version "9.0.0" From c634d0d84faf98a5cf4c1a5eb9ef920061efaad0 Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 22 Jul 2024 11:19:47 -0400 Subject: [PATCH 08/18] chore: bump @chainsafe/blst to 2.0.1 --- packages/beacon-node/package.json | 2 +- packages/cli/package.json | 2 +- packages/flare/package.json | 2 +- packages/state-transition/package.json | 2 +- packages/test-utils/package.json | 2 +- packages/validator/package.json | 2 +- yarn.lock | 39 +++++++++++++++++++++++--- 7 files changed, 41 insertions(+), 10 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 92f08a4d0a7b..96638a44caf8 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -95,7 +95,7 @@ }, "dependencies": { "@chainsafe/as-sha256": "^0.4.1", - "@chainsafe/blst": "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz", + "@chainsafe/blst": "^2.0.1", "@chainsafe/discv5": "^9.0.0", "@chainsafe/enr": "^3.0.0", "@chainsafe/libp2p-gossipsub": "^13.0.0", diff --git a/packages/cli/package.json b/packages/cli/package.json index 1b2c2e10eb3c..cbeda803ba7e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -53,7 +53,7 @@ "dependencies": { "@chainsafe/bls-keygen": "^0.4.0", "@chainsafe/bls-keystore": "^3.1.0", - "@chainsafe/blst": "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz", + "@chainsafe/blst": "^2.0.1", "@chainsafe/discv5": "^9.0.0", "@chainsafe/enr": "^3.0.0", "@chainsafe/persistent-merkle-tree": "^0.7.1", diff --git a/packages/flare/package.json b/packages/flare/package.json index bf5cbc0d6e41..a962bf3ddc2d 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -58,7 +58,7 @@ "blockchain" ], "dependencies": { - "@chainsafe/blst": "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz", + "@chainsafe/blst": "^2.0.1", "@chainsafe/bls-keygen": "^0.4.0", "@lodestar/api": "^1.20.2", "@lodestar/config": "^1.20.2", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 61250d5d0422..c9c16a672c18 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -59,7 +59,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/as-sha256": "^0.4.1", - "@chainsafe/blst": "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz", + "@chainsafe/blst": "^2.0.1", "@chainsafe/persistent-merkle-tree": "^0.7.1", "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.15.1", diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 5e2f6f0130cb..ee8935849636 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -57,7 +57,7 @@ "blockchain" ], "dependencies": { - "@chainsafe/blst": "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz", + "@chainsafe/blst": "^2.0.1", "@chainsafe/bls-keystore": "^3.1.0", "@lodestar/params": "^1.20.2", "@lodestar/utils": "^1.20.2", diff --git a/packages/validator/package.json b/packages/validator/package.json index 9292b68ff92a..3451763a94bc 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -45,7 +45,7 @@ "blockchain" ], "dependencies": { - "@chainsafe/blst": "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz", + "@chainsafe/blst": "^2.0.1", "@chainsafe/ssz": "^0.15.1", "@lodestar/api": "^1.20.2", "@lodestar/config": "^1.20.2", diff --git a/yarn.lock b/yarn.lock index 07ba38fa1711..2a565294d128 100644 --- a/yarn.lock +++ b/yarn.lock @@ -306,10 +306,41 @@ "@chainsafe/bls-keygen" "^0.4.0" bls-eth-wasm "^0.4.8" -"@chainsafe/blst@https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz": - version "0.0.0" - uid efe8ceb54b30a8fb76f3ebd65ed33c94a19a1a07 - resolved "https://github.com/chainsafe/blst-ts/raw/cayman/napi-rs-tarball/next/chainsafe-blst-v0.0.0.tgz#efe8ceb54b30a8fb76f3ebd65ed33c94a19a1a07" +"@chainsafe/blst-darwin-arm64@2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@chainsafe/blst-darwin-arm64/-/blst-darwin-arm64-2.0.1.tgz#d96d6dd906a6c9c809d6f5b5539e031ccff63b25" + integrity sha512-ZmRLimvo+BoMcpalzuS3Pj0j6l2cSDU7qMBwjch49ljkrsr4/rls7COMW8MSyDyVUSfzg0agotAByYfs+Bg3ZQ== + +"@chainsafe/blst-darwin-x64@2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@chainsafe/blst-darwin-x64/-/blst-darwin-x64-2.0.1.tgz#56018c5955337a5e2b754e941bdcda0a2c6f1a84" + integrity sha512-5DPtmKhia5/k0szjsrgxF7GCOE5pnSRcsLvtlChzkLY7KhfxnGt5XeNtCK6NoAAfcxzN8mZrwvvzDfsDmImTQg== + +"@chainsafe/blst-linux-arm64-gnu@2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@chainsafe/blst-linux-arm64-gnu/-/blst-linux-arm64-gnu-2.0.1.tgz#23a66095eaf4c23a8c54ef7df27d9c80d45ebb32" + integrity sha512-REIW0uM9a97iasE+RX+M1yEmIywOF1ly9/xl7JTYYASXoDDQD2eL06k17N5kXE/382wU28fU/h6V2qmWcqiWAQ== + +"@chainsafe/blst-linux-x64-gnu@2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@chainsafe/blst-linux-x64-gnu/-/blst-linux-x64-gnu-2.0.1.tgz#1769640e9b9140ee3e4142520c00d09c9ebc66e9" + integrity sha512-zH7GkMI+wWVKGp5MA+8A2EGx9fe5/jktz/KB2SrGhBu956IXh7qCV9mMCOzA4mmpxSuxjzD4auXpQc/D4uApLw== + +"@chainsafe/blst-win32-x64-msvc@2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@chainsafe/blst-win32-x64-msvc/-/blst-win32-x64-msvc-2.0.1.tgz#dfebc2ea23875bc02ffac11d2d15bf6e644d59ed" + integrity sha512-RcOo1Sl1Ai5igp56l3I5kwWkesfbD14PBoUcoR61phvIHnhyZzOtVX6rhuCYwcl0IIgSDSWtNNMdSwX34GGmxA== + +"@chainsafe/blst@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@chainsafe/blst/-/blst-2.0.1.tgz#9e8ceb6766fcb231e4fa378eee6378275018a159" + integrity sha512-+sIlLzFb6htv1WH3XIF2WDN+qlstGpo8Zl5ibxzT6VsiBSswsH05AQMpd4uQfRO1uFKhEk9JDKi8bHvIDmN8Jg== + optionalDependencies: + "@chainsafe/blst-darwin-arm64" "2.0.1" + "@chainsafe/blst-darwin-x64" "2.0.1" + "@chainsafe/blst-linux-arm64-gnu" "2.0.1" + "@chainsafe/blst-linux-x64-gnu" "2.0.1" + "@chainsafe/blst-win32-x64-msvc" "2.0.1" "@chainsafe/discv5@^9.0.0": version "9.0.0" From e40f25dbfe7691cf8577ecd54e21c2a1489f2746 Mon Sep 17 00:00:00 2001 From: Cayman Date: Mon, 22 Jul 2024 11:26:58 -0400 Subject: [PATCH 09/18] chore: remove async blst verifier --- .../src/chain/bls/asyncBlst/index.ts | 538 ------------------ .../src/chain/bls/asyncBlst/jobItem.ts | 127 ----- .../src/chain/bls/asyncBlst/types.ts | 33 -- .../chain/bls/asyncBlst/verifySignatures.ts | 40 -- packages/beacon-node/src/chain/bls/index.ts | 1 - .../beacon-node/src/chain/bls/maybeBatch.ts | 40 +- packages/beacon-node/src/chain/bls/utils.ts | 22 +- packages/beacon-node/src/chain/chain.ts | 10 +- packages/beacon-node/src/chain/options.ts | 2 - .../src/options/beaconNodeOptions/chain.ts | 9 - 10 files changed, 6 insertions(+), 816 deletions(-) delete mode 100644 packages/beacon-node/src/chain/bls/asyncBlst/index.ts delete mode 100644 packages/beacon-node/src/chain/bls/asyncBlst/jobItem.ts delete mode 100644 packages/beacon-node/src/chain/bls/asyncBlst/types.ts delete mode 100644 packages/beacon-node/src/chain/bls/asyncBlst/verifySignatures.ts diff --git a/packages/beacon-node/src/chain/bls/asyncBlst/index.ts b/packages/beacon-node/src/chain/bls/asyncBlst/index.ts deleted file mode 100644 index 6a3cd7308ce5..000000000000 --- a/packages/beacon-node/src/chain/bls/asyncBlst/index.ts +++ /dev/null @@ -1,538 +0,0 @@ -/* eslint-disable @typescript-eslint/strict-boolean-expressions */ -import {PublicKey, Signature} from "@chainsafe/blst"; -import {Logger} from "@lodestar/utils"; -import {ISignatureSet} from "@lodestar/state-transition"; -import {QueueError, QueueErrorCode} from "../../../util/queue/index.js"; -import {Metrics} from "../../../metrics/index.js"; -import {callInNextEventLoop} from "../../../util/eventLoop.js"; -import {LinkedList} from "../../../util/array.js"; -import {IBlsVerifier, VerifySignatureOpts} from "../interface.js"; -import {getAggregatedPubkey, getAggregatedPubkeysCount} from "../utils.js"; -import {chunkifyMaximizeChunkSize} from "../multithread/utils.js"; -import {defaultPoolSize} from "../multithread/poolSize.js"; -import {BlsWorkReq, BlsWorkResult, SignatureSet, WorkResult, WorkResultCode, WorkResultError} from "./types.js"; -import { - JobQueueItem, - JobQueueItemSameMessage, - JobQueueItemType, - jobItemSameMessageToMultiSet, - jobItemSigSets, - jobItemWorkReq, -} from "./jobItem.js"; -import {verifySignatureSetsMaybeBatch} from "./verifySignatures.js"; - -export type BlsMultiThreadWorkerPoolModules = { - logger: Logger; - metrics: Metrics | null; -}; - -export type {JobQueueItemType}; - -/** - * Split big signature sets into smaller sets so they can be sent to multiple workers. - * - * The biggest sets happen during sync, on mainnet batches of 64 blocks have around ~8000 signatures. - * The latency cost of sending the job to and from the worker is approx a single sig verification. - * If you split a big signature into 2, the extra time cost is `(2+2N)/(1+2N)`. - * For 128, the extra time cost is about 0.3%. No specific reasoning for `128`, it's just good enough. - */ -const MAX_SIGNATURE_SETS_PER_JOB = 128; - -/** - * If there are more than `MAX_BUFFERED_SIGS` buffered sigs, verify them immediately without waiting `MAX_BUFFER_WAIT_MS`. - * - * The efficiency improvement of batching sets asymptotically reaches x2. However, for batching large sets - * has more risk in case a signature is invalid, requiring to revalidate all sets in the batch. 32 is sweet - * point for this tradeoff. - */ -const MAX_BUFFERED_SIGS = 32; -/** - * Gossip objects usually come in bursts. Buffering them for a short period of time allows to increase batching - * efficiency, at the cost of delaying validation. Unless running in production shows otherwise, it's not critical - * to hold attestations and aggregates for 100ms. Lodestar existing queues may hold those objects for much more anyway. - * - * There's no exact reasoning for the `100` milliseconds number. The metric `batchSigsSuccess` should indicate if this - * value needs revision - */ -const MAX_BUFFER_WAIT_MS = 100; - -/** - * Max concurrent jobs on `canAcceptWork` status - */ -const MAX_JOBS_CAN_ACCEPT_WORK = 512; - -/** - * Split batchable sets in chunks of minimum size 16. - * Batch verify 16 has an aprox cost of 16+1. For 32 it's 32+1. After ~16 the additional savings are not significant. - * However, if a sig is invalid the whole batch has to be re-verified. So it's important to keep this number low. - * In normal network conditions almost all signatures received by the node are correct. - * After observing metrics this number can be reviewed - */ -const BATCHABLE_MIN_PER_CHUNK = 16; - -/** - */ -export class BlsAsyncBlstVerifier implements IBlsVerifier { - private readonly logger: Logger; - private readonly metrics: Metrics | null; - private readonly poolSize = defaultPoolSize; - private workerId = 0; - - private readonly jobs = new LinkedList(); - private bufferedJobs: { - jobs: LinkedList; - prioritizedJobs: LinkedList; - sigCount: number; - firstPush: number; - timeout: NodeJS.Timeout; - } | null = null; - private closed = false; - private workersBusy = 0; - - constructor(modules: BlsMultiThreadWorkerPoolModules) { - const {logger, metrics} = modules; - this.logger = logger; - this.metrics = metrics; - - if (metrics) { - metrics.blsThreadPool.queueLength.addCollect(() => { - metrics.blsThreadPool.queueLength.set(this.jobs.length); - metrics.blsThreadPool.workersBusy.set(this.workersBusy); - }); - } - } - - canAcceptWork(): boolean { - return ( - this.workersBusy < this.poolSize && - // TODO: Should also bound the jobs queue? - this.jobs.length < MAX_JOBS_CAN_ACCEPT_WORK - ); - } - - async verifySignatureSets(sets: ISignatureSet[], opts: VerifySignatureOpts = {}): Promise { - // Pubkeys are aggregated in the main thread regardless if verified in workers or in main thread - this.metrics?.bls.aggregatedPubkeys.inc(getAggregatedPubkeysCount(sets)); - this.metrics?.blsThreadPool.totalSigSets.inc(sets.length); - if (opts.priority) { - this.metrics?.blsThreadPool.prioritizedSigSets.inc(sets.length); - } - if (opts.batchable) { - this.metrics?.blsThreadPool.batchableSigSets.inc(sets.length); - } - - if (opts.verifyOnMainThread) { - const timer = this.metrics?.blsThreadPool.mainThreadDurationInThreadPool.startTimer(); - try { - return await verifySignatureSetsMaybeBatch( - sets.map((set) => ({ - publicKey: getAggregatedPubkey(set), - message: set.signingRoot.valueOf(), - signature: Signature.fromBytes(set.signature, true), - })) - ); - } finally { - if (timer) timer(); - } - } - - // Split large array of sets into smaller. - // Very helpful when syncing finalized, sync may submit +1000 sets so chunkify allows to distribute to many workers - const results = await Promise.all( - chunkifyMaximizeChunkSize(sets, MAX_SIGNATURE_SETS_PER_JOB).map( - (setsChunk) => - new Promise((resolve, reject) => { - return this.queueBlsWork({ - type: JobQueueItemType.default, - resolve, - reject, - addedTimeMs: Date.now(), - opts, - sets: setsChunk, - }); - }) - ) - ); - - // .every on an empty array returns true - if (results.length === 0) { - throw Error("Empty results array"); - } - - return results.every((isValid) => isValid === true); - } - - /** - * Verify signature sets of the same message, only supports worker verification. - */ - async verifySignatureSetsSameMessage( - sets: {publicKey: PublicKey; signature: Uint8Array}[], - message: Uint8Array, - opts: Omit = {} - ): Promise { - // chunkify so that it reduce the risk of retrying when there is at least one invalid signature - const results = await Promise.all( - chunkifyMaximizeChunkSize(sets, MAX_SIGNATURE_SETS_PER_JOB).map( - (setsChunk) => - new Promise((resolve, reject) => { - this.queueBlsWork({ - type: JobQueueItemType.sameMessage, - resolve, - reject, - addedTimeMs: Date.now(), - opts, - sets: setsChunk, - message, - }); - }) - ) - ); - - return results.flat(); - } - - async close(): Promise { - if (this.bufferedJobs) { - clearTimeout(this.bufferedJobs.timeout); - } - - // Abort all jobs - for (const job of this.jobs) { - job.reject(new QueueError({code: QueueErrorCode.QUEUE_ABORTED})); - } - this.jobs.clear(); - } - - /** - * Register BLS work to be done eventually in a worker - */ - private queueBlsWork(job: JobQueueItem): void { - if (this.closed) { - throw new QueueError({code: QueueErrorCode.QUEUE_ABORTED}); - } - - // TODO: Consider if limiting queue size is necessary here. - // It would be bad to reject signatures because the node is slow. - // However, if the worker communication broke jobs won't ever finish - - // Append batchable sets to `bufferedJobs`, starting a timeout to push them into `jobs`. - // Do not call `runJob()`, it is called from `runBufferedJobs()` - if (job.opts.batchable) { - if (!this.bufferedJobs) { - this.bufferedJobs = { - jobs: new LinkedList(), - prioritizedJobs: new LinkedList(), - sigCount: 0, - firstPush: Date.now(), - timeout: setTimeout(this.runBufferedJobs, MAX_BUFFER_WAIT_MS), - }; - } - const jobs = job.opts.priority ? this.bufferedJobs.prioritizedJobs : this.bufferedJobs.jobs; - jobs.push(job); - this.bufferedJobs.sigCount += jobItemSigSets(job); - if (this.bufferedJobs.sigCount > MAX_BUFFERED_SIGS) { - clearTimeout(this.bufferedJobs.timeout); - this.runBufferedJobs(); - } - } - - // Push job and schedule to call `runJob` in the next macro event loop cycle. - // This is useful to allow batching job submitted from a synchronous for loop, - // and to prevent large stacks since runJob may be called recursively. - else { - if (job.opts.priority) { - this.jobs.unshift(job); - } else { - this.jobs.push(job); - } - callInNextEventLoop(this.runJob); - } - } - - /** - * Potentially submit jobs to an idle worker, only if there's a worker and jobs - */ - private runJob = async (): Promise => { - if (this.closed) { - return; - } - - // Prepare work package - const jobsInput = this.prepareWork(); - if (jobsInput.length === 0) { - return; - } - - this.workersBusy++; - - try { - let startedJobsDefault = 0; - let startedJobsSameMessage = 0; - let startedSetsDefault = 0; - let startedSetsSameMessage = 0; - const workReqs: BlsWorkReq[] = []; - const jobsStarted: JobQueueItem[] = []; - - for (const job of jobsInput) { - this.metrics?.blsThreadPool.jobWaitTime.observe((Date.now() - job.addedTimeMs) / 1000); - - let workReq: BlsWorkReq; - try { - // Note: This can throw, must be handled per-job. - // Pubkey and signature aggregation is defered here - workReq = await jobItemWorkReq(job, this.metrics); - } catch (e) { - this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({type: job.type}); - - switch (job.type) { - case JobQueueItemType.default: - job.reject(e as Error); - break; - - case JobQueueItemType.sameMessage: - // there could be an invalid pubkey/signature, retry each individually - this.retryJobItemSameMessage(job); - break; - } - - continue; - } - // Re-push all jobs with matching workReq for easier accounting of results - workReqs.push(workReq); - jobsStarted.push(job); - - if (job.type === JobQueueItemType.sameMessage) { - startedJobsSameMessage += 1; - startedSetsSameMessage += job.sets.length; - } else { - startedJobsDefault += 1; - startedSetsDefault += job.sets.length; - } - } - - const startedSigSets = startedSetsDefault + startedSetsSameMessage; - this.metrics?.blsThreadPool.totalJobsGroupsStarted.inc(1); - this.metrics?.blsThreadPool.totalJobsStarted.inc({type: JobQueueItemType.default}, startedJobsDefault); - this.metrics?.blsThreadPool.totalJobsStarted.inc({type: JobQueueItemType.sameMessage}, startedJobsSameMessage); - this.metrics?.blsThreadPool.totalSigSetsStarted.inc({type: JobQueueItemType.default}, startedSetsDefault); - this.metrics?.blsThreadPool.totalSigSetsStarted.inc({type: JobQueueItemType.sameMessage}, startedSetsSameMessage); - - // Send work package to the worker - // If the job, metrics or any code below throws: the job will reject never going stale. - // Only downside is the job promise may be resolved twice, but that's not an issue - - const [jobStartSec, jobStartNs] = process.hrtime(); - const workResult = await verifyManySignatureSets(workReqs); - const [jobEndSec, jobEndNs] = process.hrtime(); - const {batchRetries, batchSigsSuccess, workerStartTime, workerEndTime, results} = workResult; - - const [workerStartSec, workerStartNs] = workerStartTime; - const [workerEndSec, workerEndNs] = workerEndTime; - - let successCount = 0; - let errorCount = 0; - - // Un-wrap work package - for (let i = 0; i < jobsStarted.length; i++) { - const job = jobsStarted[i]; - const jobResult = results[i]; - const sigSetCount = jobItemSigSets(job); - - // TODO: enable exhaustive switch case checks lint rule - switch (job.type) { - case JobQueueItemType.default: - if (!jobResult || jobResult.code !== WorkResultCode.success) { - job.reject(getJobResultError(jobResult, i)); - errorCount += sigSetCount; - } else { - job.resolve(jobResult.result); - successCount += sigSetCount; - } - break; - - // handle result of the verification of aggregated signature against aggregated pubkeys - case JobQueueItemType.sameMessage: - if (!jobResult || jobResult.code !== WorkResultCode.success) { - job.reject(getJobResultError(jobResult, i)); - errorCount += 1; - } else { - if (jobResult.result) { - // All are valid, most of the time it goes here - job.resolve(job.sets.map(() => true)); - } else { - // Retry each individually - this.retryJobItemSameMessage(job); - } - successCount += 1; - } - break; - } - } - - const workerId = this.workerId++ % this.poolSize; - const workerJobTimeSec = workerEndSec - workerStartSec + (workerEndNs - workerStartNs) / 1e9; - const latencyToWorkerSec = workerStartSec - jobStartSec + (workerStartNs - jobStartNs) / 1e9; - const latencyFromWorkerSec = jobEndSec - workerEndSec + Number(jobEndNs - workerEndNs) / 1e9; - - this.metrics?.blsThreadPool.timePerSigSet.observe(workerJobTimeSec / startedSigSets); - this.metrics?.blsThreadPool.jobsWorkerTime.inc({workerId}, workerJobTimeSec); - this.metrics?.blsThreadPool.latencyToWorker.observe(latencyToWorkerSec); - this.metrics?.blsThreadPool.latencyFromWorker.observe(latencyFromWorkerSec); - this.metrics?.blsThreadPool.successJobsSignatureSetsCount.inc(successCount); - this.metrics?.blsThreadPool.errorJobsSignatureSetsCount.inc(errorCount); - this.metrics?.blsThreadPool.batchRetries.inc(batchRetries); - this.metrics?.blsThreadPool.batchSigsSuccess.inc(batchSigsSuccess); - } catch (e) { - // Worker communications should never reject - if (!this.closed) { - this.logger.error("BlsMultiThreadWorkerPool error", {}, e as Error); - } - // Reject all - for (const job of jobsInput) { - job.reject(e as Error); - } - } - - this.workersBusy--; - - // Potentially run a new job - callInNextEventLoop(this.runJob); - }; - - /** - * Grab pending work up to a max number of signatures - */ - private prepareWork(): JobQueueItem[] { - const jobs: JobQueueItem[] = []; - let totalSigs = 0; - - while (totalSigs < MAX_SIGNATURE_SETS_PER_JOB) { - const job = this.jobs.shift(); - if (!job) { - break; - } - - jobs.push(job); - totalSigs += jobItemSigSets(job); - } - - return jobs; - } - - /** - * Add all buffered jobs to the job queue and potentially run them immediately - */ - private runBufferedJobs = (): void => { - if (this.bufferedJobs) { - for (const job of this.bufferedJobs.jobs) { - this.jobs.push(job); - } - for (const job of this.bufferedJobs.prioritizedJobs) { - this.jobs.unshift(job); - } - this.bufferedJobs = null; - callInNextEventLoop(this.runJob); - } - }; - - private retryJobItemSameMessage(job: JobQueueItemSameMessage): void { - // Create new jobs for each pubkey set, and Promise.all all the results - for (const j of jobItemSameMessageToMultiSet(job)) { - if (j.opts.priority) { - this.jobs.unshift(j); - } else { - this.jobs.push(j); - } - } - this.metrics?.blsThreadPool.sameMessageRetryJobs.inc(1); - this.metrics?.blsThreadPool.sameMessageRetrySets.inc(job.sets.length); - } - - /** For testing */ - private async waitTillInitialized(): Promise {} -} - -function getJobResultError(jobResult: WorkResultError | null, i: number): Error { - const workerError = jobResult ? Error(jobResult.error.message) : Error(`No jobResult for index ${i}`); - if (jobResult?.error?.stack) workerError.stack = jobResult.error.stack; - return workerError; -} - -async function verifyManySignatureSets(workReqArr: BlsWorkReq[]): Promise { - const [startSec, startNs] = process.hrtime(); - const results: WorkResult[] = []; - let batchRetries = 0; - let batchSigsSuccess = 0; - - // If there are multiple batchable sets attempt batch verification with them - const batchableSets: {idx: number; sets: SignatureSet[]}[] = []; - const nonBatchableSets: {idx: number; sets: SignatureSet[]}[] = []; - - // Split sets between batchable and non-batchable preserving their original index in the req array - for (let i = 0; i < workReqArr.length; i++) { - const workReq = workReqArr[i]; - const sets = workReq.sets; - - if (workReq.opts.batchable) { - batchableSets.push({idx: i, sets}); - } else { - nonBatchableSets.push({idx: i, sets}); - } - } - - if (batchableSets.length > 0) { - // Split batchable into chunks of max size ~ 32 to minimize cost if a sig is wrong - const batchableChunks = chunkifyMaximizeChunkSize(batchableSets, BATCHABLE_MIN_PER_CHUNK); - - for (const batchableChunk of batchableChunks) { - const allSets: SignatureSet[] = []; - for (const {sets} of batchableChunk) { - for (const set of sets) { - allSets.push(set); - } - } - - try { - // Attempt to verify multiple sets at once - const isValid = await verifySignatureSetsMaybeBatch(allSets); - - if (isValid) { - // The entire batch is valid, return success to all - for (const {idx, sets} of batchableChunk) { - batchSigsSuccess += sets.length; - results[idx] = {code: WorkResultCode.success, result: isValid}; - } - } else { - batchRetries++; - // Re-verify all sigs - nonBatchableSets.push(...batchableChunk); - } - } catch (e) { - // TODO: Ignore this error expecting that the same error will happen when re-verifying the set individually - // It's not ideal but '@chainsafe/blst' may throw errors on some conditions - batchRetries++; - // Re-verify all sigs - nonBatchableSets.push(...batchableChunk); - } - } - } - - for (const {idx, sets} of nonBatchableSets) { - try { - const isValid = await verifySignatureSetsMaybeBatch(sets); - results[idx] = {code: WorkResultCode.success, result: isValid}; - } catch (e) { - results[idx] = {code: WorkResultCode.error, error: e as Error}; - } - } - - const [workerEndSec, workerEndNs] = process.hrtime(); - - return { - batchRetries, - batchSigsSuccess, - workerStartTime: [startSec, startNs], - workerEndTime: [workerEndSec, workerEndNs], - results, - }; -} diff --git a/packages/beacon-node/src/chain/bls/asyncBlst/jobItem.ts b/packages/beacon-node/src/chain/bls/asyncBlst/jobItem.ts deleted file mode 100644 index e4bbabf948d9..000000000000 --- a/packages/beacon-node/src/chain/bls/asyncBlst/jobItem.ts +++ /dev/null @@ -1,127 +0,0 @@ -import {PublicKey, Signature, aggregatePublicKeys, aggregateSerializedSignaturesAsync} from "@chainsafe/blst"; -import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; -import {VerifySignatureOpts} from "../interface.js"; -import {getAggregatedPubkey} from "../utils.js"; -import {LinkedList} from "../../../util/array.js"; -import {Metrics} from "../../../metrics/metrics.js"; -import {BlsWorkReq} from "./types.js"; - -export type JobQueueItem = JobQueueItemDefault | JobQueueItemSameMessage; - -export type JobQueueItemDefault = { - type: JobQueueItemType.default; - resolve: (result: boolean) => void; - reject: (error?: Error) => void; - addedTimeMs: number; - opts: VerifySignatureOpts; - sets: ISignatureSet[]; -}; - -export type JobQueueItemSameMessage = { - type: JobQueueItemType.sameMessage; - resolve: (result: boolean[]) => void; - reject: (error?: Error) => void; - addedTimeMs: number; - opts: VerifySignatureOpts; - sets: {publicKey: PublicKey; signature: Uint8Array}[]; - message: Uint8Array; -}; - -export enum JobQueueItemType { - default = "default", - sameMessage = "same_message", -} - -/** - * Return count of signature sets from a JobQueueItem - */ -export function jobItemSigSets(job: JobQueueItem): number { - switch (job.type) { - case JobQueueItemType.default: - return job.sets.length; - case JobQueueItemType.sameMessage: - return 1; - } -} - -/** - * Prepare BlsWorkReq from JobQueueItem - * WARNING: May throw with untrusted user input - */ -export async function jobItemWorkReq(job: JobQueueItem, metrics: Metrics | null): Promise { - switch (job.type) { - case JobQueueItemType.default: - return { - opts: job.opts, - sets: job.sets.map((set) => ({ - // this can throw, handled in the consumer code - publicKey: getAggregatedPubkey(set, metrics), - signature: Signature.fromBytes(set.signature, true), - message: set.signingRoot, - })), - }; - case JobQueueItemType.sameMessage: { - // validate signature = true, this is slow code on main thread so should only run with network thread mode (useWorker=true) - // For a node subscribing to all subnets, with 1 signature per validator per epoch it takes around 80s - // to deserialize 750_000 signatures per epoch - // cpu profile on main thread has 250s idle so this only works until we reach 3M validators - // However, for normal node with only 2 to 7 subnet subscriptions per epoch this works until 27M validators - // and not a problem in the near future - // this is monitored on v1.11.0 https://github.com/ChainSafe/lodestar/pull/5912#issuecomment-1700320307 - const timer = metrics?.blsThreadPool.signatureDeserializationMainThreadDuration.startTimer(); - const signature = await aggregateSerializedSignaturesAsync( - job.sets.map((set) => set.signature), - true - ); - timer?.(); - - return { - opts: job.opts, - sets: [ - { - publicKey: aggregatePublicKeys(job.sets.map((set) => set.publicKey)), - signature, - message: job.message, - }, - ], - }; - } - } -} - -/** - * Convert a JobQueueItemSameMessage into multiple JobQueueItemDefault linked to the original promise - */ -export function jobItemSameMessageToMultiSet(job: JobQueueItemSameMessage): LinkedList { - // Retry each individually - // Create new jobs for each pubkey set, and Promise.all all the results - const promises: Promise[] = []; - const jobs = new LinkedList(); - - for (const set of job.sets) { - promises.push( - new Promise((resolve, reject) => { - jobs.push({ - type: JobQueueItemType.default, - resolve, - reject, - addedTimeMs: job.addedTimeMs, - opts: {batchable: false, priority: job.opts.priority}, - sets: [ - { - type: SignatureSetType.single, - pubkey: set.publicKey, - signature: set.signature, - signingRoot: job.message, - }, - ], - }); - }) - ); - } - - // Connect jobs to main job - Promise.all(promises).then(job.resolve, job.reject); - - return jobs; -} diff --git a/packages/beacon-node/src/chain/bls/asyncBlst/types.ts b/packages/beacon-node/src/chain/bls/asyncBlst/types.ts deleted file mode 100644 index 55470c76d658..000000000000 --- a/packages/beacon-node/src/chain/bls/asyncBlst/types.ts +++ /dev/null @@ -1,33 +0,0 @@ -import {PublicKey, Signature} from "@chainsafe/blst"; -import {VerifySignatureOpts} from "../interface.js"; - -export type SignatureSet = { - publicKey: PublicKey; - message: Uint8Array; - signature: Signature; -}; - -export type BlsWorkReq = { - opts: VerifySignatureOpts; - sets: SignatureSet[]; -}; - -export enum WorkResultCode { - success = "success", - error = "error", -} - -export type WorkResultError = {code: WorkResultCode.error; error: Error}; -export type WorkResult = {code: WorkResultCode.success; result: R} | WorkResultError; - -export type BlsWorkResult = { - /** Total num of batches that had to be retried */ - batchRetries: number; - /** Total num of sigs that have been successfully verified with batching */ - batchSigsSuccess: number; - /** Time worker function starts - UNIX timestamp in seconds and nanoseconds */ - workerStartTime: [number, number]; - /** Time worker function ends - UNIX timestamp in seconds and nanoseconds */ - workerEndTime: [number, number]; - results: WorkResult[]; -}; diff --git a/packages/beacon-node/src/chain/bls/asyncBlst/verifySignatures.ts b/packages/beacon-node/src/chain/bls/asyncBlst/verifySignatures.ts deleted file mode 100644 index 14583ff99aa3..000000000000 --- a/packages/beacon-node/src/chain/bls/asyncBlst/verifySignatures.ts +++ /dev/null @@ -1,40 +0,0 @@ -import {verifyAsync, verifyMultipleAggregateSignaturesAsync} from "@chainsafe/blst"; -import {SignatureSet} from "./types.js"; - -const MIN_SET_COUNT_TO_BATCH = 2; - -/** - * Verify signatures sets with batch verification or regular core verify depending on the set count. - * Abstracted in a separate file to be consumed by the threaded pool and the main thread implementation. - */ -export async function verifySignatureSetsMaybeBatch(sets: SignatureSet[]): Promise { - try { - if (sets.length >= MIN_SET_COUNT_TO_BATCH) { - return await verifyMultipleAggregateSignaturesAsync( - sets.map((s) => ({ - pk: s.publicKey, - msg: s.message, - sig: s.signature, - })) - ); - } - - // .every on an empty array returns true - if (sets.length === 0) { - throw Error("Empty signature set"); - } - - // If too few signature sets verify them without batching - for (const set of sets) { - if (!(await verifyAsync(set.message, set.publicKey, set.signature))) { - return false; - } - } - return true; - } catch (_) { - // A signature could be malformed, in that case fromBytes throws error - // blst-ts `verifyMultipleSignatures` is also a fallible operation if mul_n_aggregate fails - // see https://github.com/ChainSafe/blst-ts/blob/b1ba6333f664b08e5c50b2b0d18c4f079203962b/src/lib.ts#L291 - return false; - } -} diff --git a/packages/beacon-node/src/chain/bls/index.ts b/packages/beacon-node/src/chain/bls/index.ts index 347b65e7f6a2..f9898b13776b 100644 --- a/packages/beacon-node/src/chain/bls/index.ts +++ b/packages/beacon-node/src/chain/bls/index.ts @@ -2,4 +2,3 @@ export type {IBlsVerifier} from "./interface.js"; export type {BlsMultiThreadWorkerPoolModules, JobQueueItemType} from "./multithread/index.js"; export {BlsMultiThreadWorkerPool} from "./multithread/index.js"; export {BlsSingleThreadVerifier} from "./singleThread.js"; -export {BlsAsyncBlstVerifier} from "./asyncBlst/index.js"; diff --git a/packages/beacon-node/src/chain/bls/maybeBatch.ts b/packages/beacon-node/src/chain/bls/maybeBatch.ts index 76dd35720b78..e300b8ff8b76 100644 --- a/packages/beacon-node/src/chain/bls/maybeBatch.ts +++ b/packages/beacon-node/src/chain/bls/maybeBatch.ts @@ -1,10 +1,4 @@ -import { - PublicKey, - Signature, - verify, - verifyMultipleAggregateSignatures, - verifyMultipleAggregateSignaturesAsync, -} from "@chainsafe/blst"; +import {PublicKey, Signature, verify, verifyMultipleAggregateSignatures} from "@chainsafe/blst"; const MIN_SET_COUNT_TO_BATCH = 2; @@ -49,35 +43,3 @@ export function verifySignatureSetsMaybeBatch(sets: SignatureSetDeserialized[]): return false; } } - -export async function verifySignatureSetsMaybeBatchAsync(sets: SignatureSetDeserialized[]): Promise { - try { - if (sets.length >= MIN_SET_COUNT_TO_BATCH) { - return await verifyMultipleAggregateSignaturesAsync( - sets.map((s) => ({ - pk: s.publicKey, - msg: s.message, - // true = validate signature - sig: Signature.fromBytes(s.signature, true), - })) - ); - } - - // .every on an empty array returns true - if (sets.length === 0) { - throw Error("Empty signature set"); - } - - // If too few signature sets verify them without batching - return sets.every((set) => { - // true = validate signature - const sig = Signature.fromBytes(set.signature, true); - return verify(set.message, set.publicKey, sig); - }); - } catch (_) { - // A signature could be malformed, in that case fromBytes throws error - // blst-ts `verifyMultipleSignatures` is also a fallible operation if mul_n_aggregate fails - // see https://github.com/ChainSafe/blst-ts/blob/b1ba6333f664b08e5c50b2b0d18c4f079203962b/src/lib.ts#L291 - return false; - } -} diff --git a/packages/beacon-node/src/chain/bls/utils.ts b/packages/beacon-node/src/chain/bls/utils.ts index 52a72508858d..63f2bdd80458 100644 --- a/packages/beacon-node/src/chain/bls/utils.ts +++ b/packages/beacon-node/src/chain/bls/utils.ts @@ -1,4 +1,4 @@ -import {PublicKey, aggregatePublicKeys, aggregatePublicKeysAsync} from "@chainsafe/blst"; +import {PublicKey, aggregatePublicKeys} from "@chainsafe/blst"; import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; import {Metrics} from "../../metrics/metrics.js"; @@ -19,26 +19,6 @@ export function getAggregatedPubkey(signatureSet: ISignatureSet, metrics: Metric } } -export async function getAggregatedPubkeyAsync( - signatureSet: ISignatureSet, - metrics: Metrics | null = null -): Promise { - switch (signatureSet.type) { - case SignatureSetType.single: - return signatureSet.pubkey; - - case SignatureSetType.aggregate: { - const timer = metrics?.blsThreadPool.pubkeysAggregationMainThreadDuration.startTimer(); - const pubkeys = await aggregatePublicKeysAsync(signatureSet.pubkeys); - timer?.(); - return pubkeys; - } - - default: - throw Error("Unknown signature set type"); - } -} - export function getAggregatedPubkeysCount(signatureSets: ISignatureSet[]): number { let pubkeysConut = 0; for (const set of signatureSets) { diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 12e054b68b70..61f044d4f895 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -61,7 +61,7 @@ import {IChainOptions} from "./options.js"; import {QueuedStateRegenerator, RegenCaller} from "./regen/index.js"; import {initializeForkChoice} from "./forkChoice/index.js"; import {computeAnchorCheckpoint} from "./initState.js"; -import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool, BlsAsyncBlstVerifier} from "./bls/index.js"; +import {IBlsVerifier, BlsSingleThreadVerifier, BlsMultiThreadWorkerPool} from "./bls/index.js"; import { SeenAttesters, SeenAggregators, @@ -214,11 +214,9 @@ export class BeaconChain implements IBeaconChain { const signal = this.abortController.signal; const emitter = new ChainEventEmitter(); // by default, verify signatures on both main threads and worker threads - const bls = opts.blsVerifyAllAsyncBlst - ? new BlsAsyncBlstVerifier({logger, metrics}) - : opts.blsVerifyAllMainThread - ? new BlsSingleThreadVerifier({metrics}) - : new BlsMultiThreadWorkerPool(opts, {logger, metrics}); + const bls = opts.blsVerifyAllMainThread + ? new BlsSingleThreadVerifier({metrics}) + : new BlsMultiThreadWorkerPool(opts, {logger, metrics}); if (!clock) clock = new Clock({config, genesisTime: this.genesisTime, signal}); diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index b50e7ab1cbe0..7c7cfcdde75b 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -19,7 +19,6 @@ export type IChainOptions = BlockProcessOpts & LightClientServerOpts & { blsVerifyAllMainThread?: boolean; blsVerifyAllMultiThread?: boolean; - blsVerifyAllAsyncBlst?: boolean; persistProducedBlocks?: boolean; persistInvalidSszObjects?: boolean; persistInvalidSszObjectsDir?: string; @@ -95,7 +94,6 @@ export type SeenCacheOpts = { export const defaultChainOptions: IChainOptions = { blsVerifyAllMainThread: false, blsVerifyAllMultiThread: false, - blsVerifyAllAsyncBlst: false, disableBlsBatchVerify: false, proposerBoost: true, proposerBoostReorg: false, diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts index 3b7ad1e42186..aae97b6db68f 100644 --- a/packages/cli/src/options/beaconNodeOptions/chain.ts +++ b/packages/cli/src/options/beaconNodeOptions/chain.ts @@ -6,7 +6,6 @@ export type ChainArgs = { suggestedFeeRecipient: string; "chain.blsVerifyAllMultiThread"?: boolean; "chain.blsVerifyAllMainThread"?: boolean; - "chain.blsVerifyAllAsyncBlst"?: boolean; "chain.disableBlsBatchVerify"?: boolean; "chain.persistProducedBlocks"?: boolean; "chain.persistInvalidSszObjects"?: boolean; @@ -102,14 +101,6 @@ export const options: CliCommandOptions = { group: "chain", }, - "chain.blsVerifyAllAsyncBlst": { - hidden: true, - type: "boolean", - description: "Always use async blst for BLS verification", - defaultDescription: String(defaultOptions.chain.blsVerifyAllAsyncBlst), - group: "chain", - }, - "chain.disableBlsBatchVerify": { hidden: true, type: "boolean", From da7a66419eb296169013db46f6e408217a1f11b1 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 23 Jul 2024 14:21:35 -0400 Subject: [PATCH 10/18] fix: spec tests --- packages/beacon-node/test/spec/bls/bls.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/beacon-node/test/spec/bls/bls.ts b/packages/beacon-node/test/spec/bls/bls.ts index 6432ca7de3cc..5af15bcb8eb2 100644 --- a/packages/beacon-node/test/spec/bls/bls.ts +++ b/packages/beacon-node/test/spec/bls/bls.ts @@ -38,11 +38,15 @@ export const testFnByType: Record any)> = { */ function aggregate_verify(input: {pubkeys: string[]; messages: string[]; signature: string}): boolean { const {pubkeys, messages, signature} = input; - return aggregateVerify( - messages.map(fromHexString), - pubkeys.map((pk) => PublicKey.fromHex(pk)), - Signature.fromHex(signature) - ); + try { + return aggregateVerify( + messages.map(fromHexString), + pubkeys.map((pk) => PublicKey.fromHex(pk)), + Signature.fromHex(signature) + ); + } catch (e) { + return false; + } } /** @@ -131,7 +135,11 @@ function sign(input: {privkey: string; message: string}): string | null { */ function verify(input: {pubkey: string; message: string; signature: string}): boolean { const {pubkey, message, signature} = input; - return _verify(fromHexString(message), PublicKey.fromHex(pubkey), Signature.fromHex(signature)); + try { + return _verify(fromHexString(message), PublicKey.fromHex(pubkey), Signature.fromHex(signature)); + } catch (e) { + return false; + } } /** From e57c3315f0ce7bc9df44cf0d102ffcdedd4b0df1 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 23 Jul 2024 14:28:06 -0400 Subject: [PATCH 11/18] fix: light client tests --- packages/light-client/package.json | 1 + yarn.lock | 158 +++++++++++++++++++++++++++-- 2 files changed, 151 insertions(+), 8 deletions(-) diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 8b433375f07f..6576bddfee6a 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -74,6 +74,7 @@ }, "dependencies": { "@chainsafe/bls": "7.1.3", + "@chainsafe/blst": "^0.2.0", "@chainsafe/persistent-merkle-tree": "^0.7.1", "@chainsafe/ssz": "^0.15.1", "@lodestar/api": "^1.20.2", diff --git a/yarn.lock b/yarn.lock index 2a565294d128..d5c705410e0c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -331,6 +331,15 @@ resolved "https://registry.yarnpkg.com/@chainsafe/blst-win32-x64-msvc/-/blst-win32-x64-msvc-2.0.1.tgz#dfebc2ea23875bc02ffac11d2d15bf6e644d59ed" integrity sha512-RcOo1Sl1Ai5igp56l3I5kwWkesfbD14PBoUcoR61phvIHnhyZzOtVX6rhuCYwcl0IIgSDSWtNNMdSwX34GGmxA== +"@chainsafe/blst@^0.2.0": + version "0.2.11" + resolved "https://registry.yarnpkg.com/@chainsafe/blst/-/blst-0.2.11.tgz#5ec85cd663592819d1dc51127e75dfd834250e3d" + integrity sha512-URyOLq5GtxBoxibOnd2pgLydCy0UZzbiIIBcsRAvGxAsRzjZL04TsQfwRkz5aphU3a1ebeRoMmI/HHyMCiFSQg== + dependencies: + "@types/tar" "^6.1.4" + node-fetch "^2.6.1" + node-gyp "^8.4.0" + "@chainsafe/blst@^2.0.1": version "2.0.1" resolved "https://registry.yarnpkg.com/@chainsafe/blst/-/blst-2.0.1.tgz#9e8ceb6766fcb231e4fa378eee6378275018a159" @@ -1298,7 +1307,7 @@ rfdc "^1.3.0" yaml "^2.2.2" -"@gar/promisify@^1.1.3": +"@gar/promisify@^1.0.1", "@gar/promisify@^1.1.3": version "1.1.3" resolved "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz" integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw== @@ -2040,6 +2049,14 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" +"@npmcli/fs@^1.0.0": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@npmcli/fs/-/fs-1.1.1.tgz#72f719fe935e687c56a4faecf3c03d06ba593257" + integrity sha512-8KG5RD0GVP4ydEzRn/I4BNDuxDtqVbOdm8675T49OIG/NGhaK0pjPX7ZcDlvKYbA+ulvVK3ztfcF4uBdOxuJbQ== + dependencies: + "@gar/promisify" "^1.0.1" + semver "^7.3.5" + "@npmcli/fs@^2.1.0": version "2.1.0" resolved "https://registry.yarnpkg.com/@npmcli/fs/-/fs-2.1.0.tgz#f2a21c28386e299d1a9fae8051d35ad180e33109" @@ -2077,6 +2094,14 @@ npm-bundled "^3.0.0" npm-normalize-package-bin "^3.0.0" +"@npmcli/move-file@^1.0.1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@npmcli/move-file/-/move-file-1.1.2.tgz#1a82c3e372f7cae9253eb66d72543d6b8685c674" + integrity sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg== + dependencies: + mkdirp "^1.0.4" + rimraf "^3.0.2" + "@npmcli/move-file@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@npmcli/move-file/-/move-file-2.0.0.tgz#417f585016081a0184cef3e38902cd917a9bbd02" @@ -2799,6 +2824,11 @@ dependencies: defer-to-connect "^2.0.1" +"@tootallnate/once@1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" + integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== + "@tootallnate/once@2": version "2.0.0" resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf" @@ -3593,7 +3623,7 @@ JSONStream@^1.3.5: jsonparse "^1.2.0" through ">=2.2.7 <3" -abbrev@^1.0.0: +abbrev@1, abbrev@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== @@ -3677,6 +3707,13 @@ agent-base@^7.0.2, agent-base@^7.1.0: dependencies: debug "^4.3.4" +agentkeepalive@^4.1.3: + version "4.5.0" + resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz#2673ad1389b3c418c5a20c5d7364f93ca04be923" + integrity sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew== + dependencies: + humanize-ms "^1.2.1" + agentkeepalive@^4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.2.1.tgz#a7975cbb9f83b367f06c90cc51ff28fe7d499717" @@ -4513,6 +4550,30 @@ cac@^6.7.14: resolved "https://registry.yarnpkg.com/cac/-/cac-6.7.14.tgz#804e1e6f506ee363cb0e3ccbb09cad5dd9870959" integrity sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ== +cacache@^15.2.0: + version "15.3.0" + resolved "https://registry.yarnpkg.com/cacache/-/cacache-15.3.0.tgz#dc85380fb2f556fe3dda4c719bfa0ec875a7f1eb" + integrity sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ== + dependencies: + "@npmcli/fs" "^1.0.0" + "@npmcli/move-file" "^1.0.1" + chownr "^2.0.0" + fs-minipass "^2.0.0" + glob "^7.1.4" + infer-owner "^1.0.4" + lru-cache "^6.0.0" + minipass "^3.1.1" + minipass-collect "^1.0.2" + minipass-flush "^1.0.5" + minipass-pipeline "^1.2.2" + mkdirp "^1.0.3" + p-map "^4.0.0" + promise-inflight "^1.0.1" + rimraf "^3.0.2" + ssri "^8.0.1" + tar "^6.0.2" + unique-filename "^1.1.1" + cacache@^16.1.0: version "16.1.1" resolved "https://registry.yarnpkg.com/cacache/-/cacache-16.1.1.tgz#4e79fb91d3efffe0630d5ad32db55cc1b870669c" @@ -5727,7 +5788,7 @@ enabled@2.0.x: resolved "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz" integrity sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ== -encoding@^0.1.13: +encoding@^0.1.12, encoding@^0.1.13: version "0.1.13" resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.13.tgz#56574afdd791f54a8e9b2785c0582a2d26210fa9" integrity sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A== @@ -7310,6 +7371,15 @@ http-errors@2.0.0: statuses "2.0.1" toidentifier "1.0.1" +http-proxy-agent@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a" + integrity sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg== + dependencies: + "@tootallnate/once" "1" + agent-base "6" + debug "4" + http-proxy-agent@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz#5129800203520d434f142bc78ff3c170800f2b43" @@ -8890,6 +8960,28 @@ make-fetch-happen@^11.0.0, make-fetch-happen@^11.0.1, make-fetch-happen@^11.1.1: socks-proxy-agent "^7.0.0" ssri "^10.0.0" +make-fetch-happen@^9.1.0: + version "9.1.0" + resolved "https://registry.yarnpkg.com/make-fetch-happen/-/make-fetch-happen-9.1.0.tgz#53085a09e7971433e6765f7971bf63f4e05cb968" + integrity sha512-+zopwDy7DNknmwPQplem5lAZX/eCOzSvSNNcSKm5eVwTkOBzoktEfXsa9L23J/GIRhxRsaxzkPEhrJEpE2F4Gg== + dependencies: + agentkeepalive "^4.1.3" + cacache "^15.2.0" + http-cache-semantics "^4.1.0" + http-proxy-agent "^4.0.1" + https-proxy-agent "^5.0.0" + is-lambda "^1.0.1" + lru-cache "^6.0.0" + minipass "^3.1.3" + minipass-collect "^1.0.2" + minipass-fetch "^1.3.2" + minipass-flush "^1.0.5" + minipass-pipeline "^1.2.4" + negotiator "^0.6.2" + promise-retry "^2.0.1" + socks-proxy-agent "^6.0.0" + ssri "^8.0.0" + map-obj@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d" @@ -9141,6 +9233,17 @@ minipass-collect@^1.0.2: dependencies: minipass "^3.0.0" +minipass-fetch@^1.3.2: + version "1.4.1" + resolved "https://registry.yarnpkg.com/minipass-fetch/-/minipass-fetch-1.4.1.tgz#d75e0091daac1b0ffd7e9d41629faff7d0c1f1b6" + integrity sha512-CGH1eblLq26Y15+Azk7ey4xh0J/XfJfrCox5LDJiKqI2Q2iwOLOKrlmIaODiSQS8d18jalF6y2K2ePUm0CmShw== + dependencies: + minipass "^3.1.0" + minipass-sized "^1.0.3" + minizlib "^2.0.0" + optionalDependencies: + encoding "^0.1.12" + minipass-fetch@^2.0.3: version "2.1.0" resolved "https://registry.yarnpkg.com/minipass-fetch/-/minipass-fetch-2.1.0.tgz#ca1754a5f857a3be99a9271277246ac0b44c3ff8" @@ -9178,7 +9281,7 @@ minipass-json-stream@^1.0.1: jsonparse "^1.3.1" minipass "^3.0.0" -minipass-pipeline@^1.2.4: +minipass-pipeline@^1.2.2, minipass-pipeline@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz#68472f79711c084657c067c5c6ad93cddea8214c" integrity sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A== @@ -9192,7 +9295,7 @@ minipass-sized@^1.0.3: dependencies: minipass "^3.0.0" -minipass@^3.0.0, minipass@^3.1.1, minipass@^3.1.6: +minipass@^3.0.0, minipass@^3.1.0, minipass@^3.1.1, minipass@^3.1.3, minipass@^3.1.6: version "3.3.6" resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.6.tgz#7bba384db3a1520d18c9c0e5251c3444e95dd94a" integrity sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw== @@ -9214,7 +9317,7 @@ minipass@^5.0.0: resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.0.4.tgz#dbce03740f50a4786ba994c1fb908844d27b038c" integrity sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ== -minizlib@^2.1.1, minizlib@^2.1.2: +minizlib@^2.0.0, minizlib@^2.1.1, minizlib@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" integrity sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg== @@ -9406,7 +9509,7 @@ natural-compare@^1.4.0: resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= -negotiator@^0.6.3: +negotiator@^0.6.2, negotiator@^0.6.3: version "0.6.3" resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz" integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== @@ -9469,6 +9572,22 @@ node-gyp-build@^4.3.0: resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.5.0.tgz#7a64eefa0b21112f89f58379da128ac177f20e40" integrity sha512-2iGbaQBV+ITgCz76ZEjmhUKAKVf7xfY1sRl4UiKQspfZMH2h06SyhNsnSVy50cwkFQDGLyif6m/6uFXHkOZ6rg== +node-gyp@^8.4.0: + version "8.4.1" + resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-8.4.1.tgz#3d49308fc31f768180957d6b5746845fbd429937" + integrity sha512-olTJRgUtAb/hOXG0E93wZDs5YiJlgbXxTwQAFHyNlRsXQnYzUaF2aGgujZbw+hR8aF4ZG/rST57bWMWD16jr9w== + dependencies: + env-paths "^2.2.0" + glob "^7.1.4" + graceful-fs "^4.2.6" + make-fetch-happen "^9.1.0" + nopt "^5.0.0" + npmlog "^6.0.0" + rimraf "^3.0.2" + semver "^7.3.5" + tar "^6.1.2" + which "^2.0.2" + node-gyp@^9.0.0, node-gyp@^9.4.0: version "9.4.1" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-9.4.1.tgz#8a1023e0d6766ecb52764cc3a734b36ff275e185" @@ -9553,6 +9672,13 @@ node-stdlib-browser@^1.2.0: util "^0.12.4" vm-browserify "^1.0.1" +nopt@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-5.0.0.tgz#530942bb58a512fccafe53fe210f13a25355dc88" + integrity sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ== + dependencies: + abbrev "1" + nopt@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/nopt/-/nopt-6.0.0.tgz#245801d8ebf409c6df22ab9d95b65e1309cdb16d" @@ -11430,6 +11556,15 @@ snappyjs@^0.7.0: resolved "https://registry.yarnpkg.com/snappyjs/-/snappyjs-0.7.0.tgz#6096eac06382700ae7fdefa579dea5e2aa20f51c" integrity sha512-u5iEEXkMe2EInQio6Wv9LWHOQYRDbD2O9hzS27GpT/lwfIQhTCnHCTqedqHIHe9ZcvQo+9au6vngQayipz1NYw== +socks-proxy-agent@^6.0.0: + version "6.2.1" + resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-6.2.1.tgz#2687a31f9d7185e38d530bef1944fe1f1496d6ce" + integrity sha512-a6KW9G+6B3nWZ1yB8G7pJwL3ggLy1uTzKAgCb7ttblwqdz9fMGJUuTy3uFzEP48FAs9FLILlmzDlE2JJhVQaXQ== + dependencies: + agent-base "^6.0.2" + debug "^4.3.3" + socks "^2.6.2" + socks-proxy-agent@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-7.0.0.tgz#dc069ecf34436621acb41e3efa66ca1b5fed15b6" @@ -11584,6 +11719,13 @@ ssri@^10.0.0, ssri@^10.0.1: dependencies: minipass "^4.0.0" +ssri@^8.0.0, ssri@^8.0.1: + version "8.0.1" + resolved "https://registry.yarnpkg.com/ssri/-/ssri-8.0.1.tgz#638e4e439e2ffbd2cd289776d5ca457c4f51a2af" + integrity sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ== + dependencies: + minipass "^3.1.1" + ssri@^9.0.0, ssri@^9.0.1: version "9.0.1" resolved "https://registry.yarnpkg.com/ssri/-/ssri-9.0.1.tgz#544d4c357a8d7b71a19700074b6883fcb4eae057" @@ -11990,7 +12132,7 @@ tar@6.1.11: mkdirp "^1.0.3" yallist "^4.0.0" -tar@^6.1.11, tar@^6.1.13, tar@^6.1.2: +tar@^6.0.2, tar@^6.1.11, tar@^6.1.13, tar@^6.1.2: version "6.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a" integrity sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A== From bc0b103afcd786917b951f6d34c5ad74ea1a466c Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 23 Jul 2024 14:39:03 -0400 Subject: [PATCH 12/18] chore: update comment --- .../beacon-node/src/chain/bls/multithread/jobItem.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts index b6390b5a45ef..f08cfe6bdc6c 100644 --- a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts +++ b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts @@ -61,13 +61,14 @@ export function jobItemWorkReq(job: JobQueueItem, metrics: Metrics | null): BlsW })), }; case JobQueueItemType.sameMessage: { - // validate signature = true, this is slow code on main thread so should only run with network thread mode (useWorker=true) - // For a node subscribing to all subnets, with 1 signature per validator per epoch it takes around 80s - // to deserialize 750_000 signatures per epoch + // This is slow code on main thread (mainly signature deserialization + group check). + // Ideally it can be taken off-thread, but in the mean time, keep track of total time spent here. + // As of July 2024, for a node subscribing to all subnets, with 1 signature per validator per epoch, + // it takes around 2.02 min to perform this operation for a single epoch. // cpu profile on main thread has 250s idle so this only works until we reach 3M validators // However, for normal node with only 2 to 7 subnet subscriptions per epoch this works until 27M validators // and not a problem in the near future - // this is monitored on v1.11.0 https://github.com/ChainSafe/lodestar/pull/5912#issuecomment-1700320307 + // this is monitored on v1.21.0 https://github.com/ChainSafe/lodestar/pull/6894/files#r1687359225 const timer = metrics?.blsThreadPool.signatureDeserializationMainThreadDuration.startTimer(); const {pk, sig} = aggregateWithRandomness(job.sets.map((set) => ({pk: set.publicKey, sig: set.signature}))); timer?.(); From 9734e1ce8c11f84cf02c425b14da0fc36a7dfb1c Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 23 Jul 2024 14:48:48 -0400 Subject: [PATCH 13/18] chore: update metrics --- dashboards/lodestar_block_processor.json | 4 ++-- dashboards/lodestar_bls_thread_pool.json | 8 ++++---- packages/beacon-node/src/chain/bls/multithread/jobItem.ts | 2 +- packages/beacon-node/src/metrics/metrics/lodestar.ts | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dashboards/lodestar_block_processor.json b/dashboards/lodestar_block_processor.json index e29d3522ce60..50513613c680 100644 --- a/dashboards/lodestar_block_processor.json +++ b/dashboards/lodestar_block_processor.json @@ -5101,7 +5101,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_bls_thread_pool_signature_deserialization_main_thread_time_seconds_bucket[$rate_interval])", + "expr": "rate(lodestar_gossip_block_gossip_validate_time_bucket[$rate_interval])", "format": "heatmap", "instant": false, "legendFormat": "time", @@ -5273,7 +5273,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_bls_thread_pool_signature_deserialization_main_thread_time_seconds_sum[$rate_interval]) / rate(lodestar_bls_thread_pool_signature_deserialization_main_thread_time_seconds_count[$rate_interval])", + "expr": "rate(lodestar_gossip_block_gossip_validate_time_sum[$rate_interval]) / rate(lodestar_gossip_block_gossip_validate_time_count[$rate_interval])", "format": "heatmap", "instant": false, "legendFormat": "time", diff --git a/dashboards/lodestar_bls_thread_pool.json b/dashboards/lodestar_bls_thread_pool.json index 92b7d00d3783..867d9fd322c4 100644 --- a/dashboards/lodestar_bls_thread_pool.json +++ b/dashboards/lodestar_bls_thread_pool.json @@ -1174,9 +1174,9 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_bls_thread_pool_signature_deserialization_main_thread_time_seconds_sum[$rate_interval]) * 384", + "expr": "rate(lodestar_bls_thread_pool_aggregate_with_randomness_main_thread_time_seconds_sum[$rate_interval]) * 384", "instant": false, - "legendFormat": "signature_deserialization", + "legendFormat": "aggregate_with_randomness", "range": true, "refId": "A" }, @@ -1270,7 +1270,7 @@ "disableTextWrap": false, "editorMode": "code", "exemplar": false, - "expr": "rate(lodestar_bls_thread_pool_signature_deserialization_main_thread_time_seconds_bucket[$rate_interval])", + "expr": "rate(lodestar_bls_thread_pool_aggregate_with_randomness_main_thread_time_seconds_bucket[$rate_interval])", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, @@ -1281,7 +1281,7 @@ "useBackend": false } ], - "title": "Main Thread Signature Aggregation Time", + "title": "Main Thread AggregateWithRandomness Time", "type": "heatmap" }, { diff --git a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts index f08cfe6bdc6c..035d56e56df2 100644 --- a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts +++ b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts @@ -69,7 +69,7 @@ export function jobItemWorkReq(job: JobQueueItem, metrics: Metrics | null): BlsW // However, for normal node with only 2 to 7 subnet subscriptions per epoch this works until 27M validators // and not a problem in the near future // this is monitored on v1.21.0 https://github.com/ChainSafe/lodestar/pull/6894/files#r1687359225 - const timer = metrics?.blsThreadPool.signatureDeserializationMainThreadDuration.startTimer(); + const timer = metrics?.blsThreadPool.aggregateWithRandomnessMainThreadDuration.startTimer(); const {pk, sig} = aggregateWithRandomness(job.sets.map((set) => ({pk: set.publicKey, sig: set.signature}))); timer?.(); diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 5ebdbac48959..f43a3f1cdbe6 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -482,9 +482,9 @@ export function createLodestarMetrics( name: "lodestar_bls_thread_pool_batchable_sig_sets_total", help: "Count of total batchable signature sets", }), - signatureDeserializationMainThreadDuration: register.histogram({ - name: "lodestar_bls_thread_pool_signature_deserialization_main_thread_time_seconds", - help: "Total time spent deserializing signatures on main thread", + aggregateWithRandomnessMainThreadDuration: register.histogram({ + name: "lodestar_bls_thread_pool_aggregate_with_randomness_main_thread_time_seconds", + help: "Total time performing aggregateWithRandomness on main thread", buckets: [0.001, 0.005, 0.01, 0.1], }), pubkeysAggregationMainThreadDuration: register.histogram({ From 3b5da4b610749c82e65684c005a92c079a5a8ff7 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 23 Jul 2024 14:54:43 -0400 Subject: [PATCH 14/18] fix: more spec tests --- packages/beacon-node/test/spec/general/bls.ts | 58 +++++++++++++------ 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/packages/beacon-node/test/spec/general/bls.ts b/packages/beacon-node/test/spec/general/bls.ts index 5d1e3b472417..010a5b9be53d 100644 --- a/packages/beacon-node/test/spec/general/bls.ts +++ b/packages/beacon-node/test/spec/general/bls.ts @@ -70,10 +70,14 @@ type BlsTestCase = { * output: BLS Signature -- expected output, single BLS signature or empty. * ``` */ -function aggregate(input: string[]): string { - const pks = input.map((pkHex) => Signature.fromHex(pkHex)); - const agg = aggregateSignatures(pks); - return agg.toHex(); +function aggregate(input: string[]): string | null { + try { + const pks = input.map((pkHex) => Signature.fromHex(pkHex)); + const agg = aggregateSignatures(pks); + return agg.toHex(); + } catch (e) { + return null; + } } /** @@ -87,11 +91,15 @@ function aggregate(input: string[]): string { */ function aggregate_verify(input: {pubkeys: string[]; messages: string[]; signature: string}): boolean { const {pubkeys, messages, signature} = input; - return aggregateVerify( - messages.map(fromHexString), - pubkeys.map((pk) => PublicKey.fromHex(pk)), - Signature.fromHex(signature) - ); + try { + return aggregateVerify( + messages.map(fromHexString), + pubkeys.map((pk) => PublicKey.fromHex(pk)), + Signature.fromHex(signature) + ); + } catch (e) { + return false; + } } /** @@ -106,7 +114,11 @@ function eth_aggregate_pubkeys(input: string[]): string | null { if (pk === G1_POINT_AT_INFINITY) return null; } - return aggregateSerializedPublicKeys(input.map((hex) => fromHexString(hex))).toHex(); + try { + return aggregateSerializedPublicKeys(input.map((hex) => fromHexString(hex))).toHex(); + } catch (e) { + return null; + } } /** @@ -130,11 +142,15 @@ function eth_fast_aggregate_verify(input: {pubkeys: string[]; message: string; s if (pk === G1_POINT_AT_INFINITY) return false; } - return fastAggregateVerify( - fromHexString(message), - pubkeys.map((hex) => PublicKey.fromHex(hex)), - Signature.fromHex(signature) - ); + try { + return fastAggregateVerify( + fromHexString(message), + pubkeys.map((hex) => PublicKey.fromHex(hex)), + Signature.fromHex(signature) + ); + } catch (e) { + return false; + } } /** @@ -167,7 +183,11 @@ function fast_aggregate_verify(input: {pubkeys: string[]; message: string; signa */ function sign(input: {privkey: string; message: string}): string | null { const {privkey, message} = input; - return SecretKey.fromHex(privkey).sign(fromHexString(message)).toHex(); + try { + return SecretKey.fromHex(privkey).sign(fromHexString(message)).toHex(); + } catch (e) { + return null; + } } /** @@ -179,5 +199,9 @@ function sign(input: {privkey: string; message: string}): string | null { */ function verify(input: {pubkey: string; message: string; signature: string}): boolean { const {pubkey, message, signature} = input; - return _verify(fromHexString(message), PublicKey.fromHex(pubkey), Signature.fromHex(signature)); + try { + return _verify(fromHexString(message), PublicKey.fromHex(pubkey), Signature.fromHex(signature)); + } catch (e) { + return false; + } } From be7efd382774f8cf4817496fb3d9ba76015a2416 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 23 Jul 2024 14:59:45 -0400 Subject: [PATCH 15/18] chore: remove stray import --- packages/beacon-node/test/spec/presets/light_client/sync.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/beacon-node/test/spec/presets/light_client/sync.ts b/packages/beacon-node/test/spec/presets/light_client/sync.ts index 4b264e5b65e4..bfd3d0d0bb3d 100644 --- a/packages/beacon-node/test/spec/presets/light_client/sync.ts +++ b/packages/beacon-node/test/spec/presets/light_client/sync.ts @@ -1,5 +1,4 @@ import {expect} from "vitest"; -import {init} from "@chainsafe/bls/switchable"; import {isForkLightClient} from "@lodestar/params"; import {altair, phase0, RootHex, Slot, ssz, sszTypesFor} from "@lodestar/types"; import {InputType} from "@lodestar/spec-test-util"; @@ -78,8 +77,6 @@ const UPDATE_FILE_NAME = "^(update)_([0-9a-zA-Z_]+)$"; export const sync: TestRunnerFn = (fork) => { return { testFunction: async (testcase) => { - await init("blst-native"); - // Grab only the ALTAIR_FORK_EPOCH, since the domains are the same as minimal const config = createBeaconConfig( pickConfigForkEpochs(testcase.config), From 35e7ae0cc92f1e5de05941e2c01c155d97cda676 Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 23 Jul 2024 16:14:50 -0400 Subject: [PATCH 16/18] fix: unit test --- .../attestation/validateGossipAttestationsSameAttData.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts b/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts index 3ec95d1dcb55..24fae8a03f0b 100644 --- a/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts +++ b/packages/beacon-node/test/unit/chain/validation/attestation/validateGossipAttestationsSameAttData.test.ts @@ -48,7 +48,7 @@ describe("validateGossipAttestationsSameAttData", () => { const bytes = new Uint8Array(32); const dataView = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); dataView.setUint32(0, i + 1, true); - const secretKey = SecretKey.fromBytes(bytes); + const secretKey = SecretKey.fromKeygen(bytes); const publicKey = secretKey.toPublicKey(); keypair = {secretKey, publicKey}; keypairs.set(i, keypair); From 5d7256a0ff21e1f1747728048230fe4f5755a04b Mon Sep 17 00:00:00 2001 From: Cayman Date: Tue, 23 Jul 2024 16:22:35 -0400 Subject: [PATCH 17/18] fix: bls perf --- packages/beacon-node/test/perf/bls/bls.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/test/perf/bls/bls.test.ts b/packages/beacon-node/test/perf/bls/bls.test.ts index 2adebba38213..988052f4a95c 100644 --- a/packages/beacon-node/test/perf/bls/bls.test.ts +++ b/packages/beacon-node/test/perf/bls/bls.test.ts @@ -27,7 +27,7 @@ describe("BLS ops", function () { const bytes = new Uint8Array(32); const dataView = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); dataView.setUint32(0, i + 1, true); - const secretKey = SecretKey.fromBytes(bytes); + const secretKey = SecretKey.fromKeygen(bytes); const publicKey = secretKey.toPublicKey(); keypair = {secretKey, publicKey}; keypairs.set(i, keypair); From 3754e5839d6a8c08fc654863930dc3e54ad8254d Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Thu, 25 Jul 2024 13:30:12 +0700 Subject: [PATCH 18/18] chore: sync yarn.lock --- yarn.lock | 199 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 125 insertions(+), 74 deletions(-) diff --git a/yarn.lock b/yarn.lock index 4fe607c25bc7..e00d7e97c217 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1591,6 +1591,29 @@ progress-events "^1.0.0" uint8arraylist "^2.4.8" +"@libp2p/interface@^1.2.0", "@libp2p/interface@^1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@libp2p/interface/-/interface-1.6.1.tgz#dc8f1a49da7ee81cd068e872bde3445c3b631d45" + integrity sha512-bpkIYTvZhGGc/ajITKvgFpaP8UtPWoSj+xHVrj6zyAN8U/cAqN0IQQt4a7daJr5VZa8B86i4d1iccdG42/mz+g== + dependencies: + "@multiformats/multiaddr" "^12.2.3" + it-pushable "^3.2.3" + it-stream-types "^2.0.1" + multiformats "^13.1.0" + progress-events "^1.0.0" + uint8arraylist "^2.4.8" + +"@libp2p/logger@^4.0.10": + version "4.0.17" + resolved "https://registry.yarnpkg.com/@libp2p/logger/-/logger-4.0.17.tgz#1ae663bb2dec3c7e6b29d2195756f64e5e77d939" + integrity sha512-NPGN27uOXFGuKkxnX39InMvxS0lMenq6/aFqQHN1N0f0S3LaG9RuTcz/VE3qyO1Ik1aAockR6qqCwbfFxJuO0g== + dependencies: + "@libp2p/interface" "^1.6.1" + "@multiformats/multiaddr" "^12.2.3" + debug "^4.3.4" + interface-datastore "^8.2.11" + multiformats "^13.1.0" + "@libp2p/logger@^4.0.11", "@libp2p/logger@^4.0.6": version "4.0.11" resolved "https://registry.yarnpkg.com/@libp2p/logger/-/logger-4.0.11.tgz#671692a0cceee73a0c0bf9b5f05ea14fde05f5e5" @@ -1881,6 +1904,19 @@ uint8-varint "^2.0.1" uint8arrays "^5.0.0" +"@multiformats/multiaddr@^12.2.3": + version "12.3.0" + resolved "https://registry.yarnpkg.com/@multiformats/multiaddr/-/multiaddr-12.3.0.tgz#b1422813446e5cdec4b0f6cba51f93239f390884" + integrity sha512-JQ8Gc/jgucqqvEaDTFN/AvxlYDHEE7lgEWLMYW7hKZkWggER+GvG/tVxUgUxIP8M0vFpvEHKKHE0lKzyMsgi8Q== + dependencies: + "@chainsafe/is-ip" "^2.0.1" + "@chainsafe/netmask" "^2.0.0" + "@libp2p/interface" "^1.0.0" + "@multiformats/dns" "^1.0.3" + multiformats "^13.0.0" + uint8-varint "^2.0.1" + uint8arrays "^5.0.0" + "@napi-rs/snappy-android-arm-eabi@7.2.2": version "7.2.2" resolved "https://registry.yarnpkg.com/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.2.2.tgz#85fee3ba198dad4b444b5f12bceebcf72db0d65e" @@ -7443,7 +7479,7 @@ http-proxy-agent@^7.0.0: agent-base "^7.1.0" debug "^4.3.4" -http-proxy-agent@^7.0.2: +http-proxy-agent@^7.0.1, http-proxy-agent@^7.0.2: version "7.0.2" resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz#9a8b1f246866c028509486585f62b8f2c18c270e" integrity sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig== @@ -7489,14 +7525,6 @@ https-proxy-agent@^5.0.0: agent-base "6" debug "4" -https-proxy-agent@^7.0.0, https-proxy-agent@^7.0.4: - version "7.0.4" - resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz#8e97b841a029ad8ddc8731f26595bad868cb4168" - integrity sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg== - dependencies: - agent-base "^7.0.2" - debug "4" - https-proxy-agent@^7.0.2: version "7.0.2" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz#e2645b846b90e96c6e6f347fb5b2e41f1590b09b" @@ -7505,6 +7533,22 @@ https-proxy-agent@^7.0.2: agent-base "^7.0.2" debug "4" +https-proxy-agent@^7.0.3, https-proxy-agent@^7.0.5: + version "7.0.5" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz#9e8b5013873299e11fab6fd548405da2d6c602b2" + integrity sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw== + dependencies: + agent-base "^7.0.2" + debug "4" + +https-proxy-agent@^7.0.4: + version "7.0.4" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz#8e97b841a029ad8ddc8731f26595bad868cb4168" + integrity sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg== + dependencies: + agent-base "^7.0.2" + debug "4" + human-signals@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" @@ -7711,16 +7755,19 @@ internal-slot@^1.0.5: has "^1.0.3" side-channel "^1.0.4" +ip-address@^9.0.5: + version "9.0.5" + resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-9.0.5.tgz#117a960819b08780c3bd1f14ef3c1cc1d3f3ea5a" + integrity sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g== + dependencies: + jsbn "1.1.0" + sprintf-js "^1.1.3" + ip-regex@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz" integrity sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk= -ip@^1.1.8: - version "1.1.9" - resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.9.tgz#8dfbcc99a754d07f425310b86a99546b1151e396" - integrity sha512-cyRxvOEpNHNtchU3Ln9KC/auJgup87llfQpQ+t5ghoC/UhL16SWzbueiCsdTnWmqAWl7LadfuwhlqmtOaqMHdQ== - ip@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/ip/-/ip-2.0.1.tgz#e8f3595d33a3ea66490204234b77636965307105" @@ -8337,6 +8384,11 @@ js-yaml@^3.10.0: argparse "^1.0.7" esprima "^4.0.0" +jsbn@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040" + integrity sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A== + jsdom@^23.0.1: version "23.0.1" resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-23.0.1.tgz#ede7ff76e89ca035b11178d200710d8982ebfee0" @@ -10295,27 +10347,26 @@ p-waterfall@2.1.1: dependencies: p-reduce "^2.0.0" -pac-proxy-agent@^7.0.0: - version "7.0.1" - resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-7.0.1.tgz#6b9ddc002ec3ff0ba5fdf4a8a21d363bcc612d75" - integrity sha512-ASV8yU4LLKBAjqIPMbrgtaKIvxQri/yh2OpI+S6hVa9JRkUI3Y3NPFbfngDtY7oFtSMD3w31Xns89mDa3Feo5A== +pac-proxy-agent@^7.0.1: + version "7.0.2" + resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz#0fb02496bd9fb8ae7eb11cfd98386daaac442f58" + integrity sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg== dependencies: "@tootallnate/quickjs-emscripten" "^0.23.0" agent-base "^7.0.2" debug "^4.3.4" get-uri "^6.0.1" http-proxy-agent "^7.0.0" - https-proxy-agent "^7.0.2" - pac-resolver "^7.0.0" - socks-proxy-agent "^8.0.2" + https-proxy-agent "^7.0.5" + pac-resolver "^7.0.1" + socks-proxy-agent "^8.0.4" -pac-resolver@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-7.0.0.tgz#79376f1ca26baf245b96b34c339d79bff25e900c" - integrity sha512-Fd9lT9vJbHYRACT8OhCbZBbxr6KRSawSovFpy8nDGshaK99S/EBhVIHp9+crhxrsZOuvLpgL1n23iyPg6Rl2hg== +pac-resolver@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-7.0.1.tgz#54675558ea368b64d210fd9c92a640b5f3b8abb6" + integrity sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg== dependencies: degenerator "^5.0.0" - ip "^1.1.8" netmask "^2.0.2" pacote@^15.2.0: @@ -10719,19 +10770,19 @@ proxy-addr@^2.0.7: forwarded "0.2.0" ipaddr.js "1.9.1" -proxy-agent@6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.3.0.tgz#72f7bb20eb06049db79f7f86c49342c34f9ba08d" - integrity sha512-0LdR757eTj/JfuU7TL2YCuAZnxWXu3tkJbg4Oq3geW/qFNT/32T0sp2HnZ9O0lMR4q3vwAt0+xCA8SR0WAD0og== +proxy-agent@6.4.0: + version "6.4.0" + resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.4.0.tgz#b4e2dd51dee2b377748aef8d45604c2d7608652d" + integrity sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ== dependencies: agent-base "^7.0.2" debug "^4.3.4" - http-proxy-agent "^7.0.0" - https-proxy-agent "^7.0.0" + http-proxy-agent "^7.0.1" + https-proxy-agent "^7.0.3" lru-cache "^7.14.1" - pac-proxy-agent "^7.0.0" + pac-proxy-agent "^7.0.1" proxy-from-env "^1.1.0" - socks-proxy-agent "^8.0.1" + socks-proxy-agent "^8.0.2" proxy-from-env@^1.1.0: version "1.1.0" @@ -11393,6 +11444,13 @@ semver@7.5.3: dependencies: lru-cache "^6.0.0" +semver@7.6.0, semver@^7.6.0: + version "7.6.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" + integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== + dependencies: + lru-cache "^6.0.0" + semver@^6.1.0, semver@^6.2.0, semver@^6.3.1: version "6.3.1" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" @@ -11405,13 +11463,6 @@ semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semve dependencies: lru-cache "^6.0.0" -semver@^7.6.0: - version "7.6.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" - integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== - dependencies: - lru-cache "^6.0.0" - serialize-error@^11.0.1: version "11.0.3" resolved "https://registry.yarnpkg.com/serialize-error/-/serialize-error-11.0.3.tgz#b54f439e15da5b4961340fbbd376b6b04aa52e92" @@ -11620,15 +11671,6 @@ socks-proxy-agent@^7.0.0: debug "^4.3.3" socks "^2.6.2" -socks-proxy-agent@^8.0.1: - version "8.0.3" - resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.3.tgz#6b2da3d77364fde6292e810b496cb70440b9b89d" - integrity sha512-VNegTZKhuGq5vSD6XNKlbqWhyt/40CgoEw8XxD6dhnm8Jq9IEa3nIa4HwnM8XOqU0CdB0BwWVXusqiFXfHB3+A== - dependencies: - agent-base "^7.1.1" - debug "^4.3.4" - socks "^2.7.1" - socks-proxy-agent@^8.0.2: version "8.0.2" resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz#5acbd7be7baf18c46a3f293a840109a430a640ad" @@ -11638,6 +11680,15 @@ socks-proxy-agent@^8.0.2: debug "^4.3.4" socks "^2.7.1" +socks-proxy-agent@^8.0.4: + version "8.0.4" + resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz#9071dca17af95f483300316f4b063578fa0db08c" + integrity sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw== + dependencies: + agent-base "^7.1.1" + debug "^4.3.4" + socks "^2.8.3" + socks@^2.6.2, socks@^2.7.1: version "2.7.1" resolved "https://registry.yarnpkg.com/socks/-/socks-2.7.1.tgz#d8e651247178fde79c0663043e07240196857d55" @@ -11646,6 +11697,14 @@ socks@^2.6.2, socks@^2.7.1: ip "^2.0.0" smart-buffer "^4.2.0" +socks@^2.8.3: + version "2.8.3" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.3.tgz#1ebd0f09c52ba95a09750afe3f3f9f724a800cb5" + integrity sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw== + dependencies: + ip-address "^9.0.5" + smart-buffer "^4.2.0" + sonic-boom@^3.7.0: version "3.8.0" resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-3.8.0.tgz#e442c5c23165df897d77c3c14ef3ca40dec66a66" @@ -11743,6 +11802,11 @@ sprintf-js@^1.1.2: resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz" integrity sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug== +sprintf-js@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.3.tgz#4914b903a2f8b685d17fdf78a70e917e872e444a" + integrity sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA== + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz" @@ -12125,16 +12189,7 @@ tapable@^2.2.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== -tar-fs@3.0.4, tar-fs@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.4.tgz#a21dc60a2d5d9f55e0089ccd78124f1d3771dbbf" - integrity sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w== - dependencies: - mkdirp-classic "^0.5.2" - pump "^3.0.0" - tar-stream "^3.1.5" - -tar-fs@^3.0.5: +tar-fs@3.0.5, tar-fs@^3.0.5: version "3.0.5" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.5.tgz#f954d77767e4e6edf973384e1eb95f8f81d64ed9" integrity sha512-JOgGAmZyMgbqpLwct7ZV8VzkEB6pxXFBVErLtb+XCOqzc6w1xiWKI9GVd6bwk68EX7eJ4DWmfXVmq8K2ziZTGg== @@ -12145,6 +12200,15 @@ tar-fs@^3.0.5: bare-fs "^2.1.1" bare-path "^2.1.0" +tar-fs@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.4.tgz#a21dc60a2d5d9f55e0089ccd78124f1d3771dbbf" + integrity sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w== + dependencies: + mkdirp-classic "^0.5.2" + pump "^3.0.0" + tar-stream "^3.1.5" + tar-fs@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.1.tgz#e44086c1c60d31a4f0cf893b1c4e155dabfae9e2" @@ -13725,20 +13789,7 @@ yargs@16.2.0, yargs@^16.2.0: y18n "^5.0.5" yargs-parser "^20.2.2" -yargs@17.7.1: - version "17.7.1" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.1.tgz#34a77645201d1a8fc5213ace787c220eabbd0967" - integrity sha512-cwiTb08Xuv5fqF4AovYacTFNxk62th7LKJ6BL9IGUpTJrWoU7/7WdQGTP2SjKf1dUNBGzDd28p/Yfs/GI6JrLw== - dependencies: - cliui "^8.0.1" - escalade "^3.1.1" - get-caller-file "^2.0.5" - require-directory "^2.1.1" - string-width "^4.2.3" - y18n "^5.0.5" - yargs-parser "^21.1.1" - -yargs@^17.1.1, yargs@^17.5.1, yargs@^17.6.2, yargs@^17.7.1: +yargs@17.7.2, yargs@^17.1.1, yargs@^17.5.1, yargs@^17.6.2, yargs@^17.7.1: version "17.7.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==