feat: UH recursion in the browser (#11049)

Sets up yarn-project/noir-bb-bench for assessing the browser performance
of UltraHonk. (Nb 920 lines in lockfile change)

barretenberg/cpp/src/barretenberg/bb/main.cpp

@@ -135,7 +135,9 @@ bool proveAndVerifyHonkAcirFormat(acir_format::AcirProgram program, acir_format:
 
     Verifier verifier{ verification_key };
 
-    return verifier.verify_proof(proof);
+    const bool verified = verifier.verify_proof(proof);
+    vinfo(verified ? "\033[32mVERIFIED\033[0m" : "\033[31mNOT VERIFIED\033[0m");
+    return verified;
 }
 
 /**

barretenberg/cpp/src/barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp

@@ -693,9 +693,7 @@ class UltraCircuitBuilder_ : public CircuitBuilderBase<typename ExecutionTrace_:
     {
         ASSERT(circuit_finalized);
         auto minimum_circuit_size = get_tables_size() + get_lookups_size();
-        info("minimum_circuit_size: ", minimum_circuit_size);
         auto num_filled_gates = get_num_finalized_gates() + this->public_inputs.size();
-        info("num_filled_gates: ", num_filled_gates);
         return std::max(minimum_circuit_size, num_filled_gates) + NUM_RESERVED_GATES;
     }
 

barretenberg/cpp/src/barretenberg/ultra_honk/decider_prover.cpp

@@ -103,12 +103,10 @@ template <IsUltraFlavor Flavor> HonkProof DeciderProver_<Flavor>::construct_proo
     PROFILE_THIS_NAME("Decider::construct_proof");
 
     // Run sumcheck subprotocol.
-    vinfo("executing relation checking rounds...");
     execute_relation_check_rounds();
 
     // Fiat-Shamir: rho, y, x, z
     // Execute Shplemini PCS
-    vinfo("executing pcs opening rounds...");
     execute_pcs_rounds();
 
     return export_proof();

barretenberg/cpp/src/barretenberg/ultra_honk/decider_proving_key.hpp

@@ -78,8 +78,9 @@ template <IsUltraFlavor Flavor> class DeciderProvingKey_ {
 
         info("Finalized circuit size: ",
              circuit.num_gates,
-             "\nLog dyadic circuit size: ",
-             numeric::get_msb(dyadic_circuit_size));
+             ". Log dyadic circuit size: ",
+             numeric::get_msb(dyadic_circuit_size),
+             ".");
 
         // Complete the public inputs execution trace block from circuit.public_inputs
         Trace::populate_public_inputs_block(circuit);

barretenberg/cpp/src/barretenberg/ultra_honk/ultra_prover.cpp

@@ -64,14 +64,12 @@ template <IsUltraFlavor Flavor> void UltraProver_<Flavor>::generate_gate_challen
 template <IsUltraFlavor Flavor> HonkProof UltraProver_<Flavor>::construct_proof()
 {
     OinkProver<Flavor> oink_prover(proving_key, transcript);
-    vinfo("created oink prover");
     oink_prover.prove();
     vinfo("created oink proof");
 
     generate_gate_challenges();
 
     DeciderProver_<Flavor> decider_prover(proving_key, transcript);
-    vinfo("created decider prover");
     decider_prover.construct_proof();
     return export_proof();
 }

barretenberg/ts/src/barretenberg/backend.ts

@@ -5,6 +5,7 @@ import {
   deflattenFields,
   flattenFieldsAsArray,
   ProofData,
+  ProofDataForRecursion,
   reconstructHonkProof,
   reconstructUltraPlonkProof,
 } from '../proof/index.js';
@@ -216,6 +217,7 @@ export class UltraHonkBackend {
     const proofStart = proofWithPublicInputs.slice(0, publicInputsOffset);
     const publicInputsSplitIndex = numPublicInputs * fieldByteSize;
     const proofEnd = proofWithPublicInputs.slice(publicInputsOffset + publicInputsSplitIndex);
+
     // Construct the proof without the public inputs
     const proof = new Uint8Array([...proofStart, ...proofEnd]);
 
@@ -229,6 +231,61 @@ export class UltraHonkBackend {
     return { proof, publicInputs };
   }
 
+  async generateProofForRecursiveAggregation(
+    compressedWitness: Uint8Array,
+    options?: UltraHonkBackendOptions,
+  ): Promise<ProofDataForRecursion> {
+    await this.instantiate();
+
+    const proveUltraHonk = options?.keccak
+      ? this.api.acirProveUltraKeccakHonk.bind(this.api)
+      : this.api.acirProveUltraHonk.bind(this.api);
+
+    const proofWithPublicInputs = await proveUltraHonk(
+      this.acirUncompressedBytecode,
+      this.circuitOptions.recursive,
+      gunzip(compressedWitness),
+    );
+
+    // proofWithPublicInputs starts with a four-byte size
+    const numSerdeHeaderBytes = 4;
+    // some public inputs are handled specially
+    const numKZGAccumulatorFieldElements = 16;
+    // proof begins with: size, num public inputs, public input offset
+    const numProofPreambleElements = 3;
+    const publicInputsSizeIndex = 1;
+
+    // Slice serde header and convert to fields
+    const proofAsStrings = deflattenFields(proofWithPublicInputs.slice(numSerdeHeaderBytes));
+    const numPublicInputs = Number(proofAsStrings[publicInputsSizeIndex]) - numKZGAccumulatorFieldElements;
+
+    // Account for the serialized buffer size at start
+    const publicInputsOffset = publicInputsOffsetBytes + serializedBufferSize;
+    const publicInputsSplitIndex = numPublicInputs * fieldByteSize;
+
+    // Construct the proof without the public inputs
+    const numPublicInputsBytes = numPublicInputs * fieldByteSize;
+    const numHeaderPlusPreambleBytes = numSerdeHeaderBytes + numProofPreambleElements * fieldByteSize;
+    const proofNoPIs = new Uint8Array(proofWithPublicInputs.length - numPublicInputsBytes);
+    // copy the elements before the public inputs
+    proofNoPIs.set(proofWithPublicInputs.subarray(0, numHeaderPlusPreambleBytes), 0);
+    // copy the elements after the public inputs
+    proofNoPIs.set(
+      proofWithPublicInputs.subarray(numHeaderPlusPreambleBytes + numPublicInputsBytes),
+      numHeaderPlusPreambleBytes,
+    );
+    const proof: string[] = deflattenFields(proofNoPIs.slice(numSerdeHeaderBytes));
+
+    // Fetch the number of public inputs out of the proof string
+    const publicInputsConcatenated = proofWithPublicInputs.slice(
+      publicInputsOffset,
+      publicInputsOffset + publicInputsSplitIndex,
+    );
+    const publicInputs = deflattenFields(publicInputsConcatenated);
+
+    return { proof, publicInputs };
+  }
+
   async verifyProof(proofData: ProofData, options?: UltraHonkBackendOptions): Promise<boolean> {
     await this.instantiate();
 

barretenberg/ts/src/barretenberg_wasm/barretenberg_wasm_main/index.ts

@@ -63,7 +63,6 @@ export class BarretenbergWasmMain extends BarretenbergWasmBase {
       this.remoteWasms = await Promise.all(this.workers.map(getRemoteBarretenbergWasm<BarretenbergWasmThreadWorker>));
       await Promise.all(this.remoteWasms.map(w => w.initThread(module, this.memory)));
     }
-    this.logger('init complete.');
   }
 
   /**

barretenberg/ts/src/proof/index.ts

@@ -9,6 +9,17 @@ export type ProofData = {
   proof: Uint8Array;
 };
 
+/**
+ * @description
+ * The representation of a proof
+ * */
+export type ProofDataForRecursion = {
+  /** @description Public inputs of a proof */
+  publicInputs: string[];
+  /** @description An byte array representing the proof */
+  proof: string[];
+};
+
 // Buffers are prepended with their size. The size takes 4 bytes.
 const serializedBufferSize = 4;
 const fieldByteSize = 32;

yarn-project/noir-bb-bench/.eslintrc.cjs

@@ -0,0 +1 @@
+module.exports = require('@aztec/foundation/eslint');

yarn-project/noir-bb-bench/.gitignore

@@ -0,0 +1,3 @@
+artifacts/
+circuits/**/proofs/*
+circuits/**/Prover.toml

yarn-project/noir-bb-bench/.prettierignore

@@ -0,0 +1,2 @@
+crates
+artifacts

yarn-project/noir-bb-bench/README.md

@@ -0,0 +1,19 @@
+# Noir + Bb benchmarking suite
+
+The goal of this module is to provide a simple place for people to construct benchmarks of witness generation and proving. At the moment it only pertains to UltraHonk recursion in the browser, but we have a similar module in ivc-integration that shows prover performance of our ClientIVC suite.
+
+## Building
+
+The package assumes that bb.js has been built, but it is easy to rebuild, as we show below.
+
+The full build command `yarn build` deletes all circuit artifacts and generated code, compiles the circuits, computes their verification keys, generates declarations and types for parsing circuit bytecode and verification keys in typescript, generates additional type information for noir.js and bb.js, and builds the typescript. With all of this, `yarn test` will run whatever jest tests are present, and `yarn serve:app` will serve a simple app with proving for execution in a web browser. but we can build more incrementally as well.
+
+Scenario: I have made changes to bb.js and now I want to rebuild and run the browser app with multithreaded proving and symbols for the meaningful WASM stack traces. Command:
+```
+cd ../../barretenberg/ts && SKIP_ST_BUILD=1 NO_STRIP=1 yarn build && cd - && yarn build:app && yarn serve:app
+```
+
+Scenario: bb.js is unchanged, but I have changed one of my test circuits, and I want to run all tests. Command:
+```
+yarn generate && yarn build:ts && yarn test
+```

yarn-project/noir-bb-bench/circuits/circuit_1/Nargo.toml

@@ -0,0 +1,5 @@
+[package]
+name = "circuit_1"
+type = "bin"
+
+[dependencies]

yarn-project/noir-bb-bench/circuits/circuit_1/src/main.nr

@@ -0,0 +1,3 @@
+fn main(x: Field, y: pub Field) {
+    assert(x != y);
+}

yarn-project/noir-bb-bench/circuits/circuit_2/Nargo.toml

@@ -0,0 +1,5 @@
+[package]
+name = "circuit_2"
+type = "bin"
+
+[dependencies]

yarn-project/noir-bb-bench/circuits/circuit_2/src/main.nr

@@ -0,0 +1,29 @@
+use std::hash::poseidon;
+
+// This circuit aggregates a single Honk proof from `assert_statement`.
+global ULTRA_VK_SIZE: u32 = 128;
+global ULTRA_PROOF_SIZE: u32 = 459;
+global NUM_NON_ACCUMULATOR_PUBLIC_INPUTS: u32 = 1;
+global HONK_IDENTIFIER: u32 = 1;
+fn main(
+    verification_key: [Field; ULTRA_VK_SIZE],
+    proof: [Field; ULTRA_PROOF_SIZE],
+    public_inputs: pub [Field; NUM_NON_ACCUMULATOR_PUBLIC_INPUTS],
+    key_hash: Field,
+    mut z: Field
+) {
+    std::verify_proof_with_type(
+        verification_key,
+        proof,
+        public_inputs,
+        key_hash,
+        HONK_IDENTIFIER,
+    );
+
+    for _ in 0..250 {
+        z += poseidon::bn254::hash_1([z]);
+    }
+
+    // Make sure the hash value is used so it's not optimized away.
+    assert(z != 0);
+}

yarn-project/noir-bb-bench/generate_artifacts.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -eu
+source $(git rev-parse --show-toplevel)/ci3/source_bootstrap
+
+export BB=${BB:-../../barretenberg/cpp/build/bin/bb}
+export NARGO=${NARGO:-$(realpath ../../noir/noir-repo/target/release/nargo)}
+
+key_dir=artifacts/keys
+
+function compile {
+  set -euo pipefail
+  local dir=$1
+  local name=${dir//-/_}
+  local circuit_path="./circuits/$name"
+
+  echo_stderr "Generating bytecode for circuit: $name..."
+  cd $circuit_path
+  $NARGO compile
+  cd -
+  local filename="$name.json"
+  mv $circuit_path/target/$filename artifacts/
+
+  local json_path="./artifacts/$filename"
+  local write_vk_cmd="write_vk_ultra_honk -h 1"
+  local vk_as_fields_cmd="vk_as_fields_ultra_honk"
+  local key_path="$key_dir/$name.vk.data.json"
+  echo_stderr "Generating vk for circuit: $name..."
+  SECONDS=0
+  local vk_cmd="jq -r '.bytecode' $json_path | base64 -d | gunzip | $BB $write_vk_cmd -b - -o - --recursive | xxd -p -c 0"
+  vk=$(dump_fail "$vk_cmd")
+  local vkf_cmd="echo '$vk' | xxd -r -p | $BB $vk_as_fields_cmd -k - -o -"
+  vk_fields=$(dump_fail "$vkf_cmd")
+  jq -n --arg vk "$vk" --argjson vkf "$vk_fields" '{keyAsBytes: $vk, keyAsFields: $vkf}' > $key_path
+  echo "Key output at: $key_path (${SECONDS}s)"
+}
+
+compile $1

yarn-project/noir-bb-bench/package.json

@@ -0,0 +1,96 @@
+{
+  "name": "@aztec/bb-bench",
+  "version": "0.1.0",
+  "type": "module",
+  "exports": {
+    ".": "./dest/index.js",
+    "./types": "./dest/types/index.js"
+  },
+  "inherits": [
+    "../package.common.json",
+    "./package.local.json"
+  ],
+  "scripts": {
+    "build": "yarn clean && yarn generate && yarn build:app",
+    "clean": "rm -rf ./dest .tsbuildinfo src/types artifacts",
+    "generate": "yarn generate:artifacts && yarn generate:code",
+    "generate:artifacts": "mkdir -p artifacts/keys && ls circuits | xargs -n 1 ./generate_artifacts.sh",
+    "generate:code": "node --no-warnings --loader ts-node/esm src/scripts/generate_declaration_files.ts && node --no-warnings --loader ts-node/esm src/scripts/generate_ts_from_abi.ts && run -T prettier -w ./src/types",
+    "build:ts": "tsc -b",
+    "build:app": "tsc -b && rm -rf dest && webpack && cp ../../barretenberg/favicon.ico dest",
+    "build:dev": "tsc -b --watch",
+    "serve:app": "./serve.sh",
+    "formatting": "run -T prettier --check ./src && run -T eslint ./src",
+    "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
+    "formatting:fix:types": "NODE_OPTIONS='--max-old-space-size=8096' run -T eslint --fix ./src/types && run -T prettier -w ./src/types",
+    "test": "HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-4} NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
+  },
+  "dependencies": {
+    "@aztec/bb.js": "../../ts",
+    "@aztec/foundation": "workspace:^",
+    "@noir-lang/noir_codegen": "portal:../../noir/packages/noir_codegen",
+    "@noir-lang/noir_js": "file:../../noir/packages/noir_js"
+  },
+  "devDependencies": {
+    "@aztec/bb-prover": "workspace:^",
+    "@jest/globals": "^29.5.0",
+    "@types/jest": "^29.5.0",
+    "@types/node": "^22.8.1",
+    "copy-webpack-plugin": "^12.0.2",
+    "debug": "^4.3.4",
+    "favicon-emoji": "2.3.1",
+    "html-webpack-plugin": "^5.6.0",
+    "jest": "^29.5.0",
+    "resolve-typescript-plugin": "^2.0.1",
+    "serve": "^14.2.1",
+    "ts-loader": "^9.5.1",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.0.4",
+    "webpack": "^5.90.3",
+    "webpack-cli": "^5.1.4",
+    "webpack-dev-server": "^5.0.3"
+  },
+  "files": [
+    "dest",
+    "src",
+    "!*.test.*",
+    "artifacts"
+  ],
+  "types": "./dest/index.d.ts",
+  "engines": {
+    "node": ">=18"
+  },
+  "jest": {
+    "extensionsToTreatAsEsm": [
+      ".ts"
+    ],
+    "transform": {
+      "^.+\\.tsx?$": [
+        "@swc/jest",
+        {
+          "jsc": {
+            "parser": {
+              "syntax": "typescript",
+              "decorators": true
+            },
+            "transform": {
+              "decoratorVersion": "2022-03"
+            }
+          }
+        }
+      ]
+    },
+    "moduleNameMapper": {
+      "^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
+    },
+    "reporters": [
+      "default"
+    ],
+    "testTimeout": 30000,
+    "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
+    "rootDir": "./src",
+    "setupFiles": [
+      "../../foundation/src/jest/setup.mjs"
+    ]
+  }
+}

yarn-project/noir-bb-bench/package.local.json

@@ -0,0 +1,8 @@
+{
+  "scripts": {
+    "build": "yarn clean && yarn generate && yarn build:app",
+    "clean": "rm -rf ./dest .tsbuildinfo src/types artifacts",
+    "build:app": "tsc -b && rm -rf dest && webpack && cp ../../barretenberg/favicon.ico dest"
+  },
+  "files": ["dest", "src", "artifacts", "!*.test.*"]
+}