Merge branch 'community-edition' into fix/bitwise-rotate

.github/workflows/ci.yml

@@ -2,52 +2,64 @@ name: Tests
 
 on:
   push:
-    branches: ["main", "community-edition"]
+    branches: ["main"]
   pull_request:
-    branches: ["main", "community-edition"]
+    branches: ["main", "develop", "community-edition"]
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
   build:
-    runs-on: ubuntu-latest-m
+    runs-on: ubuntu-latest-64core-256ram
 
     steps:
       - uses: actions/checkout@v3
       - name: Build
         run: cargo build --verbose
       - name: Run halo2-base tests
-        working-directory: 'halo2-base'
+        working-directory: "halo2-base"
         run: |
-          cargo test -- --test-threads=1
-      - name: Run poseidon tests
-        working-directory: 'hashes/poseidon'
+          cargo test
+      - name: Run halo2-ecc tests (mock prover)
+        working-directory: "halo2-ecc"
         run: |
-          cargo test test_poseidon_compatibility
-      - name: Run halo2-ecc tests MockProver
-        working-directory: 'halo2-ecc'
+          cargo test --lib -- --skip bench
+      - name: Run halo2-ecc tests (real prover)
+        working-directory: "halo2-ecc"
         run: |
-          cargo test -- --test-threads=1 test_fp
-          cargo test -- test_ecc
-          cargo test -- test_secp
-          cargo test -- test_ecdsa
-          cargo test -- test_ec_add
-          cargo test -- test_fixed
-          cargo test -- test_msm
-          cargo test -- test_sm
-          cargo test -- test_fb
-          cargo test -- test_pairing
-          cargo test -- test_bls_signature
-      - name: Run halo2-ecc tests real prover
-        working-directory: 'halo2-ecc'
-        run: |
-          cargo test --release -- test_fp_assert_eq
-          cargo test --release -- --nocapture bench_secp256k1_ecdsa
-          cargo test --release -- --nocapture bench_ec_add
           mv configs/bn254/bench_fixed_msm.t.config configs/bn254/bench_fixed_msm.config
-          cargo test --release -- --nocapture bench_fixed_base_msm
           mv configs/bn254/bench_msm.t.config configs/bn254/bench_msm.config
-          cargo test --release -- --nocapture bench_msm
           mv configs/bn254/bench_pairing.t.config configs/bn254/bench_pairing.config
+          mv configs/secp256k1/bench_ecdsa.t.config configs/secp256k1/bench_ecdsa.config
+          cargo test --release -- --nocapture bench_secp256k1_ecdsa
+          cargo test --release -- --nocapture bench_fixed_base_msm
+          cargo test --release -- --nocapture bench_msm
           cargo test --release -- --nocapture bench_pairing
+      - name: Run zkevm tests
+        working-directory: "hashes/zkevm"
+        run: |
+          cargo test packed_multi_keccak_prover::k_14
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          override: false
+          components: rustfmt, clippy
+
+      - uses: Swatinem/rust-cache@v1
+        with:
+          cache-on-failure: true
+
+      - name: Run fmt
+        run: cargo fmt --all -- --check
+
+      - name: Run clippy
+        run: cargo clippy --all -- -D warnings

.gitignore

@@ -8,10 +8,18 @@ Cargo.lock
 
 # These are backup files generated by rustfmt
 **/*.rs.bk
+
+# Local IDE configs
+.idea/
+.vscode/
 =======
 /target
 
 *.png
 
 /halo2_ecc/src/bn254/data/
 /halo2_ecc/src/secp256k1/data/
+
+/halo2_ecc/params/
+/halo2_ecc/results/
+/halo2_base/params/

Cargo.toml

@@ -1,14 +1,10 @@
 [workspace]
-members = [
-    "halo2-base",
-    "halo2-ecc",
-    "hashes/zkevm-keccak",
-    "hashes/poseidon"
-]
+members = ["halo2-base", "halo2-ecc", "hashes/zkevm"]
+resolver = "2"
 
 [profile.dev]
 opt-level = 3
-debug = 1 # change to 0 or 2 for more or less debug info
+debug = 2              # change to 0 or 2 for more or less debug info
 overflow-checks = true
 incremental = true
 
@@ -29,7 +25,7 @@ codegen-units = 16
 opt-level = 3
 debug = false
 debug-assertions = false
-lto = "fat" 
+lto = "fat"
 # `codegen-units = 1` can lead to WORSE performance - always bench to find best profile for your machine!
 # codegen-units = 1
 panic = "unwind"
@@ -40,7 +36,6 @@ incremental = false
 inherits = "release"
 debug = true
 
-# patch so snark-verifier uses this crate's halo2-base 
 [patch."https://github.com/axiom-crypto/halo2-lib.git"]
-halo2-base = { path = "./halo2-base" }
-halo2-ecc = { path = "./halo2-ecc" }
+halo2-base = { path = "../halo2-lib/halo2-base" }
+halo2-ecc = { path = "../halo2-lib/halo2-ecc" }

halo2-base/Cargo.toml

@@ -1,62 +1,74 @@
 [package]
-name = "halo2-base"
-version = "0.3.0"
-edition = "2021"
+name="halo2-base"
+version="0.4.0"
+edition="2021"
 
 [dependencies]
-itertools = "0.10"
-num-bigint = { version = "0.4", features = ["rand"] }
-num-integer = "0.1"
-num-traits = "0.2"
-rand_chacha = "0.3"
-rustc-hash = "1.1"
-ff = "0.12"
-rayon = "1.6.1"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-log = "0.4"
+itertools="0.11"
+num-bigint={ version="0.4", features=["rand"] }
+num-integer="0.1"
+num-traits="0.2"
+rand_chacha="0.3"
+rustc-hash="1.1"
+rayon="1.7"
+serde={ version="1.0", features=["derive"] }
+serde_json="1.0"
+log="0.4"
+getset="0.1.2"
+ark-std={ version="0.3.0", features=["print-trace"], optional=true }
 
 # Use Axiom's custom halo2 monorepo for faster proving when feature = "halo2-axiom" is on
-halo2_proofs_axiom = { git = "https://github.com/axiom-crypto/halo2.git", branch = "axiom/dev", package = "halo2_proofs", optional = true }
+halo2_proofs_axiom={ git="https://github.com/axiom-crypto/halo2.git", package="halo2_proofs", optional=true }
 # Use PSE halo2 and halo2curves for compatibility when feature = "halo2-pse" is on
-halo2_proofs = { git = "https://github.com/privacy-scaling-explorations/halo2.git", tag = "v2023_02_02", optional = true }
+halo2_proofs={ git="https://github.com/privacy-scaling-explorations/halo2.git", rev="7a21656", optional=true }
 
+# This is Scroll's audited poseidon circuit. We only use it for the Native Poseidon spec. We do not use the halo2 circuit at all (and it wouldn't even work because the halo2_proofs tag is not compatbile).
+# We forked it to upgrade to ff v0.13 and removed the circuit module
+poseidon-rs={ git="https://github.com/axiom-crypto/poseidon-circuit.git", rev="1aee4a1" }
 # plotting circuit layout
-plotters = { version = "0.3.0", optional = true }
-tabbycat = { version = "0.1", features = ["attributes"], optional = true }
+plotters={ version="0.3.0", optional=true }
+tabbycat={ version="0.1", features=["attributes"], optional=true }
 
 # test-utils
-rand = { version = "0.8", optional = true }
+rand={ version="0.8", optional=true }
 
 [dev-dependencies]
-ark-std = { version = "0.3.0", features = ["print-trace"] }
-rand = "0.8"
-pprof = { version = "0.11", features = ["criterion", "flamegraph"] }
-criterion = "0.4"
-criterion-macro = "0.4"
-rayon = "1.6.1"
-test-case = "3.1.0"
-proptest = "1.1.0"
+ark-std={ version="0.3.0", features=["print-trace"] }
+rand="0.8"
+pprof={ version="0.11", features=["criterion", "flamegraph"] }
+criterion="0.4"
+criterion-macro="0.4"
+test-case="3.1.0"
+test-log="0.2.12"
+env_logger="0.10.0"
+proptest="1.1.0"
+# native poseidon for testing
+pse-poseidon={ git="https://github.com/axiom-crypto/pse-poseidon.git" }
 
 # memory allocation
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
-jemallocator = { version = "0.5", optional = true }
+jemallocator={ version="=0.5", optional=true }
 
-mimalloc = { version = "0.1", default-features = false, optional = true }
+mimalloc={ version="=0.1", default-features=false, optional=true }
 
 [features]
-default = ["halo2-axiom", "display"]
-dev-graph = ["halo2_proofs?/dev-graph", "halo2_proofs_axiom?/dev-graph", "plotters"]
-halo2-pse = ["halo2_proofs"]
-halo2-axiom = ["halo2_proofs_axiom"]
-display = []
-profile = ["halo2_proofs_axiom?/profile"]
-test-utils = ["dep:rand"]
+default=["halo2-axiom", "display", "test-utils"]
+asm=["halo2_proofs_axiom?/asm"]
+dev-graph=["halo2_proofs?/dev-graph", "halo2_proofs_axiom?/dev-graph", "plotters"]
+halo2-pse=["halo2_proofs/circuit-params"]
+halo2-axiom=["halo2_proofs_axiom"]
+display=[]
+profile=["halo2_proofs_axiom?/profile"]
+test-utils=["dep:rand", "ark-std"]
 
 [[bench]]
-name = "mul"
-harness = false
+name="mul"
+harness=false
 
 [[bench]]
-name = "inner_product"
-harness = false
+name="inner_product"
+harness=false
+
+[[example]]
+name="inner_product"
+required-features=["test-utils"]

halo2-base/benches/inner_product.rs

@@ -1,28 +1,17 @@
-#![allow(unused_imports)]
-#![allow(unused_variables)]
-use halo2_base::gates::builder::{GateCircuitBuilder, GateThreadBuilder};
-use halo2_base::gates::flex_gate::{FlexGateConfig, GateChip, GateInstructions, GateStrategy};
+use halo2_base::gates::circuit::{builder::RangeCircuitBuilder, CircuitBuilderStage};
+use halo2_base::gates::flex_gate::{GateChip, GateInstructions};
 use halo2_base::halo2_proofs::{
     arithmetic::Field,
-    circuit::*,
     dev::MockProver,
-    halo2curves::bn256::{Bn256, Fr, G1Affine},
+    halo2curves::bn256::{Bn256, Fr},
     plonk::*,
-    poly::kzg::{
-        commitment::{KZGCommitmentScheme, ParamsKZG},
-        multiopen::ProverSHPLONK,
-    },
-    transcript::{Blake2bWrite, Challenge255, TranscriptWriterBuffer},
+    poly::kzg::commitment::ParamsKZG,
 };
+use halo2_base::utils::testing::gen_proof;
 use halo2_base::utils::ScalarField;
-use halo2_base::{
-    Context,
-    QuantumCell::{Existing, Witness},
-    SKIP_FIRST_PASS,
-};
+use halo2_base::{Context, QuantumCell::Existing};
 use itertools::Itertools;
 use rand::rngs::OsRng;
-use std::marker::PhantomData;
 
 use criterion::{criterion_group, criterion_main};
 use criterion::{BenchmarkId, Criterion};
@@ -47,20 +36,20 @@ fn inner_prod_bench<F: ScalarField>(ctx: &mut Context<F>, a: Vec<F>, b: Vec<F>)
 fn bench(c: &mut Criterion) {
     let k = 19u32;
     // create circuit for keygen
-    let mut builder = GateThreadBuilder::new(false);
+    let mut builder =
+        RangeCircuitBuilder::from_stage(CircuitBuilderStage::Keygen).use_k(k as usize);
     inner_prod_bench(builder.main(0), vec![Fr::zero(); 5], vec![Fr::zero(); 5]);
-    builder.config(k as usize, Some(20));
-    let circuit = GateCircuitBuilder::mock(builder);
+    let config_params = builder.calculate_params(Some(20));
 
     // check the circuit is correct just in case
-    MockProver::run(k, &circuit, vec![]).unwrap().assert_satisfied();
+    MockProver::run(k, &builder, vec![]).unwrap().assert_satisfied();
 
     let params = ParamsKZG::<Bn256>::setup(k, OsRng);
-    let vk = keygen_vk(&params, &circuit).expect("vk should not fail");
-    let pk = keygen_pk(&params, vk, &circuit).expect("pk should not fail");
+    let vk = keygen_vk(&params, &builder).expect("vk should not fail");
+    let pk = keygen_pk(&params, vk, &builder).expect("pk should not fail");
 
-    let break_points = circuit.break_points.take();
-    drop(circuit);
+    let break_points = builder.break_points();
+    drop(builder);
 
     let mut group = c.benchmark_group("plonk-prover");
     group.sample_size(10);
@@ -69,22 +58,12 @@ fn bench(c: &mut Criterion) {
         &(&params, &pk),
         |bencher, &(params, pk)| {
             bencher.iter(|| {
-                let mut builder = GateThreadBuilder::new(true);
+                let mut builder =
+                    RangeCircuitBuilder::prover(config_params.clone(), break_points.clone());
                 let a = (0..5).map(|_| Fr::random(OsRng)).collect_vec();
                 let b = (0..5).map(|_| Fr::random(OsRng)).collect_vec();
                 inner_prod_bench(builder.main(0), a, b);
-                let circuit = GateCircuitBuilder::prover(builder, break_points.clone());
-
-                let mut transcript = Blake2bWrite::<_, _, Challenge255<_>>::init(vec![]);
-                create_proof::<
-                    KZGCommitmentScheme<Bn256>,
-                    ProverSHPLONK<'_, Bn256>,
-                    Challenge255<G1Affine>,
-                    _,
-                    Blake2bWrite<Vec<u8>, G1Affine, Challenge255<_>>,
-                    _,
-                >(params, pk, &[circuit], &[&[]], OsRng, &mut transcript)
-                .expect("prover should not fail");
+                gen_proof(params, pk, builder);
             })
         },
     );