Feat: switch to logup scheme for lookup argument (#71)

* Multi-input mv-lookup. (#49) * Add mv_lookup.rs * mv_lookup::prover, mv_lookup::verifier * Replace lookup with mv_lookup * replace halo2 with mv lookup Co-authored-by: ying tong <[email protected]> * cleanups Co-authored-by: ying tong <[email protected]> * ConstraintSystem: setup lookup_tracker Co-authored-by: Andrija <[email protected]> * mv_lookup::hybrid_prover Co-authored-by: Andrija <[email protected]> * WIP * mv_multi_lookup: enable lookup caching Co-authored-by: therealyingtong <[email protected]> * Rename hybrid_lookup -> lookup * Chunk lookups using user-provided minimum degree Co-authored-by: Andrija <[email protected]> * mv_lookup bench Co-authored-by: Andrija <[email protected]> * Introduce counter feature for FFTs and MSMs Co-authored-by: Andrija <[email protected]> * Fix off-by-one errors in chunk_lookup Co-authored-by: Andrija <[email protected]> * bench wip * time evaluate_h * KZG * more efficient batch inversion * extended lookup example * Finalize mv lookup Author: therealyingtong <[email protected]> * Remove main/ * Fix according to the comments * replace scan with parallel grand sum computation * Revert Cargo.lock * mv lookup Argument name * parallel batch invert --------- Co-authored-by: Andrija <[email protected]> Co-authored-by: ying tong <[email protected]> Co-authored-by: therealyingtong <[email protected]> * fmt * fix unit test * fix clippy errors * add todo in mv_lookup's prover * fmt and clippy * fix clippy * add detailed running time of steps in logup's prover * fmt * add more log hooks * more running time logs * use par invert * use sorted-vector to store how many times a table element occurs in input * par the process to get inputs_inv_sum * use par * fix par * add feature to skip inv sums * add new feature flag * fix clippy error --------- Co-authored-by: Sphere L <[email protected]> Co-authored-by: Andrija <[email protected]> Co-authored-by: ying tong <[email protected]> Co-authored-by: therealyingtong <[email protected]>
scroll-tech · Dec 8, 2023 · 84003a2 · 84003a2
1 parent f427aca
commit 84003a2
Show file tree

Hide file tree

Showing 20 changed files with 2,293 additions and 568 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/halo2_proofs/Cargo.toml b/halo2_proofs/Cargo.toml
@@ -39,6 +39,10 @@ harness = false
 name = "dev_lookup"
 harness = false
 
+[[bench]]
+name = "lookups"
+harness = false
+
 [[bench]]
 name = "fft"
 harness = false
@@ -63,11 +67,13 @@ crossbeam = "0.8.0"
 # Developer tooling dependencies
 plotters = { version = "0.3.0", optional = true }
 tabbycat = { version = "0.1", features = ["attributes"], optional = true }
+lazy_static = { version = "1", optional = true }
 log = "0.4.17"
 
 # timer
 ark-std = { version = "0.3.0" }
 
+
 [dev-dependencies]
 assert_matches = "1.5"
 criterion = "0.3"
@@ -80,7 +86,7 @@ rand_core = { version = "0.6", default-features = false, features = ["getrandom"
 getrandom = { version = "0.2", features = ["js"] }
 
 [features]
-default = ["batch", "gwc"]
+default = ["batch", "gwc", "logup_skip_inv"]
 dev-graph = ["plotters", "tabbycat"]
 gadget-traces = ["backtrace"]
 sanity-checks = []
@@ -90,7 +96,9 @@ gwc = []
 parallel_syn = []
 phase-check = []
 profile = ["ark-std/print-trace"]
+counter = ["lazy_static"]
 mock-batch-inv = []
+logup_skip_inv = []
 
 [lib]
 bench = false

diff --git a/halo2_proofs/benches/lookups.rs b/halo2_proofs/benches/lookups.rs
@@ -0,0 +1,239 @@
+#[macro_use]
+extern crate criterion;
+
+use halo2_proofs::circuit::{Layouter, SimpleFloorPlanner, Value};
+use halo2_proofs::plonk::*;
+use halo2_proofs::poly::kzg::multiopen::VerifierGWC;
+use halo2_proofs::poly::{commitment::ParamsProver, Rotation};
+use halo2_proofs::transcript::{Blake2bRead, Blake2bWrite, Challenge255};
+use halo2curves::bn256::{Bn256, G1Affine};
+use halo2curves::pairing::Engine;
+use rand_core::OsRng;
+
+use halo2_proofs::{
+    poly::kzg::{
+        commitment::{KZGCommitmentScheme, ParamsKZG},
+        multiopen::ProverGWC,
+        strategy::SingleStrategy,
+    },
+    transcript::{TranscriptReadBuffer, TranscriptWriterBuffer},
+};
+
+use std::marker::PhantomData;
+
+use criterion::{BenchmarkId, Criterion};
+use ff::PrimeField as Field;
+
+fn criterion_benchmark(c: &mut Criterion) {
+    #[derive(Clone, Default)]
+    struct MyCircuit<F: Field> {
+        _marker: PhantomData<F>,
+    }
+
+    #[derive(Clone)]
+    struct MyConfig {
+        selector: Selector,
+        table: TableColumn,
+        advice: Column<Advice>,
+        other_advice: Column<Advice>,
+    }
+
+    impl<F: Field> Circuit<F> for MyCircuit<F> {
+        type Config = MyConfig;
+        type FloorPlanner = SimpleFloorPlanner;
+
+        fn without_witnesses(&self) -> Self {
+            Self::default()
+        }
+
+        fn configure(meta: &mut ConstraintSystem<F>) -> MyConfig {
+            let config = MyConfig {
+                selector: meta.complex_selector(),
+                table: meta.lookup_table_column(),
+                advice: meta.advice_column(),
+                other_advice: meta.advice_column(),
+            };
+
+            let dummy_selector = meta.complex_selector();
+
+            meta.create_gate("degree 6 gate", |meta| {
+                let dummy_selector = meta.query_selector(dummy_selector);
+                let constraints = vec![dummy_selector.clone(); 4]
+                    .iter()
+                    .fold(dummy_selector.clone(), |acc, val| acc * val.clone());
+                Constraints::with_selector(dummy_selector, Some(constraints))
+            });
+
+            meta.lookup("lookup", |meta| {
+                let advice = meta.query_advice(config.advice, Rotation::cur());
+                vec![(advice, config.table)]
+            });
+
+            meta.lookup("lookup", |meta| {
+                let advice = meta.query_advice(config.advice, Rotation::cur());
+                vec![(advice, config.table)]
+            });
+
+            meta.lookup("lookup", |meta| {
+                let advice = meta.query_advice(config.advice, Rotation::cur());
+                vec![(advice, config.table)]
+            });
+
+            meta.lookup("lookup", |meta| {
+                let advice = meta.query_advice(config.advice, Rotation::cur());
+                vec![(advice, config.table)]
+            });
+
+            meta.lookup("lookup", |meta| {
+                let advice = meta.query_advice(config.advice, Rotation::cur());
+                vec![(advice, config.table)]
+            });
+
+            /*
+                - We need degree at least 6 because 6 - 1 = 5 and we need to go to extended domain of 8n
+                - Our goal is to get to max degree of 9 because now 9 - 1 = 8 and that will fit into domain
+
+                - base degree = table_deg + 2
+                - if we put input_expression_degree = 1
+                => degree = base + 1 = 3 + 1 = 4
+                - we can batch one more with 5 more lookups
+            */
+
+            config
+        }
+
+        fn synthesize(
+            &self,
+            config: MyConfig,
+            mut layouter: impl Layouter<F>,
+        ) -> Result<(), Error> {
+            layouter.assign_table(
+                || "8-bit table",
+                |mut table| {
+                    for row in 0u64..(1 << 8) {
+                        table.assign_cell(
+                            || format!("row {}", row),
+                            config.table,
+                            row as usize,
+                            || Value::known(F::from(row)),
+                        )?;
+                    }
+
+                    Ok(())
+                },
+            )?;
+
+            layouter.assign_region(
+                || "assign values",
+                |mut region| {
+                    for offset in 0u64..(1 << 10) {
+                        config.selector.enable(&mut region, offset as usize)?;
+                        region.assign_advice(
+                            || format!("offset {}", offset),
+                            config.advice,
+                            offset as usize,
+                            || Value::known(F::from(offset % 256)),
+                        )?;
+                    }
+                    for offset in 1u64..(1 << 10) {
+                        config.selector.enable(&mut region, offset as usize)?;
+                        region.assign_advice(
+                            || format!("offset {}", offset),
+                            config.other_advice,
+                            offset as usize - 1,
+                            || Value::known(F::from(offset % 256)),
+                        )?;
+                    }
+                    Ok(())
+                },
+            )
+        }
+    }
+
+    fn keygen(k: u32) -> (ParamsKZG<Bn256>, ProvingKey<G1Affine>) {
+        let params: ParamsKZG<Bn256> = ParamsKZG::new(k);
+        let empty_circuit: MyCircuit<<Bn256 as Engine>::Scalar> = MyCircuit {
+            _marker: PhantomData,
+        };
+        let vk = keygen_vk(&params, &empty_circuit).expect("keygen_vk should not fail");
+        let pk = keygen_pk(&params, vk, &empty_circuit).expect("keygen_pk should not fail");
+        (params, pk)
+    }
+
+    fn prover(_k: u32, params: &ParamsKZG<Bn256>, pk: &ProvingKey<G1Affine>) -> Vec<u8> {
+        let rng = OsRng;
+
+        let circuit: MyCircuit<<Bn256 as Engine>::Scalar> = MyCircuit {
+            _marker: PhantomData,
+        };
+
+        let mut transcript = Blake2bWrite::<_, _, Challenge255<G1Affine>>::init(vec![]);
+        create_proof::<KZGCommitmentScheme<Bn256>, ProverGWC<'_, Bn256>, _, _, _, _>(
+            params,
+            pk,
+            &[circuit],
+            &[&[]],
+            rng,
+            &mut transcript,
+        )
+        .expect("proof generation should not fail");
+        transcript.finalize()
+    }
+
+    fn verifier(params: &ParamsKZG<Bn256>, vk: &VerifyingKey<G1Affine>, proof: &[u8]) {
+        let strategy = SingleStrategy::new(params);
+        let mut transcript = Blake2bRead::<_, _, Challenge255<G1Affine>>::init(proof);
+        assert!(verify_proof::<
+            KZGCommitmentScheme<Bn256>,
+            VerifierGWC<'_, Bn256>,
+            Challenge255<G1Affine>,
+            Blake2bRead<&[u8], G1Affine, Challenge255<G1Affine>>,
+            SingleStrategy<'_, Bn256>,
+        >(params, vk, strategy, &[&[]], &mut transcript)
+        .is_ok());
+    }
+
+    let k_range = 16..=16;
+
+    let mut keygen_group = c.benchmark_group("plonk-keygen");
+    keygen_group.sample_size(10);
+    for k in k_range.clone() {
+        keygen_group.bench_with_input(BenchmarkId::from_parameter(k), &k, |b, &k| {
+            b.iter(|| keygen(k));
+        });
+    }
+    keygen_group.finish();
+
+    let mut prover_group = c.benchmark_group("plonk-prover");
+    prover_group.sample_size(10);
+    for k in k_range.clone() {
+        let (params, pk) = keygen(k);
+
+        prover_group.bench_with_input(
+            BenchmarkId::from_parameter(k),
+            &(k, &params, &pk),
+            |b, &(k, params, pk)| {
+                b.iter(|| prover(k, params, pk));
+            },
+        );
+    }
+    prover_group.finish();
+
+    let mut verifier_group = c.benchmark_group("plonk-verifier");
+    for k in k_range {
+        let (params, pk) = keygen(k);
+        let proof = prover(k, &params, &pk);
+
+        verifier_group.bench_with_input(
+            BenchmarkId::from_parameter(k),
+            &(&params, pk.get_vk(), &proof[..]),
+            |b, &(params, vk, proof)| {
+                b.iter(|| verifier(params, vk, proof));
+            },
+        );
+    }
+    verifier_group.finish();
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/halo2_proofs/src/arithmetic.rs b/halo2_proofs/src/arithmetic.rs
@@ -535,9 +535,18 @@ where
     q
 }
 
+pub fn par_invert<F: Field>(values: &mut [F]) {
+    parallelize(values, |values, _start| {
+        values.batch_invert();
+    });
+}
+
 /// This simple utility function will parallelize an operation that is to be
 /// performed over a mutable slice.
-pub fn parallelize<T: Send, F: Fn(&mut [T], usize) + Send + Sync + Clone>(v: &mut [T], f: F) {
+pub(crate) fn parallelize_internal<T: Send, F: Fn(&mut [T], usize) + Send + Sync + Clone>(
+    v: &mut [T],
+    f: F,
+) -> Vec<usize> {
     let n = v.len();
     let num_threads = multicore::current_num_threads();
     let mut chunk = (n as usize) / num_threads;
@@ -546,14 +555,23 @@ pub fn parallelize<T: Send, F: Fn(&mut [T], usize) + Send + Sync + Clone>(v: &mu
     }
 
     multicore::scope(|scope| {
+        let mut chunk_starts = vec![];
         for (chunk_num, v) in v.chunks_mut(chunk).enumerate() {
             let f = f.clone();
             scope.spawn(move |_| {
                 let start = chunk_num * chunk;
                 f(v, start);
             });
+            let start = chunk_num * chunk;
+            chunk_starts.push(start);
         }
-    });
+
+        chunk_starts
+    })
+}
+
+pub fn parallelize<T: Send, F: Fn(&mut [T], usize) + Send + Sync + Clone>(v: &mut [T], f: F) {
+    parallelize_internal(v, f);
 }
 
 fn log2_floor(num: usize) -> u32 {