Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[draft] Multi Thread PK Read/Write #21

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

[draft] Multi Thread PK Read/Write #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
afc466f
feat: derive Serialize, Deserialize for ProvingKey, VerifyingKey
jonathanpwang Jul 27, 2023
7d7527d
chore: clippy fix
jonathanpwang Jul 27, 2023
2fac78b
feat: update serialization example to compare bincode
jonathanpwang Jul 27, 2023
782df4a
chore: update example
jonathanpwang Jul 27, 2023
4beeb03
Add multi-thread pk read/write
nyunyunyunyu Oct 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions halo2_proofs/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,7 @@
[package]
name = "halo2_proofs"
version = "0.2.0"
authors = [
"Sean Bowe <[email protected]>",
"Ying Tong Lai <[email protected]>",
"Daira Hopwood <[email protected]>",
"Jack Grigg <[email protected]>",
]
authors = ["Sean Bowe <[email protected]>", "Ying Tong Lai <[email protected]>", "Daira Hopwood <[email protected]>", "Jack Grigg <[email protected]>"]
edition = "2021"
rust-version = "1.56.1"
description = """
Expand Down Expand Up @@ -55,12 +50,18 @@ ff = "0.12"
group = "0.12"
halo2curves = { path = "../arithmetic/curves" }
rand = "0.8"
rand_core = { version = "0.6", default-features = false}
rand_core = { version = "0.6", default-features = false }
tracing = "0.1"
blake2b_simd = "1"
rustc-hash = "1.1.0"
sha3 = "0.9.1"
ark-std = { version = "0.3.0", features = ["print-trace"], optional = true }
serde = { version = "1.0", default-features = false, features = ["derive"] }
bincode = "1.3.3"
maybe-rayon = { version = "0.1.0", default-features = false }
itertools = "0.10"
tokio = { version = "1.33", features = ["full"] }
serde_json = { version = "1.0", default-features = false }

# Developer tooling dependencies
plotters = { version = "0.3.0", optional = true }
Expand All @@ -72,13 +73,16 @@ criterion = "0.3"
gumdrop = "0.8"
proptest = "1"
rand_core = { version = "0.6", features = ["getrandom"] }
rand_chacha = "0.3.1"

[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dev-dependencies]
getrandom = { version = "0.2", features = ["js"] }

[features]
default = ["batch"]
default = ["batch", "multicore"]
multicore = ["maybe-rayon/threads"]
dev-graph = ["plotters", "tabbycat"]
test-dev-graph = ["dev-graph", "plotters/bitmap_backend", "plotters/bitmap_encoder", "plotters/ttf"]
gadget-traces = ["backtrace"]
sanity-checks = []
batch = ["rand/getrandom"]
Expand Down
174 changes: 118 additions & 56 deletions halo2_proofs/examples/serialization.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use std::{
fs::File,
fs::{self, File},
io::{BufReader, BufWriter, Write},
};

Expand Down Expand Up @@ -126,59 +126,121 @@ impl Circuit<Fr> for StandardPlonk {
}
}

fn main() {
let k = 4;
let circuit = StandardPlonk(Fr::random(OsRng));
let params = ParamsKZG::<Bn256>::setup(k, OsRng);
let vk = keygen_vk(&params, &circuit).expect("vk should not fail");
let pk = keygen_pk(&params, vk, &circuit).expect("pk should not fail");

let f = File::create("serialization-test.pk").unwrap();
let mut writer = BufWriter::new(f);
pk.write(&mut writer, SerdeFormat::RawBytes).unwrap();
writer.flush().unwrap();

let f = File::open("serialization-test.pk").unwrap();
let mut reader = BufReader::new(f);
let pk = ProvingKey::<G1Affine>::read::<_, StandardPlonk>(&mut reader, SerdeFormat::RawBytes)
.unwrap();

std::fs::remove_file("serialization-test.pk").unwrap();

let instances: &[&[Fr]] = &[&[circuit.0]];
let mut transcript = Blake2bWrite::<_, _, Challenge255<_>>::init(vec![]);
create_proof::<
KZGCommitmentScheme<Bn256>,
ProverGWC<'_, Bn256>,
Challenge255<G1Affine>,
_,
Blake2bWrite<Vec<u8>, G1Affine, Challenge255<_>>,
_,
>(
&params,
&pk,
&[circuit],
&[instances],
OsRng,
&mut transcript,
)
.expect("prover should not fail");
let proof = transcript.finalize();

let strategy = SingleStrategy::new(&params);
let mut transcript = Blake2bRead::<_, _, Challenge255<_>>::init(&proof[..]);
assert!(verify_proof::<
KZGCommitmentScheme<Bn256>,
VerifierGWC<'_, Bn256>,
Challenge255<G1Affine>,
Blake2bRead<&[u8], G1Affine, Challenge255<G1Affine>>,
SingleStrategy<'_, Bn256>,
>(
&params,
pk.get_vk(),
strategy,
&[instances],
&mut transcript
)
.is_ok());
#[tokio::main(flavor = "multi_thread", worker_threads = 24)]
async fn main() -> std::io::Result<()> {
let k = 22;

let buf_size = 1024 * 1024;

// let pk_path = "/home/ubuntu/playground/serialization-test.pk";
// let f = File::open(pk_path)?;
// let mut reader = BufReader::with_capacity(buf_size, f);
// let start = std::time::Instant::now();
// let pk = ProvingKey::<G1Affine>::read::<_, StandardPlonk>(&mut reader, SerdeFormat::RawBytes)
// .unwrap();
// println!("SerdeFormat::RawBytes pk read time: {:?}", start.elapsed());

// let pk_folder = "/home/ubuntu/playground/serialization-test/";
let pk_folder = "/mnt/ramdisk/serialization-test";
// let start = std::time::Instant::now();
// pk.multi_thread_write(pk_folder, SerdeFormat::RawBytes)?;
// println!(
// "SerdeFormat::RawBytes pk multi thread write time: {:?}",
// start.elapsed()
// );

let start = std::time::Instant::now();
ProvingKey::<G1Affine>::multi_thread_read::<StandardPlonk>(pk_folder, SerdeFormat::RawBytes)
.await?;
println!(
"SerdeFormat::RawBytes pk multi thread read time: {:?}",
start.elapsed()
);

Ok(())
// let circuit = StandardPlonk(Fr::random(OsRng));
// let params = ParamsKZG::<Bn256>::setup(k, OsRng);
// let vk = keygen_vk(&params, &circuit).expect("vk should not fail");
// let pk = keygen_pk(&params, vk, &circuit).expect("pk should not fail");

// for buf_size in [1024, 8 * 1024, 1024 * 1024, 1024 * 1024 * 1024] {
// println!("buf_size: {buf_size}");
// // Using halo2_proofs serde implementation
// let f = File::create("serialization-test.pk")?;
// let mut writer = BufWriter::with_capacity(buf_size, f);
// let start = std::time::Instant::now();
// pk.write(&mut writer, SerdeFormat::RawBytes)?;
// writer.flush().unwrap();
// println!("SerdeFormat::RawBytes pk write time: {:?}", start.elapsed());

// let f = File::open("serialization-test.pk")?;
// let mut reader = BufReader::with_capacity(buf_size, f);
// let start = std::time::Instant::now();
// let pk =
// ProvingKey::<G1Affine>::read::<_, StandardPlonk>(&mut reader, SerdeFormat::RawBytes)
// .unwrap();
// println!("SerdeFormat::RawBytes pk read time: {:?}", start.elapsed());

// let metadata = fs::metadata("serialization-test.pk")?;
// let file_size = metadata.len();
// println!("The size of the file is {} bytes", file_size);
// std::fs::remove_file("serialization-test.pk")?;

// // Using bincode
// let f = File::create("serialization-test.pk")?;
// let mut writer = BufWriter::with_capacity(buf_size, f);
// let start = std::time::Instant::now();
// bincode::serialize_into(&mut writer, &pk).unwrap();
// writer.flush().unwrap();
// println!("bincode pk write time: {:?}", start.elapsed());

// let f = File::open("serialization-test.pk").unwrap();
// let mut reader = BufReader::with_capacity(buf_size, f);
// let start = std::time::Instant::now();
// let pk: ProvingKey<G1Affine> = bincode::deserialize_from(&mut reader).unwrap();
// println!("bincode pk read time: {:?}", start.elapsed());

// let metadata = fs::metadata("serialization-test.pk")?;
// let file_size = metadata.len();
// println!("The size of the file is {} bytes", file_size);
// std::fs::remove_file("serialization-test.pk").unwrap();

// let instances: &[&[Fr]] = &[&[circuit.clone().0]];
// let mut transcript = Blake2bWrite::<_, _, Challenge255<_>>::init(vec![]);
// create_proof::<
// KZGCommitmentScheme<Bn256>,
// ProverGWC<'_, Bn256>,
// Challenge255<G1Affine>,
// _,
// Blake2bWrite<Vec<u8>, G1Affine, Challenge255<_>>,
// _,
// >(
// &params,
// &pk,
// &[circuit.clone()],
// &[instances],
// OsRng,
// &mut transcript,
// )
// .expect("prover should not fail");
// let proof = transcript.finalize();

// let strategy = SingleStrategy::new(&params);
// let mut transcript = Blake2bRead::<_, _, Challenge255<_>>::init(&proof[..]);
// assert!(verify_proof::<
// KZGCommitmentScheme<Bn256>,
// VerifierGWC<'_, Bn256>,
// Challenge255<G1Affine>,
// Blake2bRead<&[u8], G1Affine, Challenge255<G1Affine>>,
// SingleStrategy<'_, Bn256>,
// >(
// &params,
// pk.get_vk(),
// strategy,
// &[instances],
// &mut transcript
// )
// .is_ok());
// }
// Ok(())
}
2 changes: 1 addition & 1 deletion halo2_proofs/examples/shuffle.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@ fn main() {
K,
circuit.clone(),
Err(vec![(
((1, "z should end with 1").into(), 0, "").into(),
((1, "z should end with 1").into(), 0, "".to_owned()).into(),
FailureLocation::InRegion {
region: (0, "Shuffle original into shuffled").into(),
offset: 32,
Expand Down
28 changes: 14 additions & 14 deletions halo2_proofs/src/dev.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -834,9 +834,9 @@ impl<F: FieldExt> MockProver<F> {
Value::Real(x) if x.is_zero_vartime() => None,
Value::Real(_) => Some(VerifyFailure::ConstraintNotSatisfied {
constraint: (
(gate_index, gate.name()).into(),
(gate_index, gate.name().to_string()).into(),
poly_index,
gate.constraint_name(poly_index),
gate.constraint_name(poly_index).to_string(),
)
.into(),
location: FailureLocation::find_expressions(
Expand All @@ -860,9 +860,9 @@ impl<F: FieldExt> MockProver<F> {
}),
Value::Poison => Some(VerifyFailure::ConstraintPoisoned {
constraint: (
(gate_index, gate.name()).into(),
(gate_index, gate.name().to_string()).into(),
poly_index,
gate.constraint_name(poly_index),
gate.constraint_name(poly_index).to_string(),
)
.into(),
}),
Expand Down Expand Up @@ -995,7 +995,7 @@ impl<F: FieldExt> MockProver<F> {
assert!(table.binary_search(input).is_err());

Some(VerifyFailure::Lookup {
name: lookup.name,
name: lookup.name.clone(),
lookup_index,
location: FailureLocation::find_expressions(
&self.cs,
Expand Down Expand Up @@ -1146,7 +1146,7 @@ impl<F: FieldExt> MockProver<F> {
None
} else {
Some(VerifyFailure::CellNotAssigned {
gate: (gate_index, gate.name()).into(),
gate: (gate_index, gate.name().to_string()).into(),
region: (
r_i,
r.name.clone(),
Expand Down Expand Up @@ -1225,9 +1225,9 @@ impl<F: FieldExt> MockProver<F> {
Value::Real(x) if x.is_zero_vartime() => None,
Value::Real(_) => Some(VerifyFailure::ConstraintNotSatisfied {
constraint: (
(gate_index, gate.name()).into(),
(gate_index, gate.name().to_string()).into(),
poly_index,
gate.constraint_name(poly_index),
gate.constraint_name(poly_index).to_string(),
)
.into(),
location: FailureLocation::find_expressions(
Expand All @@ -1251,9 +1251,9 @@ impl<F: FieldExt> MockProver<F> {
}),
Value::Poison => Some(VerifyFailure::ConstraintPoisoned {
constraint: (
(gate_index, gate.name()).into(),
(gate_index, gate.name().to_string()).into(),
poly_index,
gate.constraint_name(poly_index),
gate.constraint_name(poly_index).to_string(),
)
.into(),
}),
Expand Down Expand Up @@ -1374,7 +1374,7 @@ impl<F: FieldExt> MockProver<F> {
.filter_map(move |(input, input_row)| {
if table.binary_search(input).is_err() {
Some(VerifyFailure::Lookup {
name: lookup.name,
name: lookup.name.clone(),
lookup_index,
location: FailureLocation::find_expressions(
&self.cs,
Expand Down Expand Up @@ -1781,7 +1781,7 @@ mod tests {
assert_eq!(
prover.verify(),
Err(vec![VerifyFailure::Lookup {
name: "lookup",
name: "lookup".to_owned(),
lookup_index: 0,
location: FailureLocation::InRegion {
region: (1, "Faulty synthesis").into(),
Expand Down Expand Up @@ -1913,7 +1913,7 @@ mod tests {
assert_eq!(
prover.verify(),
Err(vec![VerifyFailure::Lookup {
name: "lookup",
name: "lookup".to_owned(),
lookup_index: 0,
location: FailureLocation::InRegion {
region: (2, "Faulty synthesis").into(),
Expand Down Expand Up @@ -2030,7 +2030,7 @@ mod tests {
assert_eq!(
prover.verify(),
Err(vec![VerifyFailure::ConstraintNotSatisfied {
constraint: ((0, "Equality check").into(), 0, "").into(),
constraint: ((0, "Equality check".to_owned()).into(), 0, "".to_owned()).into(),
location: FailureLocation::InRegion {
region: (1, "Wrong synthesis").into(),
offset: 0,
Expand Down
4 changes: 2 additions & 2 deletions halo2_proofs/src/dev/failure.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ pub enum VerifyFailure {
/// A lookup input did not exist in its corresponding table.
Lookup {
/// The name of the lookup that is not satisfied.
name: &'static str,
name: String,
/// The index of the lookup that is not satisfied. These indices are assigned in
/// the order in which `ConstraintSystem::lookup` is called during
/// `Circuit::configure`.
Expand Down Expand Up @@ -280,7 +280,7 @@ impl Debug for VerifyFailure {
};

let debug = ConstraintCaseDebug {
constraint: *constraint,
constraint: constraint.clone(),
location: location.clone(),
cell_values: cell_values
.iter()
Expand Down
8 changes: 4 additions & 4 deletions halo2_proofs/src/dev/gates.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@ use crate::{

#[derive(Debug)]
struct Constraint {
name: &'static str,
name: String,
expression: String,
queries: BTreeSet<String>,
}

#[derive(Debug)]
struct Gate {
name: &'static str,
name: String,
constraints: Vec<Constraint>,
}

Expand Down Expand Up @@ -112,13 +112,13 @@ impl CircuitGates {
.gates
.iter()
.map(|gate| Gate {
name: gate.name(),
name: gate.name().to_owned(),
constraints: gate
.polynomials()
.iter()
.enumerate()
.map(|(i, constraint)| Constraint {
name: gate.constraint_name(i),
name: gate.constraint_name(i).to_owned(),
expression: constraint.evaluate(
&util::format_value,
&|selector| format!("S{}", selector.0),
Expand Down
Loading
Loading