feat(challenges): configurable challenge count (#93)

* feat(challenges): configurable challenge count

* ci(params): selectively generate 1KiB and 1GiB params

Author: laser

.circleci/config.yml

@@ -16,8 +16,8 @@ jobs:
             which paramcache || { printf '%s\n' "missing paramcache binary" >&2; exit 1; }
       - restore_parameter_cache
       - run:
-          name: Generate Groth parameters and verifying keys
-          command: paramcache
+          name: Generate Groth parameters and verifying keys for 1KiB and 1GiB sectors
+          command: paramcache -z=1024,1073741824
           no_output_timeout: 1h
       - save_parameter_cache
 
@@ -271,16 +271,6 @@ workflows:
           requires:
             - cargo_fetch
             - obtain_parameters
-      - ffi_regression:
-          name: e2e_256MiB (lifecycle)
-          sector_size: 268435456
-          test_idx: 2
-          requires:
-            - cargo_fetch
-            - obtain_parameters
-          filters:
-            branches:
-              only: master
       - ffi_regression:
           name: e2e_1GiB (lifecycle)
           sector_size: 1073741824
@@ -342,7 +332,7 @@ commands:
   restore_parameter_cache:
     steps:
       - restore_cache:
-          key: v17-proof-params-{{ arch }}
+          key: v17b-proof-params-{{ arch }}
           paths:
             - /tmp/filecoin-parameter-cache
   restore_rust_cache:
@@ -352,7 +342,7 @@ commands:
   save_parameter_cache:
     steps:
       - save_cache:
-          key: v17-proof-params-{{ arch }}
+          key: v17b-proof-params-{{ arch }}
           paths:
             - /tmp/filecoin-parameter-cache
   save_rust_cache:

Cargo.lock

@@ -867,9 +867,10 @@ unsafe fn sector_builder_lifecycle(sector_size: u64) -> Result<(), failure::Erro
     // generate and then verify a proof-of-spacetime for the sealed sector
     {
         let cseed = [1u8; 32];
+        let ccount = 1;
         let p_set = ProvingSet::new(get_sector_info(&mut ctx, b_ptr));
 
-        let candidates = generate_candidates(b_ptr, cseed, &p_set).unwrap();
+        let candidates = generate_candidates(b_ptr, cseed, ccount, &p_set).unwrap();
 
         let winners: Vec<sector_builder_ffi_FFICandidate> = candidates
             .into_iter()
@@ -881,12 +882,13 @@ unsafe fn sector_builder_lifecycle(sector_size: u64) -> Result<(), failure::Erro
             })
             .collect();
 
-        let proofs = generate_post(b_ptr, cseed, &p_set, &winners).unwrap();
+        let proofs = generate_post(b_ptr, cseed, ccount, &p_set, &winners).unwrap();
 
         assert!(
             verify_post(
                 cfg.sector_class.sector_size,
                 cseed,
+                ccount,
                 &p_set,
                 &proofs,
                 &winners,

sector-builder-ffi/examples/simple/main.rs

@@ -12,6 +12,7 @@ include!(concat!(env!("OUT_DIR"), "/libsector_builder_ffi.rs"));
 pub(crate) unsafe fn generate_candidates(
     ptr: *mut sector_builder_ffi_SectorBuilder,
     challenge_seed: [u8; 32],
+    challenge_count: u64,
     proving_set: &ProvingSet,
 ) -> Result<Vec<sector_builder_ffi_FFICandidate>, (sector_builder_ffi_FCPResponseStatus, String)> {
     let flattened_comm_rs = proving_set.flattened_comm_rs();
@@ -22,6 +23,7 @@ pub(crate) unsafe fn generate_candidates(
         flattened_comm_rs.as_ptr(),
         flattened_comm_rs.len(),
         &mut challenge_seed.clone(),
+        challenge_count,
         faulty_sector_ids.as_ptr(),
         faulty_sector_ids.len(),
     );
@@ -42,6 +44,7 @@ pub(crate) unsafe fn generate_candidates(
 pub(crate) unsafe fn generate_post(
     ptr: *mut sector_builder_ffi_SectorBuilder,
     challenge_seed: [u8; 32],
+    challenge_count: u64,
     proving_set: &ProvingSet,
     winners: &[sector_builder_ffi_FFICandidate],
 ) -> Result<Vec<u8>, (sector_builder_ffi_FCPResponseStatus, String)> {
@@ -52,6 +55,7 @@ pub(crate) unsafe fn generate_post(
         flattened_comm_rs.as_ptr(),
         flattened_comm_rs.len(),
         &mut challenge_seed.clone(),
+        challenge_count,
         winners.as_ptr(),
         winners.len(),
     );
@@ -70,6 +74,7 @@ pub(crate) unsafe fn generate_post(
 pub(crate) unsafe fn verify_post(
     sector_size: u64,
     challenge_seed: [u8; 32],
+    challenge_count: u64,
     proving_set: &ProvingSet,
     flattened_proofs: &[u8],
     winners: &[sector_builder_ffi_FFICandidate],
@@ -81,6 +86,7 @@ pub(crate) unsafe fn verify_post(
     let resp = sector_builder_ffi_reexported_verify_post(
         sector_size,
         &mut challenge_seed.clone(),
+        challenge_count,
         sector_ids.as_ptr(),
         sector_ids.len(),
         flattened_comm_rs.as_ptr(),

sector-builder-ffi/examples/simple/plumbing.rs

@@ -236,6 +236,7 @@ pub unsafe extern "C" fn sector_builder_ffi_generate_candidates(
     flattened_comm_rs_ptr: *const u8,
     flattened_comm_rs_len: libc::size_t,
     challenge_seed: &[u8; 32],
+    challenge_count: u64,
     faults_ptr: *const u64,
     faults_len: libc::size_t,
 ) -> *mut types::GenerateCandidatesResponse {
@@ -250,7 +251,7 @@ pub unsafe extern "C" fn sector_builder_ffi_generate_candidates(
             .map(|x| SectorId::from(*x))
             .collect();
 
-        let result = (*ptr).generate_candidates(&comm_rs, challenge_seed, faults);
+        let result = (*ptr).generate_candidates(&comm_rs, challenge_seed, challenge_count, faults);
 
         let mut response = types::GenerateCandidatesResponse::default();
 
@@ -304,6 +305,7 @@ pub unsafe extern "C" fn sector_builder_ffi_generate_post(
     flattened_comm_rs_ptr: *const u8,
     flattened_comm_rs_len: libc::size_t,
     challenge_seed: &[u8; 32],
+    challenge_count: u64,
     winners_ptr: *const FFICandidate,
     winners_len: libc::size_t,
 ) -> *mut types::GeneratePoStResponse {
@@ -319,7 +321,9 @@ pub unsafe extern "C" fn sector_builder_ffi_generate_post(
             .map(|c| c.try_into_candidate())
             .collect::<Result<_, _>>()
             .map_err(Into::into)
-            .and_then(|winners| (*ptr).generate_post(&comm_rs, challenge_seed, winners));
+            .and_then(|winners| {
+                (*ptr).generate_post(&comm_rs, challenge_seed, challenge_count, winners)
+            });
 
         let mut response = types::GeneratePoStResponse::default();
 

sector-builder-ffi/src/api.rs

@@ -77,6 +77,7 @@ pub unsafe extern "C" fn sector_builder_ffi_reexported_generate_data_commitment(
 pub unsafe extern "C" fn sector_builder_ffi_reexported_verify_post(
     sector_size: u64,
     randomness: &[u8; 32],
+    challenge_count: u64,
     sector_ids_ptr: *const u64,
     sector_ids_len: libc::size_t,
     flattened_comm_rs_ptr: *const u8,
@@ -93,6 +94,7 @@ pub unsafe extern "C" fn sector_builder_ffi_reexported_verify_post(
         filecoin_proofs_ffi::api::verify_post(
             sector_size,
             randomness,
+            challenge_count,
             sector_ids_ptr,
             sector_ids_len,
             flattened_comm_rs_ptr,

sector-builder-ffi/src/reexported.rs

@@ -191,10 +191,17 @@ impl<R: 'static + Send + std::io::Read> SectorBuilder<R> {
         &self,
         comm_rs: &[[u8; 32]],
         challenge_seed: &[u8; 32],
+        challenge_count: u64,
         faults: Vec<SectorId>,
     ) -> Result<Vec<Candidate>> {
         log_unrecov(self.run_blocking(|tx| {
-            SchedulerTask::GenerateCandidates(Vec::from(comm_rs), *challenge_seed, faults, tx)
+            SchedulerTask::GenerateCandidates(
+                Vec::from(comm_rs),
+                *challenge_seed,
+                challenge_count,
+                faults,
+                tx,
+            )
         }))
     }
 
@@ -203,10 +210,17 @@ impl<R: 'static + Send + std::io::Read> SectorBuilder<R> {
         &self,
         comm_rs: &[[u8; 32]],
         challenge_seed: &[u8; 32],
+        challenge_count: u64,
         winners: Vec<Candidate>,
     ) -> Result<Vec<Vec<u8>>> {
         log_unrecov(self.run_blocking(|tx| {
-            SchedulerTask::GeneratePoSt(Vec::from(comm_rs), *challenge_seed, winners, tx)
+            SchedulerTask::GeneratePoSt(
+                Vec::from(comm_rs),
+                *challenge_seed,
+                challenge_count,
+                winners,
+                tx,
+            )
         }))
     }
 

sector-builder/src/builder.rs

@@ -98,6 +98,7 @@ impl<T: KeyValueStore> SectorMetadataManager<T> {
         &self,
         comm_rs: &[[u8; 32]],
         randomness: &[u8; 32],
+        challenge_count: u64,
         faults: Option<Vec<SectorId>>,
     ) -> GeneratePoStTaskPrototype {
         let fault_set: Option<HashSet<SectorId>> = faults.map(|f| f.into_iter().collect());
@@ -139,6 +140,7 @@ impl<T: KeyValueStore> SectorMetadataManager<T> {
 
         GeneratePoStTaskPrototype {
             randomness: *randomness,
+            challenge_count,
             post_config: self.sector_store.proofs_config().post_config,
             private_replicas: replicas,
         }

sector-builder/src/metadata_manager.rs

@@ -57,12 +57,14 @@ pub enum SchedulerTask<T: Read + Send> {
     GenerateCandidates(
         Vec<[u8; 32]>,
         [u8; 32],      // seed
+        u64,           // challenge count
         Vec<SectorId>, // faults
         mpsc::SyncSender<Result<Vec<Candidate>>>,
     ),
     GeneratePoSt(
         Vec<[u8; 32]>,
         [u8; 32],       // seed
+        u64,            // challenge count
         Vec<Candidate>, // winners
         mpsc::SyncSender<Result<Vec<Vec<u8>>>>,
     ),
@@ -214,26 +216,39 @@ impl<T: KeyValueStore, V: 'static + Send + std::io::Read> TaskHandler<T, V> {
                 tx.send(self.m.read_unsealed_bytes_from(result))
                     .expects(FATAL_NOSEND);
             }
-            SchedulerTask::GenerateCandidates(comm_rs, chg_seed, faults, tx) => {
-                let proto =
-                    self.m
-                        .create_generate_post_task_proto(&comm_rs, &chg_seed, Some(faults));
+            SchedulerTask::GenerateCandidates(
+                comm_rs,
+                challenge_seed,
+                challenge_count,
+                faults,
+                tx,
+            ) => {
+                let proto = self.m.create_generate_post_task_proto(
+                    &comm_rs,
+                    &challenge_seed,
+                    challenge_count,
+                    Some(faults),
+                );
 
                 let callback = Box::new(move |r| tx.send(r).expects(FATAL_NOSEND));
 
                 self.worker_tx
                     .send(WorkerTask::GenerateCandidates {
                         randomness: proto.randomness,
+                        challenge_count: proto.challenge_count,
                         private_replicas: proto.private_replicas,
                         post_config: proto.post_config,
                         callback,
                     })
                     .expects(FATAL_NOSEND);
             }
-            SchedulerTask::GeneratePoSt(comm_rs, chg_seed, winners, tx) => {
-                let proto = self
-                    .m
-                    .create_generate_post_task_proto(&comm_rs, &chg_seed, None);
+            SchedulerTask::GeneratePoSt(comm_rs, challenge_seed, challenge_count, winners, tx) => {
+                let proto = self.m.create_generate_post_task_proto(
+                    &comm_rs,
+                    &challenge_seed,
+                    challenge_count,
+                    None,
+                );
 
                 let callback = Box::new(move |r| tx.send(r).expects(FATAL_NOSEND));
 

sector-builder/src/scheduler.rs

@@ -34,6 +34,7 @@ pub struct UnsealTaskPrototype {
 #[derive(Debug, Clone)]
 pub struct GeneratePoStTaskPrototype {
     pub(crate) randomness: [u8; 32],
+    pub(crate) challenge_count: u64,
     pub(crate) post_config: PoStConfig,
     pub(crate) private_replicas: BTreeMap<SectorId, PrivateReplicaInfo>,
 }
@@ -74,6 +75,7 @@ type SealCommitCallback = Box<dyn FnOnce(SealCommitResult) + Send>;
 pub enum WorkerTask {
     GenerateCandidates {
         randomness: [u8; 32],
+        challenge_count: u64,
         private_replicas: BTreeMap<SectorId, PrivateReplicaInfo>,
         post_config: PoStConfig,
         callback: GenerateCandidatesCallback,
@@ -139,13 +141,15 @@ impl Worker {
             match task {
                 WorkerTask::GenerateCandidates {
                     randomness,
+                    challenge_count,
                     private_replicas,
                     post_config,
                     callback,
                 } => {
                     callback(filecoin_proofs::generate_candidates(
                         post_config,
                         &randomness,
+                        challenge_count,
                         &private_replicas,
                         prover_id,
                     ));