1212- [ Preset] ( #preset )
1313 - [ Cells] ( #cells )
1414- [ Helper functions] ( #helper-functions )
15+ - [ BLS12-381 helpers] ( #bls12-381-helpers )
16+ - [ ` bytes_to_cell ` ] ( #bytes_to_cell )
1517 - [ Linear combinations] ( #linear-combinations )
1618 - [ ` g2_lincomb ` ] ( #g2_lincomb )
1719 - [ FFTs] ( #ffts )
4042 - [ ` verify_cell_proof ` ] ( #verify_cell_proof )
4143 - [ ` verify_cell_proof_batch ` ] ( #verify_cell_proof_batch )
4244- [ Reconstruction] ( #reconstruction )
45+ - [ ` construct_vanishing_polynomial ` ] ( #construct_vanishing_polynomial )
46+ - [ ` recover_shifted_data ` ] ( #recover_shifted_data )
47+ - [ ` recover_original_data ` ] ( #recover_original_data )
4348 - [ ` recover_polynomial ` ] ( #recover_polynomial )
4449
4550<!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -74,13 +79,26 @@ Cells are the smallest unit of blob data that can come with their own KZG proofs
7479
7580| Name | Value | Description |
7681| - | - | - |
82+ | ` FIELD_ELEMENTS_PER_EXT_BLOB ` | ` 2 * FIELD_ELEMENTS_PER_BLOB ` | Number of field elements in a Reed-Solomon extended blob |
7783| ` FIELD_ELEMENTS_PER_CELL ` | ` uint64(64) ` | Number of field elements in a cell |
7884| ` BYTES_PER_CELL ` | ` FIELD_ELEMENTS_PER_CELL * BYTES_PER_FIELD_ELEMENT ` | The number of bytes in a cell |
79- | ` CELLS_PER_BLOB ` | ` ((2 * FIELD_ELEMENTS_PER_BLOB) // FIELD_ELEMENTS_PER_CELL) ` | The number of cells in a blob |
85+ | ` CELLS_PER_BLOB ` | ` FIELD_ELEMENTS_PER_EXT_BLOB // FIELD_ELEMENTS_PER_CELL` | The number of cells in a blob |
8086| ` RANDOM_CHALLENGE_KZG_CELL_BATCH_DOMAIN ` | ` b'RCKZGCBATCH__V1_' ` |
8187
8288## Helper functions
8389
90+ ### BLS12-381 helpers
91+
92+ #### ` bytes_to_cell `
93+
94+ ``` python
95+ def bytes_to_cell (cell_bytes : Vector[Bytes32, FIELD_ELEMENTS_PER_CELL ]) -> Cell:
96+ """
97+ Convert untrusted bytes into a Cell.
98+ """
99+ return [bytes_to_bls_field(element) for element in cell_bytes]
100+ ```
101+
84102### Linear combinations
85103
86104#### ` g2_lincomb `
@@ -156,7 +174,9 @@ def add_polynomialcoeff(a: PolynomialCoeff, b: PolynomialCoeff) -> PolynomialCoe
156174 Sum the coefficient form polynomials ``a`` and ``b``.
157175 """
158176 a, b = (a, b) if len (a) >= len (b) else (b, a)
159- return [(a[i] + (b[i] if i < len (b) else 0 )) % BLS_MODULUS for i in range (len (a))]
177+ length_a = len (a)
178+ length_b = len (b)
179+ return [(a[i] + (b[i] if i < length_b else 0 )) % BLS_MODULUS for i in range (length_a)]
160180```
161181
162182#### ` neg_polynomialcoeff `
@@ -242,7 +262,7 @@ def interpolate_polynomialcoeff(xs: Sequence[BLSFieldElement], ys: Sequence[BLSF
242262 summand, [(- int (weight_adjustment) * int (xs[j])) % BLS_MODULUS , weight_adjustment]
243263 )
244264 r = add_polynomialcoeff(r, summand)
245-
265+
246266 return r
247267```
248268
@@ -330,13 +350,13 @@ def verify_kzg_proof_multi_impl(commitment: KZGCommitment,
330350#### ` coset_for_cell `
331351
332352``` python
333- def coset_for_cell (cell_id : int ) -> Cell:
353+ def coset_for_cell (cell_id : CellID ) -> Cell:
334354 """
335355 Get the coset for a given ``cell_id``
336356 """
337357 assert cell_id < CELLS_PER_BLOB
338358 roots_of_unity_brp = bit_reversal_permutation(
339- compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB )
359+ compute_roots_of_unity(FIELD_ELEMENTS_PER_EXT_BLOB )
340360 )
341361 return Cell(roots_of_unity_brp[FIELD_ELEMENTS_PER_CELL * cell_id:FIELD_ELEMENTS_PER_CELL * (cell_id + 1 )])
342362```
@@ -385,8 +405,8 @@ def compute_cells(blob: Blob) -> Vector[Cell, CELLS_PER_BLOB]:
385405 polynomial = blob_to_polynomial(blob)
386406 polynomial_coeff = polynomial_eval_to_coeff(polynomial)
387407
388- extended_data = fft_field(polynomial_coeff + [0 ] * FIELD_ELEMENTS_PER_BLOB ,
389- compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB ))
408+ extended_data = fft_field(polynomial_coeff + [0 ] * FIELD_ELEMENTS_PER_BLOB ,
409+ compute_roots_of_unity(FIELD_ELEMENTS_PER_EXT_BLOB ))
390410 extended_data_rbo = bit_reversal_permutation(extended_data)
391411 return [extended_data_rbo[i * FIELD_ELEMENTS_PER_CELL :(i + 1 ) * FIELD_ELEMENTS_PER_CELL ]
392412 for i in range (CELLS_PER_BLOB )]
@@ -397,30 +417,37 @@ def compute_cells(blob: Blob) -> Vector[Cell, CELLS_PER_BLOB]:
397417#### ` verify_cell_proof `
398418
399419``` python
400- def verify_cell_proof (commitment : KZGCommitment ,
401- cell_id : int ,
402- cell : Cell ,
403- proof : KZGProof ) -> bool :
420+ def verify_cell_proof (commitment_bytes : Bytes48 ,
421+ cell_id : CellID ,
422+ cell_bytes : Vector[Bytes32, FIELD_ELEMENTS_PER_CELL ] ,
423+ proof_bytes : Bytes48 ) -> bool :
404424 """
405425 Check a cell proof
406426
407427 Public method.
408428 """
409429 coset = coset_for_cell(cell_id)
410430
411- return verify_kzg_proof_multi_impl(commitment, coset, cell, proof)
431+ return verify_kzg_proof_multi_impl(
432+ bytes_to_kzg_commitment(commitment_bytes),
433+ coset,
434+ bytes_to_cell(cell_bytes),
435+ bytes_to_kzg_proof(proof_bytes))
412436```
413437
414438#### ` verify_cell_proof_batch `
415439
416440``` python
417- def verify_cell_proof_batch (row_commitments : Sequence[KZGCommitment ],
418- row_ids : Sequence[int ],
419- column_ids : Sequence[int ],
420- cells : Sequence[Cell ],
421- proofs : Sequence[KZGProof ]) -> bool :
441+ def verify_cell_proof_batch (row_commitments_bytes : Sequence[Bytes48 ],
442+ row_ids : Sequence[uint64 ],
443+ column_ids : Sequence[uint64 ],
444+ cells_bytes : Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL ] ],
445+ proofs_bytes : Sequence[Bytes48 ]) -> bool :
422446 """
423- Check multiple cell proofs. This function implements the naive algorithm of checking every cell
447+ Verify a set of cells, given their corresponding proofs and their coordinates (row_id, column_id) in the blob
448+ matrix. The list of all commitments is also provided in row_commitments_bytes.
449+
450+ This function implements the naive algorithm of checking every cell
424451 individually; an efficient algorithm can be found here:
425452 https://ethresear.ch/t/a-universal-verification-equation-for-data-availability-sampling/13240
426453
@@ -430,10 +457,16 @@ def verify_cell_proof_batch(row_commitments: Sequence[KZGCommitment],
430457
431458 Public method.
432459 """
460+ assert len (cells_bytes) == len (proofs_bytes) == len (row_ids) == len (column_ids)
433461
434462 # Get commitments via row IDs
435- commitments = [row_commitments[row_id] for row_id in row_ids]
436-
463+ commitments_bytes = [row_commitments_bytes[row_id] for row_id in row_ids]
464+
465+ # Get objects from bytes
466+ commitments = [bytes_to_kzg_commitment(commitment_bytes) for commitment_bytes in commitments_bytes]
467+ cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes]
468+ proofs = [bytes_to_kzg_proof(proof_bytes) for proof_bytes in proofs_bytes]
469+
437470 return all (
438471 verify_kzg_proof_multi_impl(commitment, coset_for_cell(column_id), cell, proof)
439472 for commitment, column_id, cell, proof in zip (commitments, column_ids, cells, proofs)
@@ -442,80 +475,159 @@ def verify_cell_proof_batch(row_commitments: Sequence[KZGCommitment],
442475
443476## Reconstruction
444477
445- ### ` recover_polynomial `
478+ ### ` construct_vanishing_polynomial `
446479
447480``` python
448- def recover_polynomial (cell_ids : Sequence[CellID], cells : Sequence[Cell]) -> Polynomial:
481+ def construct_vanishing_polynomial (missing_cell_ids : Sequence[CellID]) -> Tuple[
482+ Sequence[BLSFieldElement],
483+ Sequence[BLSFieldElement]]:
449484 """
450- Recovers a polynomial from 2 * FIELD_ELEMENTS_PER_CELL evaluations, half of which can be missing.
451-
452- This algorithm uses FFTs to recover cells faster than using Lagrange implementation. However,
453- a faster version thanks to Qi Zhou can be found here:
454- https://github.com/ethereum/research/blob/51b530a53bd4147d123ab3e390a9d08605c2cdb8/polynomial_reconstruction/polynomial_reconstruction_danksharding.py
455-
456- Public method.
485+ Given the cells that are missing from the data, compute the polynomial that vanishes at every point that
486+ corresponds to a missing field element.
457487 """
458- assert len (cell_ids) == len (cells)
459- assert len (cells) >= CELLS_PER_BLOB // 2
460- missing_cell_ids = [cell_id for cell_id in range (CELLS_PER_BLOB ) if cell_id not in cell_ids]
488+ # Get the small domain
461489 roots_of_unity_reduced = compute_roots_of_unity(CELLS_PER_BLOB )
490+
491+ # Compute polynomial that vanishes at all the missing cells (over the small domain)
462492 short_zero_poly = vanishing_polynomialcoeff([
463- roots_of_unity_reduced[reverse_bits(cell_id , CELLS_PER_BLOB )]
464- for cell_id in missing_cell_ids
493+ roots_of_unity_reduced[reverse_bits(missing_cell_id , CELLS_PER_BLOB )]
494+ for missing_cell_id in missing_cell_ids
465495 ])
466496
467- full_zero_poly = []
468- for i in short_zero_poly:
469- full_zero_poly.append(i)
470- full_zero_poly.extend([0 ] * (FIELD_ELEMENTS_PER_CELL - 1 ))
471- full_zero_poly = full_zero_poly + [0 ] * (2 * FIELD_ELEMENTS_PER_BLOB - len (full_zero_poly))
497+ # Extend vanishing polynomial to full domain using the closed form of the vanishing polynomial over a coset
498+ zero_poly_coeff = [0 ] * FIELD_ELEMENTS_PER_EXT_BLOB
499+ for i, coeff in enumerate (short_zero_poly):
500+ zero_poly_coeff[i * FIELD_ELEMENTS_PER_CELL ] = coeff
472501
473- zero_poly_eval = fft_field(full_zero_poly,
474- compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB ))
502+ # Compute evaluations of the extended vanishing polynomial
503+ zero_poly_eval = fft_field(zero_poly_coeff,
504+ compute_roots_of_unity(FIELD_ELEMENTS_PER_EXT_BLOB ))
475505 zero_poly_eval_brp = bit_reversal_permutation(zero_poly_eval)
476- for cell_id in missing_cell_ids:
477- start = cell_id * FIELD_ELEMENTS_PER_CELL
478- end = (cell_id + 1 ) * FIELD_ELEMENTS_PER_CELL
479- assert zero_poly_eval_brp[start:end] == [0 ] * FIELD_ELEMENTS_PER_CELL
480- for cell_id in cell_ids:
506+
507+ # Sanity check
508+ for cell_id in range (CELLS_PER_BLOB ):
481509 start = cell_id * FIELD_ELEMENTS_PER_CELL
482510 end = (cell_id + 1 ) * FIELD_ELEMENTS_PER_CELL
483- assert all (a != 0 for a in zero_poly_eval_brp[start:end])
511+ if cell_id in missing_cell_ids:
512+ assert all (a == 0 for a in zero_poly_eval_brp[start:end])
513+ else : # cell_id in cell_ids
514+ assert all (a != 0 for a in zero_poly_eval_brp[start:end])
515+
516+ return zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp
517+ ```
518+
519+ ### ` recover_shifted_data `
484520
485- extended_evaluation_rbo = [0 ] * (FIELD_ELEMENTS_PER_BLOB * 2 )
521+ ``` python
522+ def recover_shifted_data (cell_ids : Sequence[CellID],
523+ cells : Sequence[Cell],
524+ zero_poly_eval : Sequence[BLSFieldElement],
525+ zero_poly_coeff : Sequence[BLSFieldElement],
526+ roots_of_unity_extended : Sequence[BLSFieldElement]) -> Tuple[
527+ Sequence[BLSFieldElement],
528+ Sequence[BLSFieldElement],
529+ BLSFieldElement]:
530+ """
531+ Given Z(x), return polynomial Q_1(x)=(E*Z)(k*x) and Q_2(x)=Z(k*x) and k^{-1}.
532+ """
533+ shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY )
534+ shift_inv = div(BLSFieldElement(1 ), shift_factor)
535+
536+ extended_evaluation_rbo = [0 ] * FIELD_ELEMENTS_PER_EXT_BLOB
486537 for cell_id, cell in zip (cell_ids, cells):
487538 start = cell_id * FIELD_ELEMENTS_PER_CELL
488539 end = (cell_id + 1 ) * FIELD_ELEMENTS_PER_CELL
489540 extended_evaluation_rbo[start:end] = cell
490541 extended_evaluation = bit_reversal_permutation(extended_evaluation_rbo)
491542
543+ # Compute (E*Z)(x)
492544 extended_evaluation_times_zero = [BLSFieldElement(int (a) * int (b) % BLS_MODULUS )
493545 for a, b in zip (zero_poly_eval, extended_evaluation)]
494546
495- roots_of_unity_extended = compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB )
496-
497547 extended_evaluations_fft = fft_field(extended_evaluation_times_zero, roots_of_unity_extended, inv = True )
498548
499- shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY )
500- shift_inv = div(BLSFieldElement(1 ), shift_factor)
501-
549+ # Compute (E*Z)(k*x)
502550 shifted_extended_evaluation = shift_polynomialcoeff(extended_evaluations_fft, shift_factor)
503- shifted_zero_poly = shift_polynomialcoeff(full_zero_poly, shift_factor)
551+ # Compute Z(k*x)
552+ shifted_zero_poly = shift_polynomialcoeff(zero_poly_coeff, shift_factor)
504553
505554 eval_shifted_extended_evaluation = fft_field(shifted_extended_evaluation, roots_of_unity_extended)
506555 eval_shifted_zero_poly = fft_field(shifted_zero_poly, roots_of_unity_extended)
507-
556+
557+ return eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv
558+ ```
559+
560+ ### ` recover_original_data `
561+
562+ ``` python
563+ def recover_original_data (eval_shifted_extended_evaluation : Sequence[BLSFieldElement],
564+ eval_shifted_zero_poly : Sequence[BLSFieldElement],
565+ shift_inv : BLSFieldElement,
566+ roots_of_unity_extended : Sequence[BLSFieldElement]) -> Sequence[BLSFieldElement]:
567+ """
568+ Given Q_1, Q_2 and k^{-1}, compute P(x).
569+ """
570+ # Compute Q_3 = Q_1(x)/Q_2(x) = P(k*x)
508571 eval_shifted_reconstructed_poly = [
509572 div(a, b)
510573 for a, b in zip (eval_shifted_extended_evaluation, eval_shifted_zero_poly)
511574 ]
512575
513576 shifted_reconstructed_poly = fft_field(eval_shifted_reconstructed_poly, roots_of_unity_extended, inv = True )
514577
578+ # Unshift P(k*x) by k^{-1} to get P(x)
515579 reconstructed_poly = shift_polynomialcoeff(shifted_reconstructed_poly, shift_inv)
516580
517581 reconstructed_data = bit_reversal_permutation(fft_field(reconstructed_poly, roots_of_unity_extended))
518582
583+ return reconstructed_data
584+ ```
585+
586+ ### ` recover_polynomial `
587+
588+ ``` python
589+ def recover_polynomial (cell_ids : Sequence[CellID],
590+ cells_bytes : Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL ]]) -> Polynomial:
591+ """
592+ Recover original polynomial from FIELD_ELEMENTS_PER_EXT_BLOB evaluations, half of which can be missing. This
593+ algorithm uses FFTs to recover cells faster than using Lagrange implementation, as can be seen here:
594+ https://ethresear.ch/t/reed-solomon-erasure-code-recovery-in-n-log-2-n-time-with-ffts/3039
595+
596+ A faster version thanks to Qi Zhou can be found here:
597+ https://github.com/ethereum/research/blob/51b530a53bd4147d123ab3e390a9d08605c2cdb8/polynomial_reconstruction/polynomial_reconstruction_danksharding.py
598+
599+ Public method.
600+ """
601+ assert len (cell_ids) == len (cells_bytes)
602+ # Check we have enough cells to be able to perform the reconstruction
603+ assert CELLS_PER_BLOB / 2 <= len (cell_ids) <= CELLS_PER_BLOB
604+ # Check for duplicates
605+ assert len (cell_ids) == len (set (cell_ids))
606+
607+ # Get the extended domain
608+ roots_of_unity_extended = compute_roots_of_unity(FIELD_ELEMENTS_PER_EXT_BLOB )
609+
610+ # Convert from bytes to cells
611+ cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes]
612+
613+ missing_cell_ids = [cell_id for cell_id in range (CELLS_PER_BLOB ) if cell_id not in cell_ids]
614+ zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(missing_cell_ids)
615+
616+ eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv = recover_shifted_data(
617+ cell_ids,
618+ cells,
619+ zero_poly_eval,
620+ zero_poly_coeff,
621+ roots_of_unity_extended,
622+ )
623+
624+ reconstructed_data = recover_original_data(
625+ eval_shifted_extended_evaluation,
626+ eval_shifted_zero_poly,
627+ shift_inv,
628+ roots_of_unity_extended,
629+ )
630+
519631 for cell_id, cell in zip (cell_ids, cells):
520632 start = cell_id * FIELD_ELEMENTS_PER_CELL
521633 end = (cell_id + 1 ) * FIELD_ELEMENTS_PER_CELL
0 commit comments