Merge branch 'development' into eclesio/sync-strategy

pkg/finality-grandpa/README

@@ -0,0 +1,41 @@
+# finality-grandpa
+
+**GRANDPA**, **G**HOST-based **R**ecursive **AN**cestor **D**eriving **P**refix **A**greement, is a
+finality gadget for blockchains, implemented in Go, ported from the [parity rust implementation][rust-impl]. It 
+allows a set of nodes to come to BFT agreement on what is the canonical chain, which is produced by 
+some external block production mechanism. It works under the assumption of a partially synchronous 
+network model and with the presence of up to 1/3 Byzantine nodes.
+
+### Integration
+
+This package only implements the state machine for the GRANDPA protocol. In order to use this package it
+is necessary to implement some interfaces to do the integration which are responsible for providing 
+access to the underyling blockchain and setting up all the network communication.
+
+#### `Chain`
+
+The `Chain` interface allows the GRANDPA voter to check ancestry of a given block and also to query the
+best block in a given chain (which will be used for voting on).
+
+#### `Environment`
+
+The `Environment` trait defines the types that will be used for the input and output stream to
+receive and broadcast messages. It is also responsible for setting these up for a given round
+(through `RoundData`), as well as timers which are used for timeouts in the protocol.
+
+The interface exposes callbacks for the full lifecycle of a round:
+
+- proposed
+- prevoted
+- precommitted
+- completed
+
+As well as callbacks for notifying about block finality and voter misbehavior (equivocations).
+
+## Resources
+
+- [White paper][paper]
+- [Parity rust implementation][rust-impl]
+
+[paper]: https://github.com/w3f/consensus/blob/master/pdf/grandpa.pdf
+[rust-impl]: https://github.com/paritytech/finality-grandpa

pkg/finality-grandpa/bitfield.go

@@ -0,0 +1,155 @@
+// Copyright 2023 ChainSafe Systems (ON)
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package grandpa
+
+// A dynamically sized, write-once (per bit), lazily allocating bitfield.
+type bitfield struct {
+	bits []uint64
+}
+
+// newBitfield creates a new empty bitfield.
+func newBitfield() bitfield {
+	return bitfield{
+		bits: make([]uint64, 0),
+	}
+}
+
+// IsBlank returns whether the bitfield is blank or empty.
+func (b *bitfield) IsBlank() bool { //skipcq: GO-W1029
+	return len(b.bits) == 0
+}
+
+// Merge another bitfield into this bitfield.
+//
+// As a result, this bitfield has all bits set that are set in either bitfield.
+//
+// This function only allocates if this bitfield is shorter than the other
+// bitfield, in which case it is resized accordingly to accommodate for all
+// bits of the other bitfield.
+func (b *bitfield) Merge(other bitfield) *bitfield { //skipcq: GO-W1029
+	if len(b.bits) < len(other.bits) {
+		b.bits = append(b.bits, make([]uint64, len(other.bits)-len(b.bits))...)
+	}
+	for i, word := range other.bits {
+		b.bits[i] |= word
+	}
+	return b
+}
+
+// SetBit will set a bit in the bitfield at the specified position.
+//
+// If the bitfield is not large enough to accommodate for a bit set
+// at the specified position, it is resized accordingly.
+func (b *bitfield) SetBit(position uint) { //skipcq: GO-W1029
+	wordOff := position / 64
+	bitOff := position % 64
+
+	if wordOff >= uint(len(b.bits)) {
+		newLen := wordOff + 1
+		b.bits = append(b.bits, make([]uint64, newLen-uint(len(b.bits)))...)
+	}
+	b.bits[wordOff] |= 1 << (63 - bitOff)
+}
+
+// iter1s will get an iterator over all bits that are set (i.e. 1) in the bitfield,
+// starting at bit position `start` and moving in steps of size `2^step`
+// per word.
+func (b *bitfield) iter1s(start, step uint) (bit1s []bit1) { //skipcq: GO-W1029
+	return iter1s(b.bits, start, step)
+}
+
+// Iter1sEven will get an iterator over all bits that are set (i.e. 1) at even bit positions.
+func (b *bitfield) Iter1sEven() []bit1 { //skipcq: GO-W1029
+	return b.iter1s(0, 1)
+}
+
+// Iter1sOdd will get an iterator over all bits that are set (i.e. 1) at odd bit positions.
+func (b *bitfield) Iter1sOdd() []bit1 { //skipcq: GO-W1029
+	return b.iter1s(1, 1)
+}
+
+// iter1sMerged will get an iterator over all bits that are set (i.e. 1) when merging
+// this bitfield with another bitfield, without modifying either
+// bitfield, starting at bit position `start` and moving in steps
+// of size `2^step` per word.
+func (b *bitfield) iter1sMerged(other bitfield, start, step uint) []bit1 { //skipcq: GO-W1029
+	switch {
+	case len(b.bits) == len(other.bits):
+		zipped := make([]uint64, len(b.bits))
+		for i, a := range b.bits {
+			b := other.bits[i]
+			zipped[i] = a | b
+		}
+		return iter1s(zipped, start, step)
+	case len(b.bits) < len(other.bits):
+		zipped := make([]uint64, len(other.bits))
+		for i, bit := range other.bits {
+			var a uint64
+			if i < len(b.bits) {
+				a = b.bits[i]
+			}
+			zipped[i] = a | bit
+		}
+		return iter1s(zipped, start, step)
+	case len(b.bits) > len(other.bits):
+		zipped := make([]uint64, len(b.bits))
+		for i, a := range b.bits {
+			var b uint64
+			if i < len(other.bits) {
+				b = other.bits[i]
+			}
+			zipped[i] = a | b
+		}
+		return iter1s(zipped, start, step)
+	default:
+		panic("unreachable")
+	}
+}
+
+// Iter1sMergedEven will get an iterator over all bits that are set (i.e. 1) at even bit positions
+// when merging this bitfield with another bitfield, without modifying
+// either bitfield.
+func (b *bitfield) Iter1sMergedEven(other bitfield) []bit1 { //skipcq: GO-W1029
+	return b.iter1sMerged(other, 0, 1)
+}
+
+// Iter1sMergedOdd will get an iterator over all bits that are set (i.e. 1) at odd bit positions
+// when merging this bitfield with another bitfield, without modifying
+// either bitfield.
+func (b *bitfield) Iter1sMergedOdd(other bitfield) []bit1 { //skipcq: GO-W1029
+	return b.iter1sMerged(other, 1, 1)
+}
+
+// Turn an iterator over uint64 words into an iterator over bits that
+// are set (i.e. `1`) in these words, starting at bit position `start`
+// and moving in steps of size `2^step` per word.
+func iter1s(iter []uint64, start, step uint) (bit1s []bit1) {
+	if !(start < 64 && step < 7) {
+		panic("invalid start and step")
+	}
+	steps := (64 >> step) - (start >> step)
+	for i, word := range iter {
+		if word == 0 {
+			continue
+		}
+		for j := uint(0); j < steps; j++ {
+			bitPos := start + (j << step)
+			if testBit(word, bitPos) {
+				bit1s = append(bit1s, bit1{uint(i)*64 + bitPos})
+			}
+		}
+	}
+	return bit1s
+}
+
+func testBit(word uint64, position uint) bool {
+	mask := uint64(1 << (63 - position))
+	return word&mask == mask
+}
+
+// A bit that is set (i.e. 1) in a `bitfield`.
+type bit1 struct {
+	// The position of the bit in the bitfield.
+	position uint
+}

pkg/finality-grandpa/bitfield_test.go

@@ -0,0 +1,143 @@
+// Copyright 2023 ChainSafe Systems (ON)
+// SPDX-License-Identifier: LGPL-3.0-only
+
+package grandpa
+
+import (
+	"math"
+	"math/rand"
+	"reflect"
+	"testing"
+	"testing/quick"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Generate is used by testing/quick to genereate
+func (bitfield) Generate(rand *rand.Rand, size int) reflect.Value { //skipcq: GO-W1029
+	n := rand.Int() % size
+	bits := make([]uint64, n)
+	for i := range bits {
+		bits[i] = rand.Uint64()
+	}
+
+	// we need to make sure we don't add empty words at the end of the
+	// bitfield otherwise it would break equality on some of the tests
+	// below.
+	for len(bits) > 0 && bits[len(bits)-1] == 0 {
+		bits = bits[:len(bits)-2]
+	}
+	return reflect.ValueOf(bitfield{
+		bits: bits,
+	})
+}
+
+// Test if the bit at the specified position is set.
+func (b *bitfield) testBit(position uint) bool { //skipcq: GO-W1029
+	wordOff := position / 64
+	if wordOff >= uint(len(b.bits)) {
+		return false
+	}
+	return testBit(b.bits[wordOff], position%64)
+}
+
+func TestBitfield_SetBit(t *testing.T) {
+	f := func(a bitfield, idx uint) bool {
+		// let's bound the max bitfield index at 2^24. this is needed because when calling
+		// `SetBit` we will extend the backing vec to accommodate the given bitfield size, this
+		// way we restrict the maximum allocation size to 16MB.
+		idx = uint(math.Min(float64(idx), 1<<24))
+		a.SetBit(idx)
+		return a.testBit(idx)
+	}
+	if err := quick.Check(f, nil); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestBitfield_iter1s_bitor(t *testing.T) {
+	f := func(a, b bitfield) bool {
+		c := newBitfield()
+		copy(a.bits, c.bits)
+		cBits := c.iter1s(0, 0)
+		for _, bit := range cBits {
+			if !(a.testBit(bit.position) || b.testBit(bit.position)) {
+				return false
+			}
+		}
+		return true
+	}
+	if err := quick.Check(f, nil); err != nil {
+		t.Error(err)
+	}
+}
+
+func Test_iter1s(t *testing.T) {
+	t.Run("all", func(t *testing.T) {
+		f := func(a bitfield) bool {
+			b := newBitfield()
+			for _, bit1 := range a.iter1s(0, 0) {
+				b.SetBit(bit1.position)
+			}
+			return assert.Equal(t, a, b)
+		}
+		if err := quick.Check(f, nil); err != nil {
+			t.Error(err)
+		}
+	})
+
+	t.Run("even_odd", func(t *testing.T) {
+		f := func(a bitfield) bool {
+			b := newBitfield()
+			for _, bit1 := range a.Iter1sEven() {
+				assert.True(t, !b.testBit(bit1.position))
+				assert.True(t, bit1.position%2 == 0)
+				b.SetBit(bit1.position)
+			}
+			for _, bit1 := range a.Iter1sOdd() {
+				assert.True(t, !b.testBit(bit1.position))
+				assert.True(t, bit1.position%2 == 1)
+				b.SetBit(bit1.position)
+			}
+			return assert.Equal(t, a, b)
+		}
+		if err := quick.Check(f, nil); err != nil {
+			t.Error(err)
+		}
+	})
+}
+
+func Test_iter1sMerged(t *testing.T) {
+	t.Run("all", func(t *testing.T) {
+		f := func(a, b bitfield) bool {
+			c := newBitfield()
+			for _, bit1 := range a.iter1sMerged(b, 0, 0) {
+				c.SetBit(bit1.position)
+			}
+			return assert.Equal(t, &c, a.Merge(b))
+		}
+		if err := quick.Check(f, nil); err != nil {
+			t.Error(err)
+		}
+	})
+
+	t.Run("even_odd", func(t *testing.T) {
+		f := func(a, b bitfield) bool {
+			c := newBitfield()
+			for _, bit1 := range a.Iter1sMergedEven(b) {
+				assert.True(t, !c.testBit(bit1.position))
+				assert.True(t, bit1.position%2 == 0)
+				c.SetBit(bit1.position)
+			}
+			for _, bit1 := range a.Iter1sMergedOdd(b) {
+				assert.True(t, !c.testBit(bit1.position))
+				assert.True(t, bit1.position%2 == 1)
+				c.SetBit(bit1.position)
+			}
+			return assert.Equal(t, &c, a.Merge(b))
+		}
+		if err := quick.Check(f, nil); err != nil {
+			t.Error(err)
+		}
+	})
+}