qbft/ Full node support (ssvlabs#123)

* WIP full node support

* WIP full node no tests yet

* WIP qbft spec tests

* all qbft spec tests pass

* all spec tests pass

* rebase

* linting

* align sync decided interface

qbft/controller.go

@@ -9,31 +9,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-// HistoricalInstanceCapacity represents the upper bound of InstanceContainer a processmsg can process messages for as messages are not
-// guaranteed to arrive in a timely fashion, we physically limit how far back the processmsg will process messages for
-const HistoricalInstanceCapacity int = 5
-
-type InstanceContainer [HistoricalInstanceCapacity]*Instance
-
-func (i InstanceContainer) FindInstance(height Height) *Instance {
-	for _, inst := range i {
-		if inst != nil {
-			if inst.GetHeight() == height {
-				return inst
-			}
-		}
-	}
-	return nil
-}
-
-// addNewInstance will add the new instance at index 0, pushing all other stored InstanceContainer one index up (ejecting last one if existing)
-func (i *InstanceContainer) addNewInstance(instance *Instance) {
-	for idx := HistoricalInstanceCapacity - 1; idx > 0; idx-- {
-		i[idx] = i[idx-1]
-	}
-	i[0] = instance
-}
-
 // Controller is a QBFT coordinator responsible for starting and following the entire life cycle of multiple QBFT InstanceContainer
 type Controller struct {
 	Identifier []byte
@@ -182,33 +157,7 @@ func (c *Controller) canStartInstance(value []byte) error {
 
 // GetRoot returns the state's deterministic root
 func (c *Controller) GetRoot() ([]byte, error) {
-	rootStruct := struct {
-		Identifier             []byte
-		Height                 Height
-		InstanceRoots          [][]byte
-		HigherReceivedMessages map[types.OperatorID]Height
-		Domain                 types.DomainType
-		Share                  *types.Share
-	}{
-		Identifier:             c.Identifier,
-		Height:                 c.Height,
-		InstanceRoots:          make([][]byte, len(c.StoredInstances)),
-		HigherReceivedMessages: c.FutureMsgsContainer,
-		Domain:                 c.Domain,
-		Share:                  c.Share,
-	}
-
-	for i, inst := range c.StoredInstances {
-		if inst != nil {
-			r, err := inst.GetRoot()
-			if err != nil {
-				return nil, errors.Wrap(err, "failed getting instance root")
-			}
-			rootStruct.InstanceRoots[i] = r
-		}
-	}
-
-	marshaledRoot, err := json.Marshal(rootStruct)
+	marshaledRoot, err := json.Marshal(c)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not encode state")
 	}

qbft/decided.go

@@ -7,11 +7,6 @@ import (
 
 // UponDecided returns decided msg if decided, nil otherwise
 func (c *Controller) UponDecided(msg *SignedMessage) (*SignedMessage, error) {
-	// decided msgs for past (already decided) instances will not decide again, just return
-	if msg.Message.Height < c.Height {
-		return nil, nil
-	}
-
 	if err := validateDecided(
 		c.config,
 		msg,
@@ -26,28 +21,32 @@ func (c *Controller) UponDecided(msg *SignedMessage) (*SignedMessage, error) {
 		return nil, errors.Wrap(err, "could not get decided data")
 	}
 
-	// did previously decide?
 	inst := c.InstanceForHeight(msg.Message.Height)
 	prevDecided := inst != nil && inst.State.Decided
-
-	// Mark current instance decided
-	if inst := c.InstanceForHeight(c.Height); inst != nil && !inst.State.Decided {
-		inst.State.Decided = true
-		if c.Height == msg.Message.Height {
-			inst.State.Round = msg.Message.Round
-			inst.State.DecidedValue = data.Data
-		}
-	}
-
 	isFutureDecided := msg.Message.Height > c.Height
-	if isFutureDecided {
-		// add an instance for the decided msg
+
+	if inst == nil {
 		i := NewInstance(c.GetConfig(), c.Share, c.Identifier, msg.Message.Height)
 		i.State.Round = msg.Message.Round
 		i.State.Decided = true
 		i.State.DecidedValue = data.Data
+		i.State.CommitContainer.AddMsg(msg)
 		c.StoredInstances.addNewInstance(i)
+	} else if decided, _ := inst.IsDecided(); !decided {
+		inst.State.Decided = true
+		inst.State.Round = msg.Message.Round
+		inst.State.DecidedValue = data.Data
+		inst.State.CommitContainer.AddMsg(msg)
+	} else { // decide previously, add if has more signers
+		signers, _ := inst.State.CommitContainer.LongestUniqueSignersForRoundAndValue(msg.Message.Round, msg.Message.Data)
+		if len(msg.Signers) > len(signers) {
+			inst.State.CommitContainer.AddMsg(msg)
+		}
+	}
 
+	if isFutureDecided {
+		// sync gap
+		c.GetConfig().GetNetwork().SyncDecidedByRange(types.MessageIDFromBytes(c.Identifier), c.Height, msg.Message.Height)
 		// bump height
 		c.Height = msg.Message.Height
 	}

qbft/message_container.go

@@ -108,6 +108,15 @@ func (c *MsgContainer) AddFirstMsgForSignerAndRound(msg *SignedMessage) (bool, e
 	return true, nil
 }
 
+// AddMsg will add any message regardless of signers
+func (c *MsgContainer) AddMsg(msg *SignedMessage) {
+	if c.Msgs[msg.Message.Round] == nil {
+		c.Msgs[msg.Message.Round] = make([]*SignedMessage, 0)
+	}
+	// add msg
+	c.Msgs[msg.Message.Round] = append(c.Msgs[msg.Message.Round], msg)
+}
+
 // Encode returns the encoded struct in bytes or error
 func (c *MsgContainer) Encode() ([]byte, error) {
 	return json.Marshal(c)

qbft/message_container_test.go

@@ -71,6 +71,22 @@ func TestMsgContainer_Marshaling(t *testing.T) {
 	require.EqualValues(t, byts, decodedByts)
 }
 
+func TestMsgContainer_AddMsg(t *testing.T) {
+	t.Run("same message, one with more signers", func(t *testing.T) {
+		c := &MsgContainer{
+			Msgs: map[Round][]*SignedMessage{},
+		}
+
+		c.AddMsg(&SignedMessage{Signers: []types.OperatorID{1, 2, 3}, Message: &Message{Round: 1, Data: []byte{1, 2, 3, 4}}})
+		c.AddMsg(&SignedMessage{Signers: []types.OperatorID{1}, Message: &Message{Round: 1, Data: []byte{1, 2, 3, 4}}})
+		c.AddMsg(&SignedMessage{Signers: []types.OperatorID{1, 2, 3, 4}, Message: &Message{Round: 1, Data: []byte{1, 2, 3, 4}}})
+
+		cnt, msgs := c.LongestUniqueSignersForRoundAndValue(1, []byte{1, 2, 3, 4})
+		require.EqualValues(t, []types.OperatorID{1, 2, 3, 4}, cnt)
+		require.Len(t, msgs, 1)
+	})
+}
+
 func TestMsgContainer_UniqueSignersSetForRoundAndValue(t *testing.T) {
 	t.Run("multi common signers with different values", func(t *testing.T) {
 		c := &MsgContainer{

qbft/spectest/all_tests.go

@@ -29,17 +29,18 @@ var AllTests = []SpecTest{
 	timeout.Round5(),
 	timeout.Round20(),
 
+	decided.Valid(),
 	decided.HasQuorum(),
+	decided.LateDecided(),
+	decided.LateDecidedBiggerQuorum(),
+	decided.LateDecidedSmallerQuorum(),
 	decided.NoQuorum(),
 	decided.DuplicateMsg(),
 	decided.DuplicateSigners(),
 	decided.ImparsableData(),
 	decided.Invalid(),
 	decided.InvalidData(),
 	decided.InvalidValCheckData(),
-	decided.LateDecided(),
-	decided.LateDecidedBiggerQuorum(),
-	decided.LateDecidedSmallerQuorum(),
 	decided.PastInstance(),
 	decided.UnknownSigner(),
 	decided.WrongMsgType(),
@@ -51,7 +52,7 @@ var AllTests = []SpecTest{
 	decided.CurrentInstanceFutureRound(),
 
 	processmsg.MsgError(),
-	processmsg.SavedAndBroadcastedDecided(),
+	processmsg.BroadcastedDecided(),
 	processmsg.SingleConsensusMsg(),
 	processmsg.FullDecided(),
 	processmsg.InvalidIdentifier(),
@@ -75,7 +76,6 @@ var AllTests = []SpecTest{
 	latemsg.LateRoundChangePastRound(),
 	latemsg.FullFlowAfterDecided(),
 
-	startinstance.Valid(),
 	futuremsg.NoSigners(),
 	futuremsg.MultiSigners(),
 	futuremsg.Cleanup(),
@@ -85,6 +85,7 @@ var AllTests = []SpecTest{
 	futuremsg.UnknownSigner(),
 	futuremsg.WrongSig(),
 
+	startinstance.Valid(),
 	startinstance.EmptyValue(),
 	startinstance.NilValue(),
 	startinstance.PostFutureDecided(),

qbft/spectest/tests/controller/decided/current_instance.go

@@ -74,9 +74,11 @@ func CurrentInstance() *tests.ControllerSpecTest {
 							Data:       testingutils.CommitDataBytes([]byte{1, 2, 3, 4}),
 						}),
 				},
-				DecidedVal:         []byte{1, 2, 3, 4},
-				DecidedCnt:         1,
-				ControllerPostRoot: "a3945ad86891579e53430688e8fde340bfdb0457916c6d9627f3f6ba59d24e27",
+				ExpectedDecidedState: tests.DecidedState{
+					DecidedCnt: 1,
+					DecidedVal: []byte{1, 2, 3, 4},
+				},
+				ControllerPostRoot: "4b5f00fd0787e3985e6b7c57d13d18701c2fa345e36a9ce4e26520fa1a3a5e3b",
 			},
 		},
 	}

qbft/spectest/tests/controller/decided/current_instance_future_round.go

@@ -74,9 +74,11 @@ func CurrentInstanceFutureRound() *tests.ControllerSpecTest {
 							Data:       testingutils.CommitDataBytes([]byte{1, 2, 3, 4}),
 						}),
 				},
-				DecidedVal:         []byte{1, 2, 3, 4},
-				DecidedCnt:         1,
-				ControllerPostRoot: "2dcd43628bc2f2d87ee30171e864d65ed0fbe79c89da3629b267385d250dc112",
+				ExpectedDecidedState: tests.DecidedState{
+					DecidedCnt: 1,
+					DecidedVal: []byte{1, 2, 3, 4},
+				},
+				ControllerPostRoot: "4317e9b5d69a4b07a4811beca8d6a14486d44f1defde1fa5b188ca76cca4f8e3",
 			},
 		},
 	}

qbft/spectest/tests/controller/decided/current_instance_past_round.go

@@ -124,11 +124,13 @@ func CurrentInstancePastRound() *tests.ControllerSpecTest {
 		Name: "decide current instance past round",
 		RunInstanceData: []*tests.RunInstanceData{
 			{
-				InputValue:         []byte{1, 2, 3, 4},
-				InputMessages:      msgs,
-				DecidedVal:         []byte{1, 2, 3, 4},
-				DecidedCnt:         1,
-				ControllerPostRoot: "e7e94573bd69d0fac4ee33b931efbc3a995142e8937497b5ad536f5b2b0864f4",
+				InputValue:    []byte{1, 2, 3, 4},
+				InputMessages: msgs,
+				ExpectedDecidedState: tests.DecidedState{
+					DecidedCnt: 1,
+					DecidedVal: []byte{1, 2, 3, 4},
+				},
+				ControllerPostRoot: "4a43a4237144801c2f898f05b48a5f93044a87b4688631afcb52a621660fd6ce",
 			},
 		},
 	}

qbft/spectest/tests/controller/decided/duplicate_msg.go

@@ -39,9 +39,13 @@ func DuplicateMsg() *tests.ControllerSpecTest {
 							Data:       testingutils.CommitDataBytes([]byte{1, 2, 3, 4}),
 						}),
 				},
-				DecidedVal:         []byte{1, 2, 3, 4},
-				DecidedCnt:         1,
-				ControllerPostRoot: "c91970b0e33a3b6141567101956dffa63472e56ed041ffdd408ed822973f3caf",
+				ExpectedDecidedState: tests.DecidedState{
+					DecidedCnt:               1,
+					DecidedVal:               []byte{1, 2, 3, 4},
+					CalledSyncDecidedByRange: true,
+					DecidedByRangeValues:     [2]qbft.Height{0, 10},
+				},
+				ControllerPostRoot: "fe0a1f75a599b436548dbc0179179506eb060e2ea8d048afa145b1a2229c64c8",
 			},
 		},
 	}

qbft/spectest/tests/controller/decided/duplicate_signers.go

@@ -33,7 +33,7 @@ func DuplicateSigners() *tests.ControllerSpecTest {
 				InputMessages: []*qbft.SignedMessage{
 					msg,
 				},
-				ControllerPostRoot: "5b6ebc3aa0bfcedd466fca3fca7e1dcc0245def7d61d65aee1462436d819c7d0",
+				ControllerPostRoot: "7b74be21fcdae2e7ed495882d1a499642c15a7f732f210ee84fb40cc97d1ce96",
 			},
 		},
 		ExpectedError: "invalid decided msg: invalid decided msg: non unique signer",

qbft/spectest/tests/controller/decided/future_instance.go

@@ -28,10 +28,13 @@ func FutureInstance() *tests.ControllerSpecTest {
 							Data:       testingutils.CommitDataBytes([]byte{1, 2, 3, 4}),
 						}),
 				},
-
-				DecidedVal:         []byte{1, 2, 3, 4},
-				DecidedCnt:         1,
-				ControllerPostRoot: "c91970b0e33a3b6141567101956dffa63472e56ed041ffdd408ed822973f3caf",
+				ExpectedDecidedState: tests.DecidedState{
+					DecidedCnt:               1,
+					DecidedVal:               []byte{1, 2, 3, 4},
+					CalledSyncDecidedByRange: true,
+					DecidedByRangeValues:     [2]qbft.Height{0, 10},
+				},
+				ControllerPostRoot: "fe0a1f75a599b436548dbc0179179506eb060e2ea8d048afa145b1a2229c64c8",
 			},
 		},
 	}