Merge branch 'development' into diego/trie-interface

.github/dependabot.yml

@@ -5,16 +5,16 @@ updates:
     schedule:
       interval: "weekly"
     labels:
-      - "dependencies"
+      - "S-dependencies"
   - package-ecosystem: docker
     directory: /
     schedule:
       interval: "weekly"
     labels:
-      - "dependencies"
+      - "S-dependencies"
   - package-ecosystem: gomod
     directory: /
     schedule:
       interval: "weekly"
     labels:
-      - "dependencies"
+      - "S-dependencies"

.github/labels.yml

@@ -236,4 +236,14 @@
 - name: S-dependencies
   color: "#1D76DB"
   aliases: []
-  description: issues related to polkadot host disputes subsystem functionality.
+  description: issues related to polkadot host disputes subsystem functionality.
+
+- name: S-infrastructure
+  color: "#1D76DB"
+  aliases: []
+  description: issues related to infrastructure and DevOps.
+
+- name: S-dependencies
+  color: "#1D76DB"
+  aliases: []
+  description: issues related to dependencies changes. Used by dependabot.

.github/workflows/build.yml

@@ -103,7 +103,7 @@ jobs:
       - name: Generate coverage report
         run: |
           go test ./... -coverprofile=coverage.out -covermode=atomic -timeout=20m
-      - uses: codecov/codecov-action@v4.0.2
+      - uses: codecov/codecov-action@v4.1.0
         with:
           files: ./coverage.out
           flags: unit-tests

.github/workflows/unit-tests.yml

@@ -89,7 +89,7 @@ jobs:
       - name: Test - Race
         run: make test-using-race-detector
 
-      - uses: codecov/codecov-action@v4.0.2
+      - uses: codecov/codecov-action@v4.1.0
         with:
           if_ci_failed: success
           informational: true

docs/docs/repo/labels.md

@@ -50,3 +50,5 @@ Below is the list of labels and their descriptions used in Gossamer repository.
   - **S-subsystems-backing** -  issues related to polkadot host backing subsystem functionality.
   - **S-subsystems-availability** - issues related to polkadot host availability subsystem functionality.
   - **S-subsystems-disputes** - issues related to polkadot host disputes subsystem functionality.
+  - **S-infrastructure** - issues related to infrastructure and DevOps.
+  - **S-dependencies** - issues related to dependencies changes. Used by dependabot.

dot/state/block.go

@@ -58,6 +58,8 @@ type BlockState struct {
 	lock              sync.RWMutex
 	genesisHash       common.Hash
 	lastFinalised     common.Hash
+	lastRound         uint64
+	lastSetID         uint64
 	unfinalisedBlocks *hashToBlockMap
 	tries             *Tries
 

dot/state/block_finalisation.go

@@ -54,6 +54,14 @@ func (bs *BlockState) GetFinalisedHeader(round, setID uint64) (*types.Header, er
 	return header, nil
 }
 
+// GetRoundAndSetID returns the finalised round and setID
+func (bs *BlockState) GetRoundAndSetID() (uint64, uint64) {
+	bs.lock.Lock()
+	defer bs.lock.Unlock()
+
+	return bs.lastRound, bs.lastSetID
+}
+
 // GetFinalisedHash gets the finalised block header by round and setID
 func (bs *BlockState) GetFinalisedHash(round, setID uint64) (common.Hash, error) {
 	h, err := bs.db.Get(finalisedHashKey(round, setID))
@@ -182,6 +190,8 @@ func (bs *BlockState) SetFinalisedHash(hash common.Hash, round, setID uint64) er
 	}
 
 	bs.lastFinalised = hash
+	bs.lastRound = round
+	bs.lastSetID = setID
 
 	logger.Infof(
 		"🔨 finalised block #%d (%s), round %d, set id %d", header.Number, hash, round, setID)

dot/sync/chain_sync.go

@@ -402,11 +402,11 @@ func (cs *chainSync) requestChainBlocks(announcedHeader, bestBlockHeader *types.
 		startAtBlock = announcedHeader.Number - uint(*request.Max) + 1
 		totalBlocks = *request.Max
 
-		logger.Infof("requesting %d blocks, descending request from #%d (%s)",
-			peerWhoAnnounced, gapLength, announcedHeader.Number, announcedHeader.Hash().Short())
+		logger.Infof("requesting %d blocks from peer: %v, descending request from #%d (%s)",
+			gapLength, peerWhoAnnounced, announcedHeader.Number, announcedHeader.Hash().Short())
 	} else {
 		request = network.NewBlockRequest(startingBlock, 1, network.BootstrapRequestData, network.Descending)
-		logger.Infof("requesting a single block #%d (%s)",
+		logger.Infof("requesting a single block from peer: %v with Number: #%d and Hash: (%s)",
 			peerWhoAnnounced, announcedHeader.Number, announcedHeader.Hash().Short())
 	}
 
@@ -445,8 +445,8 @@ func (cs *chainSync) requestForkBlocks(bestBlockHeader, highestFinalizedHeader,
 		request = network.NewBlockRequest(startingBlock, gapLength, network.BootstrapRequestData, network.Descending)
 	}
 
-	logger.Infof("requesting %d fork blocks, starting at #%d (%s)",
-		peerWhoAnnounced, gapLength, announcedHeader.Number, announcedHash.Short())
+	logger.Infof("requesting %d fork blocks from peer: %v starting at #%d (%s)",
+		gapLength, peerWhoAnnounced, announcedHeader.Number, announcedHash.Short())
 
 	resultsQueue := make(chan *syncTaskResult)
 	cs.workerPool.submitRequest(request, &peerWhoAnnounced, resultsQueue)

dot/sync/worker_pool.go

@@ -4,9 +4,8 @@
 package sync
 
 import (
-	"fmt"
-
 	"crypto/rand"
+	"fmt"
 	"math/big"
 	"sync"
 	"time"
@@ -161,9 +160,14 @@ func (s *syncWorkerPool) submitRequest(request *network.BlockRequestMessage,
 	defer s.mtx.RUnlock()
 
 	if who != nil {
-		syncWorker := s.workers[*who]
-		syncWorker.queue <- task
-		return
+		syncWorker, inMap := s.workers[*who]
+		if inMap {
+			if syncWorker == nil {
+				panic("sync worker should not be nil")
+			}
+			syncWorker.queue <- task
+			return
+		}
 	}
 
 	// if the exact peer is not specified then

go.mod

@@ -35,13 +35,13 @@ require (
 	github.com/qdm12/gotree v0.2.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.18.2
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/tetratelabs/wazero v1.1.0
 	go.uber.org/mock v0.4.0
-	golang.org/x/crypto v0.19.0
+	golang.org/x/crypto v0.21.0
 	golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e
-	golang.org/x/term v0.17.0
-	google.golang.org/protobuf v1.32.0
+	golang.org/x/term v0.18.0
+	google.golang.org/protobuf v1.33.0
 )
 
 require (
@@ -171,7 +171,7 @@ require (
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
@@ -189,7 +189,7 @@ require (
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.16.0 // indirect
 	gonum.org/v1/gonum v0.13.0 // indirect

go.sum

@@ -590,8 +590,9 @@ github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ=
 github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -600,8 +601,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
@@ -675,8 +677,8 @@ golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE=
 golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
@@ -767,11 +769,11 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -840,8 +842,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
-google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

lib/grandpa/message_handler.go

@@ -80,28 +80,25 @@ func (h *MessageHandler) handleMessage(from peer.ID, m GrandpaMessage) (network.
 }
 
 func (h *MessageHandler) handleNeighbourMessage(msg *NeighbourPacketV1) error {
-	if h.grandpa.authority {
-		// TODO(#2931): this is a simple hack to ensure that the neighbour messages
-		// sent by gossamer are being received by substrate nodes
-		// not intended to be production code
-		h.grandpa.roundLock.Lock()
-		neighbourMessage := &NeighbourPacketV1{
-			Round:  h.grandpa.state.round,
-			SetID:  h.grandpa.state.setID,
-			Number: uint32(h.grandpa.head.Number),
-		}
-		h.grandpa.roundLock.Unlock()
-
-		cm, err := neighbourMessage.ToConsensusMessage()
-		if err != nil {
-			return fmt.Errorf("converting neighbour message to network message: %w", err)
-		}
+	// TODO(#2931): this is a simple hack to ensure that the neighbour messages
+	// sent by gossamer are being received by substrate nodes
+	// not intended to be production code
+	round, setID := h.blockState.GetRoundAndSetID()
+	neighbourMessage := &NeighbourPacketV1{
+		Round:  round,
+		SetID:  setID,
+		Number: uint32(h.grandpa.head.Number),
+	}
 
-		logger.Debugf("sending neighbour message: %v", neighbourMessage)
-		h.grandpa.network.GossipMessage(cm)
+	cm, err := neighbourMessage.ToConsensusMessage()
+	if err != nil {
+		return fmt.Errorf("converting neighbour message to network message: %w", err)
 	}
 
-	currFinalized, err := h.blockState.GetFinalisedHeader(0, 0)
+	logger.Debugf("sending neighbour message: %v", neighbourMessage)
+	h.grandpa.network.GossipMessage(cm)
+
+	currFinalized, err := h.blockState.GetFinalisedHeader(round, setID)
 	if err != nil {
 		return err
 	}

lib/grandpa/state.go

@@ -23,6 +23,7 @@ type BlockState interface {
 	LowestCommonAncestor(a, b common.Hash) (common.Hash, error)
 	HasFinalisedBlock(round, setID uint64) (bool, error)
 	GetFinalisedHeader(round, setID uint64) (*types.Header, error)
+	GetRoundAndSetID() (uint64, uint64)
 	GetFinalisedHash(round, setID uint64) (common.Hash, error)
 	SetFinalisedHash(common.Hash, uint64, uint64) error
 	BestBlockHeader() (*types.Header, error)

lib/runtime/wazero/imports.go

@@ -9,6 +9,7 @@ import (
 	"crypto/rand"
 	"encoding/binary"
 	"fmt"
+	"math"
 	"math/big"
 	"reflect"
 	"time"
@@ -2139,21 +2140,19 @@ func ext_storage_clear_prefix_version_2(ctx context.Context, m api.Module, prefi
 
 	limitBytes := read(m, lim)
 
-	var limit []byte
-	err := scale.Unmarshal(limitBytes, &limit)
+	var limitPtr *uint32
+	err := scale.Unmarshal(limitBytes, &limitPtr)
 	if err != nil {
 		logger.Warnf("failed scale decoding limit: %s", err)
 		panic(err)
 	}
 
-	if len(limit) == 0 {
-		// limit is None, set limit to max
-		limit = []byte{0xff, 0xff, 0xff, 0xff}
+	if limitPtr == nil {
+		maxLimit := uint32(math.MaxUint32)
+		limitPtr = &maxLimit
 	}
 
-	limitUint := binary.LittleEndian.Uint32(limit)
-
-	numRemoved, all, err := storage.ClearPrefixLimit(prefix, limitUint)
+	numRemoved, all, err := storage.ClearPrefixLimit(prefix, *limitPtr)
 	if err != nil {
 		logger.Errorf("failed to clear prefix limit: %s", err)
 		panic(err)

lib/runtime/wazero/imports_test.go

@@ -1972,8 +1972,7 @@ func Test_ext_storage_clear_prefix_version_2(t *testing.T) {
 	testLimitBytes := make([]byte, 4)
 	binary.LittleEndian.PutUint32(testLimitBytes, testLimit)
 
-	optLimit, err := scale.Marshal(&testLimitBytes)
-	require.NoError(t, err)
+	optLimit := append([]byte{1}, testLimitBytes...)
 
 	// clearing prefix for "noo" prefix with limit 2
 	encValue, err := inst.Exec("rtm_ext_storage_clear_prefix_version_2", append(enc, optLimit...))