Merge branch 'development' into eclesio/move-slot-to-header

.github/workflows/build.yml

@@ -103,7 +103,7 @@ jobs:
       - name: Generate coverage report
         run: |
           go test ./... -coverprofile=coverage.out -covermode=atomic -timeout=20m
-      - uses: codecov/codecov-action@v4.0.2
+      - uses: codecov/codecov-action@v4.1.0
         with:
           files: ./coverage.out
           flags: unit-tests

.github/workflows/docker-network.yml

@@ -0,0 +1,38 @@
+on:
+  pull_request:
+  # Commented paths to avoid skipping required workflow
+  # See https://github.community/t/feature-request-conditional-required-checks/16761
+  # paths:
+  #   - .github/workflows/docker-grandpa.yml
+  #   - "**/*.go"
+  #   - "chain/**"
+  #   - "cmd/**"
+  #   - "dot/**"
+  #   - "internal/**"
+  #   - "lib/**"
+  #   - "pkg/**"
+  #   - "tests/stress/**"
+  #   - go.mod
+  #   - go.sum
+name: docker-network
+
+jobs:
+  docker-network-tests:
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          all_but_latest: true
+
+      - uses: docker/build-push-action@v5
+        with:
+          load: true
+          target: builder
+          tags: chainsafe/gossamer:test
+
+      - name: Run grandpa
+        run: |
+          docker run chainsafe/gossamer:test sh -c "make it-network"

.github/workflows/unit-tests.yml

@@ -89,7 +89,7 @@ jobs:
       - name: Test - Race
         run: make test-using-race-detector
 
-      - uses: codecov/codecov-action@v4.0.2
+      - uses: codecov/codecov-action@v4.1.0
         with:
           if_ci_failed: success
           informational: true

.releaserc

@@ -25,7 +25,8 @@
                     {
                         "path": "dist/**"
                     }
-                ]
+                ],
+                "successComment": false
             }
         ]
     ]

Makefile

@@ -51,6 +51,10 @@ it-rpc: build
 	@echo "  >  \033[32mRunning Integration Tests RPC Specs mode...\033[0m "
 	MODE=rpc go test ./tests/rpc/... -timeout=10m -v
 
+it-network: build
+	@echo "  >  \033[32mRunning Integration Tests Kademlia...\033[0m "
+	MODE=network go test ./tests/network/... -timeout=10m -v
+
 it-sync: build
 	@echo "  >  \033[32mRunning Integration Tests sync mode...\033[0m "
 	MODE=sync go test ./tests/sync/... -timeout=5m -v

docs/docs/testing-and-debugging/test-suite.md

@@ -44,6 +44,13 @@ To run Gossamer **RPC** integration tests run the following command:
 make it-rpc
 ```
 
+
+To run Gossamer **Network** integration tests run the following command:
+
+```
+make it-network
+```
+
 To run Gossamer **Sync** integration tests run the following command:
 
 ```

dot/network/discovery.go

@@ -94,13 +94,14 @@ func (d *discovery) waitForPeers() (peers []peer.AddrInfo, err error) {
 func (d *discovery) start() error {
 	// this basically only works with enabled mDNS which is used only for local test setups. Without bootnodes kademilia
 	// would not bee able to connect to any peers and mDNS is used to find peers in local network.
-	// TODO: should be refactored because this if is basically used for local integration test purpose
+	// TODO: should be refactored because this if is basically used for local integration test purpose.
+	// Instead of waiting for peers to connect to start kad we can upgrade the kad routing table on every connection,
+	// I think that using d.dht.{LAN/WAN}.RoutingTable().UsefulNewPeer(peerID) should be a good option
 	if len(d.bootnodes) == 0 {
 		peers, err := d.waitForPeers()
 		if err != nil {
 			return fmt.Errorf("failed while waiting for peers: %w", err)
 		}
-
 		d.bootnodes = peers
 	}
 	logger.Debugf("starting DHT with bootnodes %v...", d.bootnodes)
@@ -133,17 +134,6 @@ func (d *discovery) start() error {
 	return d.discoverAndAdvertise()
 }
 
-func (d *discovery) stop() error {
-	if d.dht == nil {
-		return nil
-	}
-
-	ethmetrics.Unregister(checkPeerCountMetrics)
-	ethmetrics.Unregister(peersStoreMetrics)
-
-	return d.dht.Close()
-}
-
 func (d *discovery) discoverAndAdvertise() error {
 	d.rd = routing.NewRoutingDiscovery(d.dht)
 
@@ -233,6 +223,13 @@ func (d *discovery) findPeers() {
 	}
 }
 
-func (d *discovery) findPeer(peerID peer.ID) (peer.AddrInfo, error) {
-	return d.dht.FindPeer(d.ctx, peerID)
+func (d *discovery) stop() error {
+	if d.dht == nil {
+		return nil
+	}
+
+	ethmetrics.Unregister(checkPeerCountMetrics)
+	ethmetrics.Unregister(peersStoreMetrics)
+
+	return d.dht.Close()
 }

dot/network/service.go

@@ -36,7 +36,8 @@ const (
 	blockAnnounceID = "/block-announces/1"
 	transactionsID  = "/transactions/1"
 
-	maxMessageSize = 1024 * 64 // 64kb for now
+	maxMessageSize       = 1024 * 64 // 64kb for now
+	findPeerQueryTimeout = 10 * time.Second
 )
 
 var (
@@ -292,12 +293,11 @@ func (s *Service) Start() error {
 	// this handles all new connections (incoming and outgoing)
 	// it creates a per-protocol mutex for sending outbound handshakes to the peer
 	// connectHandler is a part of libp2p.Notifiee interface implementation and getting called in the very end
-	//after or Incoming or Outgoing node is connected
+	// after or Incoming or Outgoing node is connected.
 	s.host.cm.connectHandler = func(peerID peer.ID) {
 		for _, prtl := range s.notificationsProtocols {
 			prtl.peersData.setMutex(peerID)
 		}
-		// TODO: currently we only have one set so setID is 0, change this once we have more set in peerSet
 		const setID = 0
 		s.host.cm.peerSetHandler.Incoming(setID, peerID)
 	}
@@ -711,7 +711,9 @@ func (s *Service) processMessage(msg peerset.Message) {
 		addrInfo := s.host.p2pHost.Peerstore().PeerInfo(peerID)
 		if len(addrInfo.Addrs) == 0 {
 			var err error
-			addrInfo, err = s.host.discovery.findPeer(peerID)
+			ctx, cancel := context.WithTimeout(s.host.discovery.ctx, findPeerQueryTimeout)
+			defer cancel()
+			addrInfo, err = s.host.discovery.dht.FindPeer(ctx, peerID)
 			if err != nil {
 				logger.Warnf("failed to find peer id %s: %s", peerID, err)
 				return

dot/state/block.go

@@ -57,6 +57,8 @@ type BlockState struct {
 	lock              sync.RWMutex
 	genesisHash       common.Hash
 	lastFinalised     common.Hash
+	lastRound         uint64
+	lastSetID         uint64
 	unfinalisedBlocks *hashToBlockMap
 	tries             *Tries
 

dot/state/block_finalisation.go

@@ -54,6 +54,14 @@ func (bs *BlockState) GetFinalisedHeader(round, setID uint64) (*types.Header, er
 	return header, nil
 }
 
+// GetRoundAndSetID returns the finalised round and setID
+func (bs *BlockState) GetRoundAndSetID() (uint64, uint64) {
+	bs.lock.Lock()
+	defer bs.lock.Unlock()
+
+	return bs.lastRound, bs.lastSetID
+}
+
 // GetFinalisedHash gets the finalised block header by round and setID
 func (bs *BlockState) GetFinalisedHash(round, setID uint64) (common.Hash, error) {
 	h, err := bs.db.Get(finalisedHashKey(round, setID))
@@ -182,6 +190,8 @@ func (bs *BlockState) SetFinalisedHash(hash common.Hash, round, setID uint64) er
 	}
 
 	bs.lastFinalised = hash
+	bs.lastRound = round
+	bs.lastSetID = setID
 
 	logger.Infof(
 		"🔨 finalised block #%d (%s), round %d, set id %d", header.Number, hash, round, setID)

dot/sync/chain_sync.go

@@ -402,11 +402,11 @@ func (cs *chainSync) requestChainBlocks(announcedHeader, bestBlockHeader *types.
 		startAtBlock = announcedHeader.Number - uint(*request.Max) + 1
 		totalBlocks = *request.Max
 
-		logger.Infof("requesting %d blocks, descending request from #%d (%s)",
-			peerWhoAnnounced, gapLength, announcedHeader.Number, announcedHeader.Hash().Short())
+		logger.Infof("requesting %d blocks from peer: %v, descending request from #%d (%s)",
+			gapLength, peerWhoAnnounced, announcedHeader.Number, announcedHeader.Hash().Short())
 	} else {
 		request = network.NewBlockRequest(startingBlock, 1, network.BootstrapRequestData, network.Descending)
-		logger.Infof("requesting a single block #%d (%s)",
+		logger.Infof("requesting a single block from peer: %v with Number: #%d and Hash: (%s)",
 			peerWhoAnnounced, announcedHeader.Number, announcedHeader.Hash().Short())
 	}
 
@@ -445,8 +445,8 @@ func (cs *chainSync) requestForkBlocks(bestBlockHeader, highestFinalizedHeader,
 		request = network.NewBlockRequest(startingBlock, gapLength, network.BootstrapRequestData, network.Descending)
 	}
 
-	logger.Infof("requesting %d fork blocks, starting at #%d (%s)",
-		peerWhoAnnounced, gapLength, announcedHeader.Number, announcedHash.Short())
+	logger.Infof("requesting %d fork blocks from peer: %v starting at #%d (%s)",
+		gapLength, peerWhoAnnounced, announcedHeader.Number, announcedHash.Short())
 
 	resultsQueue := make(chan *syncTaskResult)
 	cs.workerPool.submitRequest(request, &peerWhoAnnounced, resultsQueue)

dot/sync/syncer.go

@@ -89,7 +89,7 @@ func (s *Service) Stop() error {
 // HandleBlockAnnounceHandshake notifies the `chainSync` module that
 // we have received a BlockAnnounceHandshake from the given peer.
 func (s *Service) HandleBlockAnnounceHandshake(from peer.ID, msg *network.BlockAnnounceHandshake) error {
-	logger.Infof("received block announce handshake from: %s, #%d (%s)",
+	logger.Debugf("received block announce handshake from: %s, #%d (%s)",
 		from, msg.BestBlockNumber, msg.BestBlockHash.Short())
 	return s.chainSync.onBlockAnnounceHandshake(from, msg.BestBlockHash, uint(msg.BestBlockNumber))
 }
@@ -98,7 +98,7 @@ func (s *Service) HandleBlockAnnounceHandshake(from peer.ID, msg *network.BlockA
 func (s *Service) HandleBlockAnnounce(from peer.ID, msg *network.BlockAnnounceMessage) error {
 	blockAnnounceHeader := types.NewHeader(msg.ParentHash, msg.StateRoot, msg.ExtrinsicsRoot, msg.Number, msg.Digest)
 	blockAnnounceHeaderHash := blockAnnounceHeader.Hash()
-	logger.Infof("received block announce from: %s, #%d (%s)", from,
+	logger.Debugf("received block announce from: %s, #%d (%s)", from,
 		blockAnnounceHeader.Number, blockAnnounceHeaderHash.Short())
 
 	// if the peer reports a lower or equal best block number than us,

dot/sync/worker_pool.go

@@ -4,9 +4,8 @@
 package sync
 
 import (
-	"fmt"
-
 	"crypto/rand"
+	"fmt"
 	"math/big"
 	"sync"
 	"time"
@@ -161,9 +160,14 @@ func (s *syncWorkerPool) submitRequest(request *network.BlockRequestMessage,
 	defer s.mtx.RUnlock()
 
 	if who != nil {
-		syncWorker := s.workers[*who]
-		syncWorker.queue <- task
-		return
+		syncWorker, inMap := s.workers[*who]
+		if inMap {
+			if syncWorker == nil {
+				panic("sync worker should not be nil")
+			}
+			syncWorker.queue <- task
+			return
+		}
 	}
 
 	// if the exact peer is not specified then

go.mod

@@ -38,19 +38,19 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/tetratelabs/wazero v1.1.0
 	go.uber.org/mock v0.4.0
-	golang.org/x/crypto v0.19.0
+	golang.org/x/crypto v0.21.0
 	golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e
-	golang.org/x/term v0.17.0
-	google.golang.org/protobuf v1.32.0
+	golang.org/x/term v0.18.0
+	google.golang.org/protobuf v1.33.0
 )
 
 require (
-	github.com/DataDog/zstd v1.5.2 // indirect
+	github.com/DataDog/zstd v1.4.5 // indirect
 	github.com/Jorropo/jsync v1.0.1 // indirect
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/errors v1.11.1 // indirect
@@ -65,9 +65,9 @@ require (
 	github.com/decred/base58 v1.0.5 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
-	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
+	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/flynn/noise v1.0.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
@@ -82,7 +82,7 @@ require (
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/glog v1.0.0 // indirect
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
@@ -91,7 +91,7 @@ require (
 	github.com/gtank/ristretto255 v0.1.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/uint256 v1.2.4 // indirect
 	github.com/huin/goupnp v1.3.0 // indirect
@@ -157,15 +157,14 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
-	github.com/quic-go/quic-go v0.38.2 // indirect
+	github.com/quic-go/quic-go v0.38.1 // indirect
 	github.com/quic-go/webtransport-go v0.5.3 // indirect
 	github.com/raulk/go-watchdog v1.3.0 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/rs/cors v1.8.2 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
-	github.com/smartystreets/assertions v1.13.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
@@ -189,7 +188,7 @@ require (
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.16.0 // indirect
 	gonum.org/v1/gonum v0.13.0 // indirect