Merge branch 'development' into diego/trie-interface

.github/workflows/docker-network.yml

@@ -0,0 +1,38 @@
+on:
+  pull_request:
+  # Commented paths to avoid skipping required workflow
+  # See https://github.community/t/feature-request-conditional-required-checks/16761
+  # paths:
+  #   - .github/workflows/docker-grandpa.yml
+  #   - "**/*.go"
+  #   - "chain/**"
+  #   - "cmd/**"
+  #   - "dot/**"
+  #   - "internal/**"
+  #   - "lib/**"
+  #   - "pkg/**"
+  #   - "tests/stress/**"
+  #   - go.mod
+  #   - go.sum
+name: docker-network
+
+jobs:
+  docker-network-tests:
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          all_but_latest: true
+
+      - uses: docker/build-push-action@v5
+        with:
+          load: true
+          target: builder
+          tags: chainsafe/gossamer:test
+
+      - name: Run grandpa
+        run: |
+          docker run chainsafe/gossamer:test sh -c "make it-network"

.releaserc

@@ -25,7 +25,8 @@
                     {
                         "path": "dist/**"
                     }
-                ]
+                ],
+                "successComment": false
             }
         ]
     ]

Makefile

@@ -51,6 +51,10 @@ it-rpc: build
 	@echo "  >  \033[32mRunning Integration Tests RPC Specs mode...\033[0m "
 	MODE=rpc go test ./tests/rpc/... -timeout=10m -v
 
+it-network: build
+	@echo "  >  \033[32mRunning Integration Tests Kademlia...\033[0m "
+	MODE=network go test ./tests/network/... -timeout=10m -v
+
 it-sync: build
 	@echo "  >  \033[32mRunning Integration Tests sync mode...\033[0m "
 	MODE=sync go test ./tests/sync/... -timeout=5m -v

docs/docs/testing-and-debugging/test-suite.md

@@ -44,6 +44,13 @@ To run Gossamer **RPC** integration tests run the following command:
 make it-rpc
 ```
 
+
+To run Gossamer **Network** integration tests run the following command:
+
+```
+make it-network
+```
+
 To run Gossamer **Sync** integration tests run the following command:
 
 ```

dot/network/discovery.go

@@ -94,13 +94,14 @@ func (d *discovery) waitForPeers() (peers []peer.AddrInfo, err error) {
 func (d *discovery) start() error {
 	// this basically only works with enabled mDNS which is used only for local test setups. Without bootnodes kademilia
 	// would not bee able to connect to any peers and mDNS is used to find peers in local network.
-	// TODO: should be refactored because this if is basically used for local integration test purpose
+	// TODO: should be refactored because this if is basically used for local integration test purpose.
+	// Instead of waiting for peers to connect to start kad we can upgrade the kad routing table on every connection,
+	// I think that using d.dht.{LAN/WAN}.RoutingTable().UsefulNewPeer(peerID) should be a good option
 	if len(d.bootnodes) == 0 {
 		peers, err := d.waitForPeers()
 		if err != nil {
 			return fmt.Errorf("failed while waiting for peers: %w", err)
 		}
-
 		d.bootnodes = peers
 	}
 	logger.Debugf("starting DHT with bootnodes %v...", d.bootnodes)
@@ -133,17 +134,6 @@ func (d *discovery) start() error {
 	return d.discoverAndAdvertise()
 }
 
-func (d *discovery) stop() error {
-	if d.dht == nil {
-		return nil
-	}
-
-	ethmetrics.Unregister(checkPeerCountMetrics)
-	ethmetrics.Unregister(peersStoreMetrics)
-
-	return d.dht.Close()
-}
-
 func (d *discovery) discoverAndAdvertise() error {
 	d.rd = routing.NewRoutingDiscovery(d.dht)
 
@@ -233,6 +223,13 @@ func (d *discovery) findPeers() {
 	}
 }
 
-func (d *discovery) findPeer(peerID peer.ID) (peer.AddrInfo, error) {
-	return d.dht.FindPeer(d.ctx, peerID)
+func (d *discovery) stop() error {
+	if d.dht == nil {
+		return nil
+	}
+
+	ethmetrics.Unregister(checkPeerCountMetrics)
+	ethmetrics.Unregister(peersStoreMetrics)
+
+	return d.dht.Close()
 }

dot/network/service.go

@@ -36,7 +36,8 @@ const (
 	blockAnnounceID = "/block-announces/1"
 	transactionsID  = "/transactions/1"
 
-	maxMessageSize = 1024 * 64 // 64kb for now
+	maxMessageSize       = 1024 * 64 // 64kb for now
+	findPeerQueryTimeout = 10 * time.Second
 )
 
 var (
@@ -292,12 +293,11 @@ func (s *Service) Start() error {
 	// this handles all new connections (incoming and outgoing)
 	// it creates a per-protocol mutex for sending outbound handshakes to the peer
 	// connectHandler is a part of libp2p.Notifiee interface implementation and getting called in the very end
-	//after or Incoming or Outgoing node is connected
+	// after or Incoming or Outgoing node is connected.
 	s.host.cm.connectHandler = func(peerID peer.ID) {
 		for _, prtl := range s.notificationsProtocols {
 			prtl.peersData.setMutex(peerID)
 		}
-		// TODO: currently we only have one set so setID is 0, change this once we have more set in peerSet
 		const setID = 0
 		s.host.cm.peerSetHandler.Incoming(setID, peerID)
 	}
@@ -711,7 +711,9 @@ func (s *Service) processMessage(msg peerset.Message) {
 		addrInfo := s.host.p2pHost.Peerstore().PeerInfo(peerID)
 		if len(addrInfo.Addrs) == 0 {
 			var err error
-			addrInfo, err = s.host.discovery.findPeer(peerID)
+			ctx, cancel := context.WithTimeout(s.host.discovery.ctx, findPeerQueryTimeout)
+			defer cancel()
+			addrInfo, err = s.host.discovery.dht.FindPeer(ctx, peerID)
 			if err != nil {
 				logger.Warnf("failed to find peer id %s: %s", peerID, err)
 				return

dot/state/epoch.go

@@ -22,7 +22,6 @@ var (
 	errHashNotInMemory    = errors.New("hash not found in memory map")
 	errEpochNotInDatabase = errors.New("epoch data not found in the database")
 	errHashNotPersisted   = errors.New("hash with next epoch not found in database")
-	errNoPreRuntimeDigest = errors.New("header does not contain pre-runtime digest")
 )
 
 var (
@@ -197,39 +196,12 @@ func (s *EpochState) GetEpochForBlock(header *types.Header) (uint64, error) {
 		return 0, err
 	}
 
-	for _, d := range header.Digest {
-		digestValue, err := d.Value()
-		if err != nil {
-			continue
-		}
-		predigest, ok := digestValue.(types.PreRuntimeDigest)
-		if !ok {
-			continue
-		}
-
-		digest, err := types.DecodeBabePreDigest(predigest.Data)
-		if err != nil {
-			return 0, fmt.Errorf("failed to decode babe header: %w", err)
-		}
-
-		var slotNumber uint64
-		switch d := digest.(type) {
-		case types.BabePrimaryPreDigest:
-			slotNumber = d.SlotNumber
-		case types.BabeSecondaryVRFPreDigest:
-			slotNumber = d.SlotNumber
-		case types.BabeSecondaryPlainPreDigest:
-			slotNumber = d.SlotNumber
-		}
-
-		if slotNumber < firstSlot {
-			return 0, nil
-		}
-
-		return (slotNumber - firstSlot) / s.epochLength, nil
+	slotNumber, err := header.SlotNumber()
+	if err != nil {
+		return 0, fmt.Errorf("getting slot number: %w", err)
 	}
 
-	return 0, errNoPreRuntimeDigest
+	return (slotNumber - firstSlot) / s.epochLength, nil
 }
 
 // SetEpochDataRaw sets the epoch data raw for a given epoch

dot/sync/syncer.go

@@ -89,7 +89,7 @@ func (s *Service) Stop() error {
 // HandleBlockAnnounceHandshake notifies the `chainSync` module that
 // we have received a BlockAnnounceHandshake from the given peer.
 func (s *Service) HandleBlockAnnounceHandshake(from peer.ID, msg *network.BlockAnnounceHandshake) error {
-	logger.Infof("received block announce handshake from: %s, #%d (%s)",
+	logger.Debugf("received block announce handshake from: %s, #%d (%s)",
 		from, msg.BestBlockNumber, msg.BestBlockHash.Short())
 	return s.chainSync.onBlockAnnounceHandshake(from, msg.BestBlockHash, uint(msg.BestBlockNumber))
 }
@@ -98,7 +98,7 @@ func (s *Service) HandleBlockAnnounceHandshake(from peer.ID, msg *network.BlockA
 func (s *Service) HandleBlockAnnounce(from peer.ID, msg *network.BlockAnnounceMessage) error {
 	blockAnnounceHeader := types.NewHeader(msg.ParentHash, msg.StateRoot, msg.ExtrinsicsRoot, msg.Number, msg.Digest)
 	blockAnnounceHeaderHash := blockAnnounceHeader.Hash()
-	logger.Infof("received block announce from: %s, #%d (%s)", from,
+	logger.Debugf("received block announce from: %s, #%d (%s)", from,
 		blockAnnounceHeader.Number, blockAnnounceHeaderHash.Short())
 
 	// if the peer reports a lower or equal best block number than us,

dot/types/header.go

@@ -4,12 +4,15 @@
 package types
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/pkg/scale"
 )
 
+var ErrNoPreRuntimeDigest = errors.New("header does not contain pre-runtime digest")
+
 // Header is a state block header
 type Header struct {
 	ParentHash     common.Hash `json:"parentHash"`
@@ -96,3 +99,32 @@ func (bh *Header) Hash() common.Hash {
 
 	return bh.hash
 }
+
+func (bh *Header) SlotNumber() (uint64, error) {
+	for _, d := range bh.Digest {
+		digestValue, err := d.Value()
+		if err != nil {
+			continue
+		}
+		predigest, ok := digestValue.(PreRuntimeDigest)
+		if !ok {
+			continue
+		}
+
+		digest, err := DecodeBabePreDigest(predigest.Data)
+		if err != nil {
+			return 0, fmt.Errorf("failed to decode babe header: %w", err)
+		}
+
+		switch d := digest.(type) {
+		case BabePrimaryPreDigest:
+			return d.SlotNumber, nil
+		case BabeSecondaryVRFPreDigest:
+			return d.SlotNumber, nil
+		case BabeSecondaryPlainPreDigest:
+			return d.SlotNumber, nil
+		}
+	}
+
+	return 0, ErrNoPreRuntimeDigest
+}

go.mod

@@ -45,12 +45,12 @@ require (
 )
 
 require (
-	github.com/DataDog/zstd v1.5.2 // indirect
+	github.com/DataDog/zstd v1.4.5 // indirect
 	github.com/Jorropo/jsync v1.0.1 // indirect
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/errors v1.11.1 // indirect
@@ -65,9 +65,9 @@ require (
 	github.com/decred/base58 v1.0.5 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
-	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
+	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/flynn/noise v1.0.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
@@ -82,7 +82,7 @@ require (
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/glog v1.0.0 // indirect
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
@@ -91,7 +91,7 @@ require (
 	github.com/gtank/ristretto255 v0.1.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/uint256 v1.2.4 // indirect
 	github.com/huin/goupnp v1.3.0 // indirect
@@ -157,15 +157,14 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
-	github.com/quic-go/quic-go v0.38.2 // indirect
+	github.com/quic-go/quic-go v0.38.1 // indirect
 	github.com/quic-go/webtransport-go v0.5.3 // indirect
 	github.com/raulk/go-watchdog v1.3.0 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/rs/cors v1.8.2 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
-	github.com/smartystreets/assertions v1.13.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect