Merge branch 'development' into eclesio/sync-strategy

.github/workflows/checks.yml

@@ -40,7 +40,7 @@ jobs:
       - name: Up to date Go proto generated
         run: |
           go install google.golang.org/protobuf/cmd/[email protected] && \
-          PROTOC_VERSION=21.10 && \
+          PROTOC_VERSION=24.4 && \
           curl -sL --output protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" && \
           sudo unzip protoc.zip -d /usr/local && \
           rm protoc.zip && \

.github/workflows/dispatch.yml

@@ -0,0 +1,59 @@
+name: Dispatch workflow to deploy Gossamer nodes
+
+on:
+  workflow_dispatch:
+    inputs:
+      commit:
+        description: 'commit'
+        required: true
+      nodeType:
+        description: 'Node type'
+        required: true
+        default: 'genesis'
+        type: choice 
+        options:
+          - snapshot 
+          - genesis
+      cluster:
+        description: 'Cluster'
+        default: 'shared-gossamer'
+        type: choice 
+        options:
+          - shared-gossamer
+          - shared-gossamer-2
+        required: true
+      chain:
+        description: 'Chain'
+        required: true
+        default: 'westend'
+        type: choice
+        options:
+          - paseo
+          - westend
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get Token
+        id: get_workflow_token
+        uses: peter-murray/workflow-application-token-action@v3
+        with:
+          application_id: ${{ vars.APPLICATION_ID }}
+          application_private_key: ${{ secrets.APPLICATION_PRIVATE_KEY }}
+
+      - name: Dispatch Repository Event for ${{ github.event.inputs.chain }}
+        uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ steps.get_workflow_token.outputs.token }}
+          repository: Chainsafe/infrastructure-general
+          event-type: deploy-gossamer-${{ github.event.inputs.nodeType }}
+          client-payload: |
+            {
+              "ref": "refs/heads/main",
+              "inputs": {
+                "commit": "${{ github.event.inputs.commit }}",
+                "chain": "${{ github.event.inputs.chain }}",
+                "cluster": "${{ github.event.inputs.cluster }}"
+              }
+            }

.github/workflows/staging.yml

@@ -46,7 +46,7 @@ jobs:
           image: ${{ steps.build-image.outputs.image }}
 
       - name: Deploy Amazon ECS task definition
-        uses: aws-actions/amazon-ecs-deploy-task-definition@v1
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
         with:
           task-definition: ${{ steps.task-def.outputs.task-definition }}
           service: gossamer-${{ matrix.chain }}-svc

Dockerfile.development

@@ -1,4 +1,4 @@
-FROM golang:1.22
+FROM golang:1.23
 
 ARG chain="polkadot"
 ARG basepath="~/.local/share/gossamer"

Dockerfile.staging

@@ -1,4 +1,4 @@
-FROM golang:1.22
+FROM golang:1.23
 
 ARG chain="polkadot"
 ARG basepath="~/.local/share/gossamer"

README.md

@@ -159,6 +159,30 @@ Then start the three hosts:
 ./bin/gossamer --chain westend-local --charlie
 ```
 
+## Standalone Packages
+
+In addition to the core functionality provided by Gossamer, we have developed some standalone packages that can be used independently. These packages are located in the `pkg` directory and serve various utility purposes. Here’s an overview of some of these packages:
+
+### Scale
+
+To support parity's [scale type encoding](https://docs.substrate.io/reference/scale-codec/) in golang.  
+More details [here](./pkg/scale/README.md)
+
+### Trie
+
+The `trie` package includes multiple implementations of polkadot merkle tries. These implementations are useful for various applications requiring efficient data structures. Here are the key implementations:
+
+#### In-Memory Trie
+
+This implementation supports polkadot merkle tries that are stored entirely in memory. It includes features for backing up the trie using a database to ensure persistence.
+
+#### TrieDB
+
+An evolution of the in-memory trie, `TrieDB` introduces lazy loading capabilities from the underlying database. This approach optimizes performance and resource usage by loading data only as needed.
+
+For more details on how to use `TrieDB`, refer to the specific [README](pkg/trie/triedb/README.md)
+
+
 ## Contribute
 
 - Check out the [Contributing Guidelines](.github/CONTRIBUTING.md) and our

dot/core/service.go

@@ -280,7 +280,7 @@ func (s *Service) handleBlock(block *types.Block, state *rtstorage.TrieState) er
 	}
 
 	logger.Debugf("imported block %s and stored state trie with root %s",
-		block.Header.Hash(), state.MustRoot())
+		block.Header.Hash(), state.Trie().MustHash())
 
 	parentRuntimeInstance, err := s.blockState.GetRuntime(block.Header.ParentHash)
 	if err != nil {

dot/network/errors.go

@@ -10,19 +10,16 @@ import (
 var (
 	ErrNoPeersConnected     = errors.New("no peers connected")
 	ErrReceivedEmptyMessage = errors.New("received empty message")
-	ErrNilBlockInResponse   = errors.New("nil block in response")
 
-	errCannotValidateHandshake       = errors.New("failed to validate handshake")
-	errMessageTypeNotValid           = errors.New("message type is not valid")
-	errInvalidHandshakeForPeer       = errors.New("peer previously sent invalid handshake")
-	errHandshakeTimeout              = errors.New("handshake timeout reached")
-	errBlockRequestFromNumberInvalid = errors.New("block request message From number is not valid")
-	errInvalidStartingBlockType      = errors.New("invalid StartingBlock in messsage")
-	errInboundHanshakeExists         = errors.New("an inbound handshake already exists for given peer")
-	errInvalidRole                   = errors.New("invalid role")
-	ErrFailedToReadEntireMessage     = errors.New("failed to read entire message")
-	ErrNilStream                     = errors.New("nil stream")
-	ErrInvalidLEB128EncodedData      = errors.New("invalid LEB128 encoded data")
-	ErrGreaterThanMaxSize            = errors.New("greater than maximum size")
-	ErrStreamReset                   = errors.New("stream reset")
+	errCannotValidateHandshake   = errors.New("failed to validate handshake")
+	errMessageTypeNotValid       = errors.New("message type is not valid")
+	errInvalidHandshakeForPeer   = errors.New("peer previously sent invalid handshake")
+	errHandshakeTimeout          = errors.New("handshake timeout reached")
+	errInboundHanshakeExists     = errors.New("an inbound handshake already exists for given peer")
+	errInvalidRole               = errors.New("invalid role")
+	ErrFailedToReadEntireMessage = errors.New("failed to read entire message")
+	ErrNilStream                 = errors.New("nil stream")
+	ErrInvalidLEB128EncodedData  = errors.New("invalid LEB128 encoded data")
+	ErrGreaterThanMaxSize        = errors.New("greater than maximum size")
+	ErrStreamReset               = errors.New("stream reset")
 )

dot/network/helpers_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ChainSafe/gossamer/dot/network/messages"
 	"github.com/ChainSafe/gossamer/dot/types"
 	"github.com/ChainSafe/gossamer/internal/log"
 	"github.com/ChainSafe/gossamer/lib/common"
@@ -32,14 +33,14 @@ const (
 )
 
 type testStreamHandler struct {
-	messages map[peer.ID][]Message
+	messages map[peer.ID][]messages.P2PMessage
 	decoder  messageDecoder
 	exit     bool
 }
 
 func newTestStreamHandler(decoder messageDecoder) *testStreamHandler {
 	return &testStreamHandler{
-		messages: make(map[peer.ID][]Message),
+		messages: make(map[peer.ID][]messages.P2PMessage),
 		decoder:  decoder,
 	}
 }
@@ -55,7 +56,7 @@ func (s *testStreamHandler) handleStream(stream libp2pnetwork.Stream) {
 	s.readStream(stream, peer, s.decoder, s.handleMessage)
 }
 
-func (s *testStreamHandler) handleMessage(stream libp2pnetwork.Stream, msg Message) error {
+func (s *testStreamHandler) handleMessage(stream libp2pnetwork.Stream, msg messages.P2PMessage) error {
 	msgs := s.messages[stream.Conn().RemotePeer()]
 	s.messages[stream.Conn().RemotePeer()] = append(msgs, msg)
 
@@ -65,7 +66,7 @@ func (s *testStreamHandler) handleMessage(stream libp2pnetwork.Stream, msg Messa
 	return s.writeToStream(stream, announceHandshake)
 }
 
-func (s *testStreamHandler) writeToStream(stream libp2pnetwork.Stream, msg Message) error {
+func (s *testStreamHandler) writeToStream(stream libp2pnetwork.Stream, msg messages.P2PMessage) error {
 	encMsg, err := msg.Encode()
 	if err != nil {
 		return err
@@ -124,24 +125,26 @@ var starting, _ = variadic.NewUint32OrHash(uint32(1))
 
 var one = uint32(1)
 
-func newTestBlockRequestMessage(t *testing.T) *BlockRequestMessage {
+func newTestBlockRequestMessage(t *testing.T) *messages.BlockRequestMessage {
 	t.Helper()
 
-	return &BlockRequestMessage{
-		RequestedData: RequestedDataHeader + RequestedDataBody + RequestedDataJustification,
+	return &messages.BlockRequestMessage{
+		RequestedData: messages.RequestedDataHeader +
+			messages.RequestedDataBody +
+			messages.RequestedDataJustification,
 		StartingBlock: *starting,
 		Direction:     1,
 		Max:           &one,
 	}
 }
 
-func testBlockRequestMessageDecoder(in []byte, _ peer.ID, _ bool) (Message, error) {
-	msg := new(BlockRequestMessage)
+func testBlockRequestMessageDecoder(in []byte, _ peer.ID, _ bool) (messages.P2PMessage, error) {
+	msg := new(messages.BlockRequestMessage)
 	err := msg.Decode(in)
 	return msg, err
 }
 
-func testBlockAnnounceMessageDecoder(in []byte, _ peer.ID, _ bool) (Message, error) {
+func testBlockAnnounceMessageDecoder(in []byte, _ peer.ID, _ bool) (messages.P2PMessage, error) {
 	msg := BlockAnnounceMessage{
 		Number: 0,
 		Digest: types.NewDigest(),
@@ -150,7 +153,7 @@ func testBlockAnnounceMessageDecoder(in []byte, _ peer.ID, _ bool) (Message, err
 	return &msg, err
 }
 
-func testBlockAnnounceHandshakeDecoder(in []byte, _ peer.ID, _ bool) (Message, error) {
+func testBlockAnnounceHandshakeDecoder(in []byte, _ peer.ID, _ bool) (messages.P2PMessage, error) {
 	msg := new(BlockAnnounceHandshake)
 	err := msg.Decode(in)
 	return msg, err
@@ -280,11 +283,11 @@ func createTestService(t *testing.T, cfg *Config) (srvc *Service) {
 	return srvc
 }
 
-func newTestBlockResponseMessage(t *testing.T) *BlockResponseMessage {
+func newTestBlockResponseMessage(t *testing.T) *messages.BlockResponseMessage {
 	t.Helper()
 
 	const blockRequestSize = 128
-	msg := &BlockResponseMessage{
+	msg := &messages.BlockResponseMessage{
 		BlockData: make([]*types.BlockData, blockRequestSize),
 	}
 

dot/network/host.go

@@ -14,6 +14,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ChainSafe/gossamer/dot/network/messages"
 	"github.com/ChainSafe/gossamer/dot/peerset"
 	"github.com/ChainSafe/gossamer/internal/pubip"
 	"github.com/dgraph-io/ristretto"
@@ -330,7 +331,7 @@ func (h *host) bootstrap() {
 
 // send creates a new outbound stream with the given peer and writes the message. It also returns
 // the newly created stream.
-func (h *host) send(p peer.ID, pid protocol.ID, msg Message) (network.Stream, error) {
+func (h *host) send(p peer.ID, pid protocol.ID, msg messages.P2PMessage) (network.Stream, error) {
 	// open outbound stream with host protocol id
 	stream, err := h.p2pHost.NewStream(h.ctx, p, pid)
 	if err != nil {
@@ -354,7 +355,7 @@ func (h *host) send(p peer.ID, pid protocol.ID, msg Message) (network.Stream, er
 	return stream, nil
 }
 
-func (h *host) writeToStream(s network.Stream, msg Message) error {
+func (h *host) writeToStream(s network.Stream, msg messages.P2PMessage) error {
 	encMsg, err := msg.Encode()
 	if err != nil {
 		return err

dot/network/light.go

@@ -6,6 +6,7 @@ package network
 import (
 	"fmt"
 
+	"github.com/ChainSafe/gossamer/dot/network/messages"
 	"github.com/ChainSafe/gossamer/dot/types"
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/pkg/scale"
@@ -19,7 +20,7 @@ func (s *Service) handleLightStream(stream libp2pnetwork.Stream) {
 	s.readStream(stream, s.decodeLightMessage, s.handleLightMsg, MaxBlockResponseSize)
 }
 
-func (s *Service) decodeLightMessage(in []byte, peer peer.ID, _ bool) (Message, error) {
+func (s *Service) decodeLightMessage(in []byte, peer peer.ID, _ bool) (messages.P2PMessage, error) {
 	s.lightRequestMu.RLock()
 	defer s.lightRequestMu.RUnlock()
 
@@ -33,7 +34,7 @@ func (s *Service) decodeLightMessage(in []byte, peer peer.ID, _ bool) (Message,
 	return newLightRequestFromBytes(in)
 }
 
-func (s *Service) handleLightMsg(stream libp2pnetwork.Stream, msg Message) (err error) {
+func (s *Service) handleLightMsg(stream libp2pnetwork.Stream, msg messages.P2PMessage) (err error) {
 	defer func() {
 		err := stream.Close()
 		if err != nil && err.Error() != ErrStreamReset.Error() {