Merge pull request #127 from bro-n-bro/126-add-bostrom

126 add bostrom

.env

@@ -1,7 +1,7 @@
 # Application settings
 START_TIMEOUT=20s # Start application timeout duration
 STOP_TIMEOUT=20s # Stop application timeout duration
-MODULES=bank,core,auth,authz,distribution,gov,mint,staking,slashing,feegrant,ibc,liquidity # Modules for processing
+MODULES=auth,authz,bandwidth,bank,core,distribution,dmn,feegrant,gov,graph,grid,ibc,liquidity,mint,rank,slashing,staking,wasm # Modules for processing
 CHAIN_PREFIX=cosmos # Prefix of indexing chain
 DEFAULT_DEMON=uatom # Default demon of chain coins
 PARSE_AVATAR_URL=false # Parse avatar url for validator from keychain. It will decrease index performance!!!
@@ -14,6 +14,8 @@ METRICS_ENABLED=true
 RPC_URL=http://127.0.0.1:26657 # RPC API
 GRPC_URL=http://127.0.0.1:8090 # GRPC API
 GRPC_SECURE_CONNECTION=false # GRPC secure connection
+GRPC_TIMEOUT=15s # GRPC requests timeout
+RPC_TIMEOUT=15s # RPC requests timeout
 WS_ENABLED=true # Websocket enabled
 
 # Broker settings

.github/workflows/build_and_push.yml

@@ -18,15 +18,13 @@ jobs:
         DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
       run: |
         docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-    - name: Build specify tag
+    - name: Build and push specify tag
       run: |
         echo 'build image: bronbro/spacebox-crawler:${{github.ref_name}}'
         docker build -t bronbro/spacebox-crawler:${{github.ref_name}} --target=app --build-arg version=${{github.ref_name}} .
-    - name: Build latest tag
+        docker push bronbro/spacebox-crawler:${{github.ref_name}}
+    - name: Build and push latest tag
       if: startsWith(github.ref, 'refs/tags/v')
-      run: docker build -t bronbro/spacebox-crawler:latest --target=app --build-arg version=latest .
-    - name: push specify tag to registry
-      run: docker push bronbro/spacebox-crawler:${{github.ref_name}}
-    - name: push latest tag to registry
-      if: startsWith(github.ref, 'refs/tags/v')
-      run: docker push bronbro/spacebox-crawler:latest
+      run: |
+        docker build -t bronbro/spacebox-crawler:latest --target=app --build-arg version=latest .
+        docker push bronbro/spacebox-crawler:latest

.github/workflows/lint.yml

@@ -13,7 +13,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-go@v4
       with:
-        go-version: 1.20.5
+        go-version: 1.21.4
 
     - name: lint
       run: make lint

.gitignore

@@ -21,4 +21,6 @@
 vendor/
 .idea/
 volumes/
-.env.local
+.env.local
+tmp
+bin

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20.5-alpine as builder
+FROM golang:1.21.4-alpine as builder
 
 ARG version
 
@@ -8,10 +8,18 @@ RUN apk update && apk add --no-cache make git build-base musl-dev librdkafka lib
 WORKDIR /go/src/github.com/spacebox-crawler
 COPY . ./
 
+
+ADD https://github.com/CosmWasm/wasmvm/releases/download/v1.5.0/libwasmvm_muslc.x86_64.a /lib/libwasmvm_muslc.x86_64.a
+RUN sha256sum /lib/libwasmvm_muslc.x86_64.a | grep 465e3a088e96fd009a11bfd234c69fb8a0556967677e54511c084f815cf9ce63
+
+# Copy the library you want to the final location that will be found by the linker flag `-lwasmvm_muslc`
+RUN cp /lib/libwasmvm_muslc.x86_64.a /lib/libwasmvm_muslc.a
+
+
 RUN echo "build binary" && \
     export PATH=$PATH:/usr/local/go/bin && \
     go mod download && \
-    go build -ldflags="-X 'main.Version=$version'" -tags musl /go/src/github.com/spacebox-crawler/cmd/main.go && \
+    go build -ldflags="-X 'main.Version=$version' -linkmode=external -extldflags '-Wl,-z,muldefs -static'" -tags musl,muslc,netgo /go/src/github.com/spacebox-crawler/cmd/main.go && \
     mkdir -p /spacebox-crawler && \
     mv main /spacebox-crawler/main && \
     rm -Rf /usr/local/go/src

Makefile

@@ -24,7 +24,9 @@ race: dep ## Run data race detector
 	@go test -race ./... -count=1
 
 install-linter: ## Install golangci-lint
-	@go install github.com/golangci/golangci-lint/cmd/[email protected]
+	@mkdir -p bin
+	@curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b bin v1.55.2
+	@bin/golangci-lint --version
 
 lint: install-linter ## Lint the files
 	./scripts/golint.sh

adapter/storage/block.go

@@ -98,7 +98,7 @@ func (s *Storage) GetErrorBlockHeights(ctx context.Context) ([]int64, error) {
 	return res, nil
 }
 
-func (s *Storage) GetAllBlocks(ctx context.Context) ([]*model.Block, error) {
+func (s *Storage) GetAllBlocks(ctx context.Context) (blocks []*model.Block, err error) {
 	cursor, err := s.blocksCollection.Find(ctx, bson.D{})
 	if err != nil {
 		return nil, err
@@ -109,7 +109,6 @@ func (s *Storage) GetAllBlocks(ctx context.Context) ([]*model.Block, error) {
 		}
 	}()
 
-	blocks := make([]*model.Block, 0)
 	if err = cursor.All(ctx, &blocks); err != nil {
 		return nil, err
 	}

adapter/storage/model/tx.go

@@ -0,0 +1,10 @@
+package model
+
+import "time"
+
+type Tx struct {
+	Created      time.Time
+	ErrorMessage string `bson:"error_message"`
+	Hash         string `bson:"hash"`
+	Height       int64  `bson:"height"`
+}

adapter/storage/storage.go

@@ -15,6 +15,7 @@ type Storage struct {
 	cli                *mongo.Client
 	blocksCollection   *mongo.Collection
 	messagesCollection *mongo.Collection
+	txCollection       *mongo.Collection
 
 	cfg Config
 }
@@ -61,6 +62,7 @@ func (s *Storage) Start(ctx context.Context) error {
 	blocksCollection := s.cli.Database("spacebox").Collection("blocks")
 	s.blocksCollection = blocksCollection
 	s.messagesCollection = s.cli.Database("spacebox").Collection("error_messages")
+	s.txCollection = s.cli.Database("spacebox").Collection("error_txs")
 
 	mod := mongo.IndexModel{
 		Keys: bson.M{"height": 1}, // index in ascending order or -1 for descending order

adapter/storage/tx.go

@@ -0,0 +1,21 @@
+package storage
+
+import (
+	"context"
+
+	"go.mongodb.org/mongo-driver/bson"
+
+	"github.com/bro-n-bro/spacebox-crawler/adapter/storage/model"
+)
+
+func (s *Storage) InsertErrorTx(ctx context.Context, tx model.Tx) error {
+	if _, err := s.txCollection.InsertOne(ctx, tx); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Storage) CountErrorTxs(ctx context.Context) (int64, error) {
+	return s.txCollection.CountDocuments(ctx, bson.D{})
+}

client/grpc/client.go

@@ -17,53 +17,75 @@ import (
 	slashingtypes "github.com/cosmos/cosmos-sdk/x/slashing/types"
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
 	ibctransfertypes "github.com/cosmos/ibc-go/v7/modules/apps/transfer/types"
+	bandwidthtypes "github.com/cybercongress/go-cyber/x/bandwidth/types"
+	dmntypes "github.com/cybercongress/go-cyber/x/dmn/types"
+	graphtypes "github.com/cybercongress/go-cyber/x/graph/types"
+	gridtypes "github.com/cybercongress/go-cyber/x/grid/types"
+	ranktypes "github.com/cybercongress/go-cyber/x/rank/types"
+	resourcestypes "github.com/cybercongress/go-cyber/x/resources/types"
+	"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/timeout"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/rs/zerolog"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 
+	"github.com/bro-n-bro/spacebox-crawler/adapter/storage/model"
 	liquiditytypes "github.com/bro-n-bro/spacebox-crawler/types/liquidity"
 )
 
-type Client struct {
-	SlashingQueryClient     slashingtypes.QueryClient
-	TmsService              tmservice.ServiceClient
-	TxService               tx.ServiceClient
-	BankQueryClient         banktypes.QueryClient
-	AuthQueryClient         authtypes.QueryClient
-	GovQueryClient          govtypes.QueryClient
-	MintQueryClient         minttypes.QueryClient
-	StakingQueryClient      stakingtypes.QueryClient
-	DistributionQueryClient distributiontypes.QueryClient
-	AuthzQueryClient        authztypes.QueryClient
-	FeegrantQueryClient     feegranttypes.QueryClient
-	IbcTransferQueryClient  ibctransfertypes.QueryClient
-	LiquidityQueryClient    liquiditytypes.QueryClient
-	conn                    *grpc.ClientConn
-	log                     *zerolog.Logger
-	cfg                     Config
-}
+type (
+	storage interface {
+		InsertErrorTx(ctx context.Context, tx model.Tx) error
+	}
+
+	Client struct {
+		SlashingQueryClient     slashingtypes.QueryClient
+		TmsService              tmservice.ServiceClient
+		TxService               tx.ServiceClient
+		BankQueryClient         banktypes.QueryClient
+		AuthQueryClient         authtypes.QueryClient
+		GovQueryClient          govtypes.QueryClient
+		MintQueryClient         minttypes.QueryClient
+		StakingQueryClient      stakingtypes.QueryClient
+		DistributionQueryClient distributiontypes.QueryClient
+		AuthzQueryClient        authztypes.QueryClient
+		FeegrantQueryClient     feegranttypes.QueryClient
+		IbcTransferQueryClient  ibctransfertypes.QueryClient
+		LiquidityQueryClient    liquiditytypes.QueryClient
+		GraphQueryClient        graphtypes.QueryClient
+		BandwidthQueryClient    bandwidthtypes.QueryClient
+		DMNQueryClient          dmntypes.QueryClient
+		GridQueryClient         gridtypes.QueryClient
+		RankQueryClient         ranktypes.QueryClient
+		ResourcesQueryClient    resourcestypes.QueryClient
+		conn                    *grpc.ClientConn
+		log                     *zerolog.Logger
+		storage                 storage
+		cfg                     Config
+	}
+)
 
-func New(cfg Config, l zerolog.Logger) *Client {
+func New(cfg Config, l zerolog.Logger, st storage) *Client {
 	l = l.With().Str("cmp", "grpc-client").Logger()
 
-	return &Client{cfg: cfg, log: &l}
+	return &Client{cfg: cfg, log: &l, storage: st}
 }
 
 func (c *Client) Start(ctx context.Context) error {
-	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second) // dial timeout
 	defer cancel()
 
 	options := []grpc.DialOption{
 		grpc.WithBlock(),
+		grpc.WithChainUnaryInterceptor(timeout.UnaryClientInterceptor(c.cfg.Timeout)), // request timeout
 	}
 
 	if c.cfg.MetricsEnabled {
 		options = append(
 			options,
-			grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
-			grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor),
+			grpc.WithChainUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
+			grpc.WithChainStreamInterceptor(grpc_prometheus.StreamClientInterceptor),
 		)
 
 		grpc_prometheus.EnableClientHandlingTimeHistogram()
@@ -99,12 +121,16 @@ func (c *Client) Start(ctx context.Context) error {
 	c.IbcTransferQueryClient = ibctransfertypes.NewQueryClient(grpcConn)
 	c.LiquidityQueryClient = liquiditytypes.NewQueryClient(grpcConn)
 	c.AuthQueryClient = authtypes.NewQueryClient(grpcConn)
+	c.GraphQueryClient = graphtypes.NewQueryClient(grpcConn)
+	c.BandwidthQueryClient = bandwidthtypes.NewQueryClient(grpcConn)
+	c.DMNQueryClient = dmntypes.NewQueryClient(grpcConn)
+	c.GridQueryClient = gridtypes.NewQueryClient(grpcConn)
+	c.RankQueryClient = ranktypes.NewQueryClient(grpcConn)
+	c.ResourcesQueryClient = resourcestypes.NewQueryClient(grpcConn)
 
 	c.conn = grpcConn
 
 	return nil
 }
 
-func (c *Client) Stop(_ context.Context) error {
-	return c.conn.Close()
-}
+func (c *Client) Stop(_ context.Context) error { return c.conn.Close() }

client/grpc/config.go

@@ -1,7 +1,12 @@
 package grpc
 
-type Config struct {
-	Host             string `env:"GRPC_URL" envDefault:"http://localhost:9090"`
-	SecureConnection bool   `env:"GRPC_SECURE_CONNECTION" envDefault:"false"`
-	MetricsEnabled   bool   `env:"METRICS_ENABLED" envDefault:"false"`
-}
+import "time"
+
+type (
+	Config struct {
+		Host             string        `env:"GRPC_URL" envDefault:"http://localhost:9090"`
+		SecureConnection bool          `env:"GRPC_SECURE_CONNECTION" envDefault:"false"`
+		MetricsEnabled   bool          `env:"METRICS_ENABLED" envDefault:"false"`
+		Timeout          time.Duration `env:"GRPC_TIMEOUT" envDefault:"15s"`
+	}
+)

client/grpc/tx.go

@@ -3,9 +3,12 @@ package grpc
 import (
 	"context"
 	"encoding/hex"
+	"time"
 
 	cometbfttypes "github.com/cometbft/cometbft/types"
 	"github.com/cosmos/cosmos-sdk/types/tx"
+
+	"github.com/bro-n-bro/spacebox-crawler/adapter/storage/model"
 )
 
 // Txs queries for all the transactions in a block. Transactions are returned
@@ -15,9 +18,18 @@ func (c *Client) Txs(ctx context.Context, height int64, txs cometbfttypes.Txs) (
 	txResponses := make([]*tx.GetTxResponse, 0, len(txs))
 
 	for _, tmTx := range txs {
-		respPb, err := c.TxService.GetTx(ctx, &tx.GetTxRequest{Hash: hex.EncodeToString(tmTx.Hash())})
+		hash := hex.EncodeToString(tmTx.Hash())
+
+		respPb, err := c.TxService.GetTx(ctx, &tx.GetTxRequest{Hash: hash})
 		if err != nil {
-			c.log.Error().Err(err).Int64("height", height).Msg("GetTx error")
+			_ = c.storage.InsertErrorTx(ctx, model.Tx{
+				Created:      time.Now(),
+				ErrorMessage: err.Error(),
+				Hash:         hash,
+				Height:       height,
+			})
+
+			c.log.Warn().Err(err).Int64("height", height).Msg("GetTx error")
 			continue
 		}
 

client/rpc/blocker.go

@@ -8,6 +8,9 @@ import (
 
 // GetBlockEvents returns begin block and end block events.
 func (c *Client) GetBlockEvents(ctx context.Context, height int64) (begin, end types.BlockerEvents, err error) {
+	ctx, cancel := context.WithTimeout(ctx, c.cfg.Timeout)
+	defer cancel()
+
 	result, err := c.RPCClient.BlockResults(ctx, &height)
 	if err != nil {
 		return nil, nil, err