Fix: jailing without events. Add square size column (#154)

celenium-io · Mar 10, 2024 · 3b7a45a · 3b7a45a
1 parent 35e0555
commit 3b7a45a
Show file tree

Hide file tree

Showing 17 changed files with 203 additions and 80 deletions.
diff --git a/cmd/api/docs/docs.go b/cmd/api/docs/docs.go
diff --git a/cmd/api/docs/swagger.json b/cmd/api/docs/swagger.json
diff --git a/cmd/api/docs/swagger.yaml b/cmd/api/docs/swagger.yaml
diff --git a/cmd/api/handler/responses/block.go b/cmd/api/handler/responses/block.go
@@ -80,6 +80,7 @@ type BlockStats struct {
 	GasLimit       int64                   `example:"1234"                            json:"gas_limit"       swaggertype:"integer"`
 	GasUsed        int64                   `example:"1234"                            json:"gas_used"        swaggertype:"integer"`
 	BytesInBlock   int64                   `example:"1234"                            json:"bytes_in_block"  swaggertype:"integer"`
+	SquareSize     uint64                  `example:"16"                              json:"square_size"     swaggertype:"integer"`
 	MessagesCounts map[types.MsgType]int64 `example:"{MsgPayForBlobs:10,MsgUnjail:1}" json:"messages_counts" swaggertype:"string"`
 }
 
@@ -103,6 +104,7 @@ func NewBlockStats(stats storage.BlockStats) *BlockStats {
 		GasLimit:       stats.GasLimit,
 		GasUsed:        stats.GasUsed,
 		BytesInBlock:   stats.BytesInBlock,
+		SquareSize:     stats.SquareSize,
 		FillRate:       fmt.Sprintf("%.4f", float64(stats.BytesInBlock)/float64(maxSize)),
 	}
 }
diff --git a/internal/storage/block_stats.go b/internal/storage/block_stats.go
@@ -27,19 +27,20 @@ type BlockStats struct {
 	Height pkgTypes.Level `bun:"height"                    comment:"The number (height) of this block" stats:"func:min max,filterable"`
 	Time   time.Time      `bun:"time,pk,notnull"           comment:"The time of block"                 stats:"func:min max,filterable"`
 
-	TxCount       int64           `bun:"tx_count"                 comment:"Count of transactions in block"                          stats:"func:min max sum avg"`
-	EventsCount   int64           `bun:"events_count"             comment:"Count of events in begin and end of block"               stats:"func:min max sum avg"`
-	BlobsSize     int64           `bun:"blobs_size"               comment:"Summary blocks size from pay for blob"                   stats:"func:min max sum avg"`
-	BlobsCount    int             `bun:"blobs_count"              comment:"Summary blobs count in the block"                        stats:"func:min max sum avg"`
-	BlockTime     uint64          `bun:"block_time"               comment:"Time in milliseconds between current and previous block" stats:"func:min max sum avg"`
+	TxCount       int64           `bun:"tx_count"                 comment:"Count of transactions in block"                                    stats:"func:min max sum avg"`
+	EventsCount   int64           `bun:"events_count"             comment:"Count of events in begin and end of block"                         stats:"func:min max sum avg"`
+	BlobsSize     int64           `bun:"blobs_size"               comment:"Summary blocks size from pay for blob"                             stats:"func:min max sum avg"`
+	BlobsCount    int             `bun:"blobs_count"              comment:"Summary blobs count in the block"                                  stats:"func:min max sum avg"`
+	BlockTime     uint64          `bun:"block_time"               comment:"Time in milliseconds between current and previous block"           stats:"func:min max sum avg"`
 	GasLimit      int64           `bun:"gas_limit"                comment:"Total gas limit in the block"`
 	GasUsed       int64           `bun:"gas_used"                 comment:"Total gas used in the block"`
-	SupplyChange  decimal.Decimal `bun:",type:numeric"            comment:"Change of total supply in the block"                     stats:"func:min max sum avg"`
-	InflationRate decimal.Decimal `bun:",type:numeric"            comment:"Inflation rate"                                          stats:"func:min max avg"`
-	Fee           decimal.Decimal `bun:"fee,type:numeric"         comment:"Summary block fee"                                       stats:"func:min max sum avg"`
-	Rewards       decimal.Decimal `bun:"rewards,type:numeric"     comment:"Total rewards per block"                                 stats:"func:min max sum avg"`
-	Commissions   decimal.Decimal `bun:"commissions,type:numeric" comment:"Total commissions per block"                             stats:"func:min max sum avg"`
-	BytesInBlock  int64           `bun:"bytes_in_block"           comment:"Size of all transactions in bytes"                       stats:"func:min max sum avg"`
+	SupplyChange  decimal.Decimal `bun:",type:numeric"            comment:"Change of total supply in the block"                               stats:"func:min max sum avg"`
+	InflationRate decimal.Decimal `bun:",type:numeric"            comment:"Inflation rate"                                                    stats:"func:min max avg"`
+	Fee           decimal.Decimal `bun:"fee,type:numeric"         comment:"Summary block fee"                                                 stats:"func:min max sum avg"`
+	Rewards       decimal.Decimal `bun:"rewards,type:numeric"     comment:"Total rewards per block"                                           stats:"func:min max sum avg"`
+	Commissions   decimal.Decimal `bun:"commissions,type:numeric" comment:"Total commissions per block"                                       stats:"func:min max sum avg"`
+	BytesInBlock  int64           `bun:"bytes_in_block"           comment:"Size of all transactions in bytes"                                 stats:"func:min max sum avg"`
+	SquareSize    uint64          `bun:"square_size"              comment:"Size of the square after splitting all the block data into shares"`
 
 	MessagesCounts map[types.MsgType]int64 `bun:"-"`
 }

diff --git a/internal/storage/postgres/block.go b/internal/storage/postgres/block.go
@@ -54,7 +54,7 @@ func (b *Blocks) ByHeightWithStats(ctx context.Context, height types.Level) (blo
 
 	err = b.DB().NewSelect().
 		ColumnExpr("block.id, block.height, block.time, block.version_block, block.version_app, block.message_types, block.hash, block.parent_hash, block.last_commit_hash, block.data_hash, block.validators_hash, block.next_validators_hash, block.consensus_hash, block.app_hash, block.last_results_hash, block.evidence_hash, block.proposer_id").
-		ColumnExpr("stats.id AS stats__id, stats.height AS stats__height, stats.time AS stats__time, stats.tx_count AS stats__tx_count, stats.events_count AS stats__events_count, stats.blobs_size AS stats__blobs_size, stats.block_time AS stats__block_time, stats.gas_limit AS stats__gas_limit, stats.gas_used AS stats__gas_used, stats.supply_change AS stats__supply_change, stats.inflation_rate AS stats__inflation_rate, stats.fee AS stats__fee, stats.bytes_in_block AS stats__bytes_in_block, stats.blobs_count AS stats__blobs_count, stats.rewards AS stats__rewards, stats.commissions AS stats__commissions").
+		ColumnExpr("stats.id AS stats__id, stats.height AS stats__height, stats.time AS stats__time, stats.tx_count AS stats__tx_count, stats.events_count AS stats__events_count, stats.blobs_size AS stats__blobs_size, stats.block_time AS stats__block_time, stats.gas_limit AS stats__gas_limit, stats.gas_used AS stats__gas_used, stats.supply_change AS stats__supply_change, stats.inflation_rate AS stats__inflation_rate, stats.fee AS stats__fee, stats.bytes_in_block AS stats__bytes_in_block, stats.blobs_count AS stats__blobs_count, stats.rewards AS stats__rewards, stats.commissions AS stats__commissions, stats.square_size AS stats__square_size").
 		ColumnExpr("proposer.id AS proposer__id, proposer.cons_address AS proposer__cons_address, proposer.moniker AS proposer__moniker").
 		With("q", subQuery).
 		TableExpr("q as block").
@@ -97,7 +97,7 @@ func (b *Blocks) ByIdWithRelations(ctx context.Context, id uint64) (block storag
 
 	err = b.DB().NewSelect().
 		ColumnExpr("block.id, block.height, block.time, block.version_block, block.version_app, block.message_types, block.hash, block.parent_hash, block.last_commit_hash, block.data_hash, block.validators_hash, block.next_validators_hash, block.consensus_hash, block.app_hash, block.last_results_hash, block.evidence_hash, block.proposer_id").
-		ColumnExpr("stats.id AS stats__id, stats.height AS stats__height, stats.time AS stats__time, stats.tx_count AS stats__tx_count, stats.events_count AS stats__events_count, stats.blobs_size AS stats__blobs_size, stats.block_time AS stats__block_time, stats.gas_limit AS stats__gas_limit, stats.gas_used AS stats__gas_used, stats.supply_change AS stats__supply_change, stats.inflation_rate AS stats__inflation_rate, stats.fee AS stats__fee, stats.bytes_in_block AS stats__bytes_in_block, stats.blobs_count AS stats__blobs_count, stats.rewards AS stats__rewards, stats.commissions AS stats__commissions").
+		ColumnExpr("stats.id AS stats__id, stats.height AS stats__height, stats.time AS stats__time, stats.tx_count AS stats__tx_count, stats.events_count AS stats__events_count, stats.blobs_size AS stats__blobs_size, stats.block_time AS stats__block_time, stats.gas_limit AS stats__gas_limit, stats.gas_used AS stats__gas_used, stats.supply_change AS stats__supply_change, stats.inflation_rate AS stats__inflation_rate, stats.fee AS stats__fee, stats.bytes_in_block AS stats__bytes_in_block, stats.blobs_count AS stats__blobs_count, stats.rewards AS stats__rewards, stats.commissions AS stats__commissions, stats.square_size AS stats__square_size").
 		ColumnExpr("proposer.id AS proposer__id, proposer.cons_address AS proposer__cons_address, proposer.moniker AS proposer__moniker").
 		With("q", subQuery).
 		TableExpr("q as block").
@@ -152,7 +152,7 @@ func (b *Blocks) ByHash(ctx context.Context, hash []byte) (block storage.Block,
 
 	err = b.DB().NewSelect().
 		ColumnExpr("block.id, block.height, block.time, block.version_block, block.version_app, block.message_types, block.hash, block.parent_hash, block.last_commit_hash, block.data_hash, block.validators_hash, block.next_validators_hash, block.consensus_hash, block.app_hash, block.last_results_hash, block.evidence_hash, block.proposer_id").
-		ColumnExpr("stats.id AS stats__id, stats.height AS stats__height, stats.time AS stats__time, stats.tx_count AS stats__tx_count, stats.events_count AS stats__events_count, stats.blobs_size AS stats__blobs_size, stats.block_time AS stats__block_time, stats.gas_limit AS stats__gas_limit, stats.gas_used AS stats__gas_used, stats.supply_change AS stats__supply_change, stats.inflation_rate AS stats__inflation_rate, stats.fee AS stats__fee, stats.bytes_in_block AS stats__bytes_in_block, stats.blobs_count AS stats__blobs_count, stats.rewards AS stats__rewards, stats.commissions AS stats__commissions").
+		ColumnExpr("stats.id AS stats__id, stats.height AS stats__height, stats.time AS stats__time, stats.tx_count AS stats__tx_count, stats.events_count AS stats__events_count, stats.blobs_size AS stats__blobs_size, stats.block_time AS stats__block_time, stats.gas_limit AS stats__gas_limit, stats.gas_used AS stats__gas_used, stats.supply_change AS stats__supply_change, stats.inflation_rate AS stats__inflation_rate, stats.fee AS stats__fee, stats.bytes_in_block AS stats__bytes_in_block, stats.blobs_count AS stats__blobs_count, stats.rewards AS stats__rewards, stats.commissions AS stats__commissions, stats.square_size AS stats__square_size").
 		ColumnExpr("proposer.id AS proposer__id, proposer.cons_address AS proposer__cons_address, proposer.moniker AS proposer__moniker").
 		With("q", subQuery).
 		TableExpr("q as block").
@@ -179,7 +179,7 @@ func (b *Blocks) ListWithStats(ctx context.Context, limit, offset uint64, order
 		ColumnExpr("v.id AS proposer__id, v.cons_address as proposer__cons_address, v.moniker as proposer__moniker").
 		ColumnExpr("stats.id AS stats__id, stats.height AS stats__height, stats.time AS stats__time, stats.tx_count AS stats__tx_count, stats.events_count AS stats__events_count, stats.blobs_count as stats__blobs_count").
 		ColumnExpr("stats.blobs_size AS stats__blobs_size, stats.block_time AS stats__block_time, stats.bytes_in_block AS stats__bytes_in_block, stats.rewards AS stats__rewards, stats.commissions AS stats__commissions").
-		ColumnExpr("stats.supply_change AS stats__supply_change, stats.inflation_rate AS stats__inflation_rate, stats.fee AS stats__fee, stats.gas_used AS stats__gas_used, stats.gas_limit AS stats__gas_limit").
+		ColumnExpr("stats.supply_change AS stats__supply_change, stats.inflation_rate AS stats__inflation_rate, stats.fee AS stats__fee, stats.gas_used AS stats__gas_used, stats.gas_limit AS stats__gas_limit, stats.square_size AS stats__square_size").
 		TableExpr("(?) as block", subQuery).
 		Join("LEFT JOIN block_stats as stats ON stats.height = block.height").
 		Join("LEFT JOIN validator as v ON v.id = block.proposer_id")
@@ -232,17 +232,24 @@ func (b *Blocks) ListWithStats(ctx context.Context, limit, offset uint64, order
 }
 
 func (b *Blocks) ByProposer(ctx context.Context, proposerId uint64, limit, offset int) (blocks []storage.Block, err error) {
-	query := b.DB().NewSelect().Model(&blocks).
+	blocksQuery := b.DB().NewSelect().Model(&blocks).
 		Where("proposer_id = ?", proposerId).
-		Relation("Stats").
-		Order("id desc")
+		Order("time desc")
 
-	query = limitScope(query, limit)
+	blocksQuery = limitScope(blocksQuery, limit)
 	if offset > 0 {
-		query = query.Offset(offset)
+		blocksQuery = blocksQuery.Offset(offset)
 	}
 
-	err = query.Scan(ctx)
+	err = b.DB().NewSelect().
+		ColumnExpr("block.*").
+		ColumnExpr("stats.id AS stats__id, stats.height AS stats__height, stats.time AS stats__time, stats.tx_count AS stats__tx_count, stats.events_count AS stats__events_count, stats.blobs_count as stats__blobs_count").
+		ColumnExpr("stats.blobs_size AS stats__blobs_size, stats.block_time AS stats__block_time, stats.bytes_in_block AS stats__bytes_in_block, stats.rewards AS stats__rewards, stats.commissions AS stats__commissions").
+		ColumnExpr("stats.supply_change AS stats__supply_change, stats.inflation_rate AS stats__inflation_rate, stats.fee AS stats__fee, stats.gas_used AS stats__gas_used, stats.gas_limit AS stats__gas_limit, stats.square_size AS stats__square_size").
+		TableExpr("(?) as block", blocksQuery).
+		Join("LEFT JOIN block_stats as stats ON stats.height = block.height").
+		Order("time desc").
+		Scan(ctx, &blocks)
 	return
 }
 

diff --git a/pkg/indexer/decode/context/context.go b/pkg/indexer/decode/context/context.go
@@ -9,35 +9,32 @@ import (
 	"github.com/celenium-io/celestia-indexer/pkg/indexer/decode/decoder"
 	pkgTypes "github.com/celenium-io/celestia-indexer/pkg/types"
 	"github.com/dipdup-net/indexer-sdk/pkg/sync"
-	"github.com/shopspring/decimal"
 )
 
 type Context struct {
-	Validators       *sync.Map[string, *storage.Validator]
-	JailedValidators *sync.Map[string, *storage.Validator]
-	Addresses        *sync.Map[string, *storage.Address]
-	Delegations      *sync.Map[string, *storage.Delegation]
+	Validators  *sync.Map[string, *storage.Validator]
+	Addresses   *sync.Map[string, *storage.Address]
+	Delegations *sync.Map[string, *storage.Delegation]
+	Jails       *sync.Map[string, *storage.Jail]
 
 	Redelegations   []storage.Redelegation
 	Undelegations   []storage.Undelegation
 	CancelUnbonding []storage.Undelegation
-	Jails           []storage.Jail
 	StakingLogs     []storage.StakingLog
 
 	Block *storage.Block
 }
 
 func NewContext() *Context {
 	return &Context{
-		Validators:       sync.NewMap[string, *storage.Validator](),
-		Addresses:        sync.NewMap[string, *storage.Address](),
-		Delegations:      sync.NewMap[string, *storage.Delegation](),
-		JailedValidators: sync.NewMap[string, *storage.Validator](),
-		Redelegations:    make([]storage.Redelegation, 0),
-		Undelegations:    make([]storage.Undelegation, 0),
-		CancelUnbonding:  make([]storage.Undelegation, 0),
-		StakingLogs:      make([]storage.StakingLog, 0),
-		Jails:            make([]storage.Jail, 0),
+		Validators:      sync.NewMap[string, *storage.Validator](),
+		Addresses:       sync.NewMap[string, *storage.Address](),
+		Delegations:     sync.NewMap[string, *storage.Delegation](),
+		Jails:           sync.NewMap[string, *storage.Jail](),
+		Redelegations:   make([]storage.Redelegation, 0),
+		Undelegations:   make([]storage.Undelegation, 0),
+		CancelUnbonding: make([]storage.Undelegation, 0),
+		StakingLogs:     make([]storage.StakingLog, 0),
 	}
 }
 
@@ -155,17 +152,21 @@ func (ctx *Context) AddCancelUndelegation(u storage.Undelegation) {
 	ctx.CancelUnbonding = append(ctx.CancelUnbonding, u)
 }
 
-func (ctx *Context) AddJailedValidator(address string, burned decimal.Decimal) {
-	jailed := true
-	ctx.JailedValidators.Set(address, &storage.Validator{
-		ConsAddress: address,
-		Stake:       burned.Neg(),
-		Jailed:      &jailed,
-	})
-}
-
-func (ctx *Context) AddJail(j storage.Jail) {
-	ctx.Jails = append(ctx.Jails, j)
+func (ctx *Context) AddJail(jail storage.Jail) {
+	if j, ok := ctx.Jails.Get(jail.Validator.ConsAddress); ok {
+		if jail.Reason != "" {
+			j.Reason = jail.Reason
+		}
+		if !jail.Burned.IsZero() {
+			j.Validator.Stake = j.Validator.Stake.Sub(jail.Burned)
+			j.Burned = j.Burned.Add(jail.Burned)
+		}
+		if jail.Validator.Jailed != nil {
+			j.Validator.Jailed = jail.Validator.Jailed
+		}
+	} else {
+		ctx.Jails.Set(jail.Validator.ConsAddress, &jail)
+	}
 }
 
 func (ctx *Context) AddStakingLog(l storage.StakingLog) {

diff --git a/pkg/indexer/parser/event_handlers.go b/pkg/indexer/parser/event_handlers.go
@@ -186,15 +186,17 @@ func parseSlash(ctx *context.Context, data map[string]any) error {
 			return err
 		}
 		consAddress := strings.ToUpper(hex.EncodeToString(hash))
-		ctx.AddJailedValidator(consAddress, slash.BurnedCoins.Copy())
 
+		jailed := true
 		ctx.AddJail(storage.Jail{
 			Height: ctx.Block.Height,
 			Time:   ctx.Block.Time,
 			Reason: slash.Reason,
 			Burned: slash.BurnedCoins,
 			Validator: &storage.Validator{
 				ConsAddress: consAddress,
+				Stake:       slash.BurnedCoins.Copy(),
+				Jailed:      &jailed,
 			},
 		})
 	}

diff --git a/pkg/indexer/parser/parse.go b/pkg/indexer/parser/parse.go
@@ -59,9 +59,12 @@ func (p *Module) parse(b types.BlockData) error {
 			InflationRate: decimal.Zero,
 			Commissions:   decimal.Zero,
 			Rewards:       decimal.Zero,
+			SquareSize:    b.Block.Data.SquareSize,
 		},
 	}
 
+	parseValidatorUpdates(decodeCtx, b.ValidatorUpdates)
+
 	txs, err := parseTxs(decodeCtx, b)
 	if err != nil {
 		return errors.Wrapf(err, "while parsing block on level=%d", b.Height)

diff --git a/pkg/indexer/parser/validator_updates.go b/pkg/indexer/parser/validator_updates.go
@@ -0,0 +1,37 @@
+// SPDX-FileCopyrightText: 2024 PK Lab AG <[email protected]>
+// SPDX-License-Identifier: MIT
+
+package parser
+
+import (
+	"encoding/hex"
+	"strings"
+
+	"github.com/celenium-io/celestia-indexer/internal/storage"
+	"github.com/celenium-io/celestia-indexer/pkg/indexer/decode/context"
+	"github.com/celenium-io/celestia-indexer/pkg/types"
+	"github.com/shopspring/decimal"
+)
+
+func parseValidatorUpdates(ctx *context.Context, updates []types.ValidatorUpdate) {
+	for i := range updates {
+		if updates[i].Power != nil {
+			continue
+		}
+		key := updates[i].PubKey.Sum.Value.Ed25519
+		consAddressBytes := types.GetConsAddressBytesFromPubKey(key)
+		consAddress := strings.ToUpper(hex.EncodeToString(consAddressBytes))
+
+		jailed := true
+		ctx.AddJail(storage.Jail{
+			Height: ctx.Block.Height,
+			Time:   ctx.Block.Time,
+			Burned: decimal.Zero,
+			Validator: &storage.Validator{
+				ConsAddress: consAddress,
+				Stake:       decimal.Zero,
+				Jailed:      &jailed,
+			},
+		})
+	}
+}
diff --git a/pkg/indexer/storage/storage.go b/pkg/indexer/storage/storage.go
@@ -312,7 +312,7 @@ func (module *Module) processBlockInTransaction(ctx context.Context, tx storage.
 		return state, err
 	}
 
-	totalValidators, err := module.saveValidators(ctx, tx, dCtx.GetValidators(), dCtx.JailedValidators, dCtx.Jails)
+	totalValidators, err := module.saveValidators(ctx, tx, dCtx.GetValidators(), dCtx.Jails)
 	if err != nil {
 		return state, err
 	}