From f471d950665743220cd0b309e27e4ee556f1870f Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Fri, 8 Nov 2024 16:01:55 +0000 Subject: [PATCH] Add LogIndex and TxIndex into logs/event response body (#862) * Thor client (#818) * feat: add thorclient * refactor: remove roundTripper * refactor: change null check * clean: remove commented code * feat: add account revision and pending tx * fix: add licence headers and fix linter issue * refactor: rename package * refactor: change revision type to string * refactor: rename GetLogs and GetTransfers to FilterEvents and FilterTransfers * refactor: change FilterEvents and FilterTransactions request type to EventFilter * Adding common.EventWrapper to handle channel errors * tweak * update rawclient + update account tests * tidy up names * update tests * pr comments * adding raw tx * Tidy up method names and calls * options client * tweaks * pr comments * Update thorclient/common/common.go Co-authored-by: libotony * pr comments * Adding Subscriptions * Pr comments * adjust func orders * pr comments * changing subscribe to use the channel close vs multiple channels * adding go-doc * no error after unsubscribe * pr comments * checking status code is 2xx * fix: change FilterTransfers argument --------- Co-authored-by: otherview Co-authored-by: libotony * Show all issues on lint (#869) * Show all issues on lint * fix lint * fix(docker): using AWS docker repo for trivy (#872) * fix(docker): using AWS docker repo for trivy * fix(docker): using AWS docker repo for trivy * Darren/feat/add subscription cache (#866) * ehancement: create a cache for block based subscriptions * minor: change function names for subscriptions * test: add unit test for message cache * chore: add license headers * refactor: fix up error handling * fix: remove bad test * fix: PR comments * fix: PR comments - remove block cache * refactor(subscriptions): store structs in cache, not bytes * fix(license): add license header * chore(subscriptions): revert unit test changes * enhancement: resolve pr comments to use simplelru * enhancement: resolve pr comments - use id as key * Add additional block tests (#863) * enhancement(logging): leverage trace level (#873) * Add testchain package (#844) * Refactor thor node * thorchain allows insertion of blocks * remove thorNode, added testchain * clean up + comments * adding license headers * adding templating tests for thorclient * Remove test event hacks * remove types * removed chain_builder + added logdb to testchain * pr comments * Update test/testchain/chain.go Co-authored-by: libotony --------- Co-authored-by: libotony * chore(docs): update spec for validator nodes (#875) * chore(docs): update spec for validator nodes * chore(docs): update cores * chore(docs): remove public node stuff * Darren/logdb remove leading zeros (#865) * feat: add new txIndex column to event meta response * test: add convert event test * feat: make txLog and txIndex as optional return params * chore: update swagger with new event optional data * feat: save logIndex in sequence * feat: tweaked bits in sequence * refactor: rename optional log meta field * refactor: comments, yaml and txIndex counts * rebase to master * fix: remove stale struct * add txIndex to returned logdb query * reset to 0 eventCount and transferCount each receipt and write blockId only once * fix lint * rephrase logIndex description in yaml file * refactor: use filter.Option instead of eventFilter.Option * move includeIndexes to api --------- Co-authored-by: otherview Co-authored-by: libotony Co-authored-by: Darren Kelly <107671032+darrenvechain@users.noreply.github.com> Co-authored-by: Makis Christou --- api/doc/thor.yaml | 38 +++++++++++++++++- api/events/events.go | 9 +++-- api/events/events_test.go | 52 +++++++++++++++++++++++- api/events/types.go | 63 +++++++++++++---------------- api/events/types_test.go | 71 ++++++++++++++++++++++++++------- api/transfers/transfers.go | 16 +++++--- api/transfers/transfers_test.go | 53 +++++++++++++++++++++++- api/transfers/types.go | 15 +++++-- cmd/thor/sync_logdb.go | 12 +++++- logdb/logdb.go | 33 ++++++++------- logdb/logdb_test.go | 6 ++- logdb/sequence.go | 41 ++++++++++++++----- logdb/sequence_test.go | 36 +++++++++++------ logdb/types.go | 6 ++- thor/params.go | 5 +++ 15 files changed, 348 insertions(+), 108 deletions(-) diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml index 732a5f1a3..2a0d30b9e 100644 --- a/api/doc/thor.yaml +++ b/api/doc/thor.yaml @@ -1325,6 +1325,16 @@ components: description: The index of the clause in the transaction, from which the log was generated. example: 0 nullable: false + txIndex: + description: The index of the transaction in the block, from which the log was generated. + type: integer + nullable: true + example: 1 + logIndex: + description: The index of the log in the receipt's outputs. This is an overall index among all clauses. + type: integer + nullable: true + example: 1 Block: title: Block @@ -1855,6 +1865,11 @@ components: The limit of records to be included in the output. Use this parameter for pagination. Default's to all results. + includeIndexes: + type: boolean + example: true + nullable: true + description: Include both transaction and log index in the response. description: | Include these parameters to receive filtered results in a paged format. @@ -1865,7 +1880,8 @@ components: { "options": { "offset": 0, - "limit": 10 + "limit": 10, + "includeIndexes": true } } ``` @@ -1916,6 +1932,26 @@ components: } ``` This refers to the range from block 10 to block 1000. + + EventOptionalData: + nullable: true + type: object + title: EventOptionalData + properties: + txIndex: + type: boolean + example: true + nullable: true + description: | + Specifies whether to include in the response the event transaction index. + loglIndex: + type: boolean + example: true + nullable: true + description: | + Specifies whether to include in the response the event log index. + description: | + Specifies all the optional data that can be included in the response. EventCriteria: type: object diff --git a/api/events/events.go b/api/events/events.go index 40dff7b09..b4c93fadc 100644 --- a/api/events/events.go +++ b/api/events/events.go @@ -44,7 +44,7 @@ func (e *Events) filter(ctx context.Context, ef *EventFilter) ([]*FilteredEvent, } fes := make([]*FilteredEvent, len(events)) for i, e := range events { - fes[i] = convertEvent(e) + fes[i] = convertEvent(e, ef.Options.IncludeIndexes) } return fes, nil } @@ -60,9 +60,10 @@ func (e *Events) handleFilter(w http.ResponseWriter, req *http.Request) error { if filter.Options == nil { // if filter.Options is nil, set to the default limit +1 // to detect whether there are more logs than the default limit - filter.Options = &logdb.Options{ - Offset: 0, - Limit: e.limit + 1, + filter.Options = &Options{ + Offset: 0, + Limit: e.limit + 1, + IncludeIndexes: false, } } diff --git a/api/events/events_test.go b/api/events/events_test.go index b1268d378..89aafd36f 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -56,6 +56,56 @@ func TestEvents(t *testing.T) { testEventWithBlocks(t, blocksToInsert) } +func TestOptionalIndexes(t *testing.T) { + thorChain := initEventServer(t, defaultLogLimit) + defer ts.Close() + insertBlocks(t, thorChain.LogDB(), 5) + tclient = thorclient.New(ts.URL) + + testCases := []struct { + name string + includeIndexes bool + expected *uint32 + }{ + { + name: "do not include indexes", + includeIndexes: false, + expected: nil, + }, + { + name: "include indexes", + includeIndexes: true, + expected: new(uint32), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + filter := events.EventFilter{ + CriteriaSet: make([]*events.EventCriteria, 0), + Range: nil, + Options: &events.Options{Limit: 6, IncludeIndexes: tc.includeIndexes}, + Order: logdb.DESC, + } + + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/event", filter) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, statusCode) + var tLogs []*events.FilteredEvent + if err := json.Unmarshal(res, &tLogs); err != nil { + t.Fatal(err) + } + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, 5, len(tLogs)) + + for _, tLog := range tLogs { + assert.Equal(t, tc.expected, tLog.Meta.TxIndex) + assert.Equal(t, tc.expected, tLog.Meta.LogIndex) + } + }) + } +} + func TestOption(t *testing.T) { thorChain := initEventServer(t, 5) defer ts.Close() @@ -65,7 +115,7 @@ func TestOption(t *testing.T) { filter := events.EventFilter{ CriteriaSet: make([]*events.EventCriteria, 0), Range: nil, - Options: &logdb.Options{Limit: 6}, + Options: &events.Options{Limit: 6}, Order: logdb.DESC, } diff --git a/api/events/types.go b/api/events/types.go index 0dce06aa4..575f8d855 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -6,7 +6,6 @@ package events import ( - "fmt" "math" "github.com/ethereum/go-ethereum/common/hexutil" @@ -23,6 +22,8 @@ type LogMeta struct { TxID thor.Bytes32 `json:"txID"` TxOrigin thor.Address `json:"txOrigin"` ClauseIndex uint32 `json:"clauseIndex"` + TxIndex *uint32 `json:"txIndex,omitempty"` + LogIndex *uint32 `json:"logIndex,omitempty"` } type TopicSet struct { @@ -42,8 +43,8 @@ type FilteredEvent struct { } // convert a logdb.Event into a json format Event -func convertEvent(event *logdb.Event) *FilteredEvent { - fe := FilteredEvent{ +func convertEvent(event *logdb.Event, addIndexes bool) *FilteredEvent { + fe := &FilteredEvent{ Address: event.Address, Data: hexutil.Encode(event.Data), Meta: LogMeta{ @@ -55,38 +56,19 @@ func convertEvent(event *logdb.Event) *FilteredEvent { ClauseIndex: event.ClauseIndex, }, } + + if addIndexes { + fe.Meta.TxIndex = &event.TxIndex + fe.Meta.LogIndex = &event.LogIndex + } + fe.Topics = make([]*thor.Bytes32, 0) for i := 0; i < 5; i++ { if event.Topics[i] != nil { fe.Topics = append(fe.Topics, event.Topics[i]) } } - return &fe -} - -func (e *FilteredEvent) String() string { - return fmt.Sprintf(` - Event( - address: %v, - topics: %v, - data: %v, - meta: (blockID %v, - blockNumber %v, - blockTimestamp %v), - txID %v, - txOrigin %v, - clauseIndex %v) - )`, - e.Address, - e.Topics, - e.Data, - e.Meta.BlockID, - e.Meta.BlockNumber, - e.Meta.BlockTimestamp, - e.Meta.TxID, - e.Meta.TxOrigin, - e.Meta.ClauseIndex, - ) + return fe } type EventCriteria struct { @@ -94,11 +76,17 @@ type EventCriteria struct { TopicSet } +type Options struct { + Offset uint64 + Limit uint64 + IncludeIndexes bool +} + type EventFilter struct { - CriteriaSet []*EventCriteria `json:"criteriaSet"` - Range *Range `json:"range"` - Options *logdb.Options `json:"options"` - Order logdb.Order `json:"order"` + CriteriaSet []*EventCriteria + Range *Range + Options *Options + Order logdb.Order // default asc } func convertEventFilter(chain *chain.Chain, filter *EventFilter) (*logdb.EventFilter, error) { @@ -107,9 +95,12 @@ func convertEventFilter(chain *chain.Chain, filter *EventFilter) (*logdb.EventFi return nil, err } f := &logdb.EventFilter{ - Range: rng, - Options: filter.Options, - Order: filter.Order, + Range: rng, + Options: &logdb.Options{ + Offset: filter.Options.Offset, + Limit: filter.Options.Limit, + }, + Order: filter.Order, } if len(filter.CriteriaSet) > 0 { f.CriteriaSet = make([]*logdb.EventCriteria, len(filter.CriteriaSet)) diff --git a/api/events/types_test.go b/api/events/types_test.go index a02f441c5..75eafe3a7 100644 --- a/api/events/types_test.go +++ b/api/events/types_test.go @@ -3,19 +3,20 @@ // Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying // file LICENSE or -package events_test +package events import ( "math" "testing" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/stretchr/testify/assert" - "github.com/vechain/thor/v2/api/events" "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/genesis" "github.com/vechain/thor/v2/logdb" "github.com/vechain/thor/v2/muxdb" "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/thor" ) func TestEventsTypes(t *testing.T) { @@ -33,13 +34,13 @@ func TestEventsTypes(t *testing.T) { } func testConvertRangeWithBlockRangeType(t *testing.T, chain *chain.Chain) { - rng := &events.Range{ - Unit: events.BlockRangeType, + rng := &Range{ + Unit: BlockRangeType, From: 1, To: 2, } - convertedRng, err := events.ConvertRange(chain, rng) + convertedRng, err := ConvertRange(chain, rng) assert.NoError(t, err) assert.Equal(t, uint32(rng.From), convertedRng.From) @@ -47,8 +48,8 @@ func testConvertRangeWithBlockRangeType(t *testing.T, chain *chain.Chain) { } func testConvertRangeWithTimeRangeTypeLessThenGenesis(t *testing.T, chain *chain.Chain) { - rng := &events.Range{ - Unit: events.TimeRangeType, + rng := &Range{ + Unit: TimeRangeType, From: 1, To: 2, } @@ -57,7 +58,7 @@ func testConvertRangeWithTimeRangeTypeLessThenGenesis(t *testing.T, chain *chain To: math.MaxUint32, } - convRng, err := events.ConvertRange(chain, rng) + convRng, err := ConvertRange(chain, rng) assert.NoError(t, err) assert.Equal(t, expectedEmptyRange, convRng) @@ -68,8 +69,8 @@ func testConvertRangeWithTimeRangeType(t *testing.T, chain *chain.Chain) { if err != nil { t.Fatal(err) } - rng := &events.Range{ - Unit: events.TimeRangeType, + rng := &Range{ + Unit: TimeRangeType, From: 1, To: genesis.Timestamp(), } @@ -78,7 +79,7 @@ func testConvertRangeWithTimeRangeType(t *testing.T, chain *chain.Chain) { To: 0, } - convRng, err := events.ConvertRange(chain, rng) + convRng, err := ConvertRange(chain, rng) assert.NoError(t, err) assert.Equal(t, expectedZeroRange, convRng) @@ -89,8 +90,8 @@ func testConvertRangeWithFromGreaterThanGenesis(t *testing.T, chain *chain.Chain if err != nil { t.Fatal(err) } - rng := &events.Range{ - Unit: events.TimeRangeType, + rng := &Range{ + Unit: TimeRangeType, From: genesis.Timestamp() + 1_000, To: genesis.Timestamp() + 10_000, } @@ -99,7 +100,7 @@ func testConvertRangeWithFromGreaterThanGenesis(t *testing.T, chain *chain.Chain To: math.MaxUint32, } - convRng, err := events.ConvertRange(chain, rng) + convRng, err := ConvertRange(chain, rng) assert.NoError(t, err) assert.Equal(t, expectedEmptyRange, convRng) @@ -123,3 +124,45 @@ func initChain(t *testing.T) *chain.Chain { return repo.NewBestChain() } + +func TestConvertEvent(t *testing.T) { + event := &logdb.Event{ + Address: thor.Address{0x01}, + Data: []byte{0x02, 0x03}, + BlockID: thor.Bytes32{0x04}, + BlockNumber: 5, + BlockTime: 6, + TxID: thor.Bytes32{0x07}, + TxIndex: 8, + LogIndex: 9, + TxOrigin: thor.Address{0x0A}, + ClauseIndex: 10, + Topics: [5]*thor.Bytes32{ + {0x0B}, + {0x0C}, + nil, + nil, + nil, + }, + } + + expectedTopics := []*thor.Bytes32{ + {0x0B}, + {0x0C}, + } + expectedData := hexutil.Encode(event.Data) + + result := convertEvent(event, true) + + assert.Equal(t, event.Address, result.Address) + assert.Equal(t, expectedData, result.Data) + assert.Equal(t, event.BlockID, result.Meta.BlockID) + assert.Equal(t, event.BlockNumber, result.Meta.BlockNumber) + assert.Equal(t, event.BlockTime, result.Meta.BlockTimestamp) + assert.Equal(t, event.TxID, result.Meta.TxID) + assert.Equal(t, event.TxIndex, *result.Meta.TxIndex) + assert.Equal(t, event.LogIndex, *result.Meta.LogIndex) + assert.Equal(t, event.TxOrigin, result.Meta.TxOrigin) + assert.Equal(t, event.ClauseIndex, result.Meta.ClauseIndex) + assert.Equal(t, expectedTopics, result.Topics) +} diff --git a/api/transfers/transfers.go b/api/transfers/transfers.go index cad4ee6b3..2a6cbfb9e 100644 --- a/api/transfers/transfers.go +++ b/api/transfers/transfers.go @@ -42,15 +42,18 @@ func (t *Transfers) filter(ctx context.Context, filter *TransferFilter) ([]*Filt transfers, err := t.db.FilterTransfers(ctx, &logdb.TransferFilter{ CriteriaSet: filter.CriteriaSet, Range: rng, - Options: filter.Options, - Order: filter.Order, + Options: &logdb.Options{ + Offset: filter.Options.Offset, + Limit: filter.Options.Limit, + }, + Order: filter.Order, }) if err != nil { return nil, err } tLogs := make([]*FilteredTransfer, len(transfers)) for i, trans := range transfers { - tLogs[i] = convertTransfer(trans) + tLogs[i] = convertTransfer(trans, filter.Options.IncludeIndexes) } return tLogs, nil } @@ -66,9 +69,10 @@ func (t *Transfers) handleFilterTransferLogs(w http.ResponseWriter, req *http.Re if filter.Options == nil { // if filter.Options is nil, set to the default limit +1 // to detect whether there are more logs than the default limit - filter.Options = &logdb.Options{ - Offset: 0, - Limit: t.limit + 1, + filter.Options = &events.Options{ + Offset: 0, + Limit: t.limit + 1, + IncludeIndexes: false, } } diff --git a/api/transfers/transfers_test.go b/api/transfers/transfers_test.go index 04a8c7b42..eb028414f 100644 --- a/api/transfers/transfers_test.go +++ b/api/transfers/transfers_test.go @@ -65,7 +65,7 @@ func TestOption(t *testing.T) { filter := transfers.TransferFilter{ CriteriaSet: make([]*logdb.TransferCriteria, 0), Range: nil, - Options: &logdb.Options{Limit: 6}, + Options: &events.Options{Limit: 6}, Order: logdb.DESC, } @@ -100,6 +100,57 @@ func TestOption(t *testing.T) { assert.Equal(t, "the number of filtered logs exceeds the maximum allowed value of 5, please use pagination", strings.Trim(string(res), "\n")) } +func TestOptionalData(t *testing.T) { + db := createDb(t) + initTransferServer(t, db, defaultLogLimit) + defer ts.Close() + insertBlocks(t, db, 5) + tclient = thorclient.New(ts.URL) + + testCases := []struct { + name string + includeIndexes bool + expected *uint32 + }{ + { + name: "do not include indexes", + includeIndexes: false, + expected: nil, + }, + { + name: "include indexes", + includeIndexes: true, + expected: new(uint32), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + filter := transfers.TransferFilter{ + CriteriaSet: make([]*logdb.TransferCriteria, 0), + Range: nil, + Options: &events.Options{Limit: 5, IncludeIndexes: tc.includeIndexes}, + Order: logdb.DESC, + } + + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/transfers", filter) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, statusCode) + var tLogs []*transfers.FilteredTransfer + if err := json.Unmarshal(res, &tLogs); err != nil { + t.Fatal(err) + } + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, 5, len(tLogs)) + + for _, tLog := range tLogs { + assert.Equal(t, tc.expected, tLog.Meta.TxIndex) + assert.Equal(t, tc.expected, tLog.Meta.LogIndex) + } + }) + } +} + // Test functions func testTransferBadRequest(t *testing.T) { badBody := []byte{0x00, 0x01, 0x02} diff --git a/api/transfers/types.go b/api/transfers/types.go index 29ad9b328..1574acf5a 100644 --- a/api/transfers/types.go +++ b/api/transfers/types.go @@ -19,6 +19,8 @@ type LogMeta struct { TxID thor.Bytes32 `json:"txID"` TxOrigin thor.Address `json:"txOrigin"` ClauseIndex uint32 `json:"clauseIndex"` + TxIndex *uint32 `json:"txIndex,omitempty"` + LogIndex *uint32 `json:"logIndex,omitempty"` } type FilteredTransfer struct { @@ -28,9 +30,9 @@ type FilteredTransfer struct { Meta LogMeta `json:"meta"` } -func convertTransfer(transfer *logdb.Transfer) *FilteredTransfer { +func convertTransfer(transfer *logdb.Transfer, addIndexes bool) *FilteredTransfer { v := math.HexOrDecimal256(*transfer.Amount) - return &FilteredTransfer{ + ft := &FilteredTransfer{ Sender: transfer.Sender, Recipient: transfer.Recipient, Amount: &v, @@ -43,11 +45,18 @@ func convertTransfer(transfer *logdb.Transfer) *FilteredTransfer { ClauseIndex: transfer.ClauseIndex, }, } + + if addIndexes { + ft.Meta.TxIndex = &transfer.TxIndex + ft.Meta.LogIndex = &transfer.LogIndex + } + + return ft } type TransferFilter struct { CriteriaSet []*logdb.TransferCriteria Range *events.Range - Options *logdb.Options + Options *events.Options Order logdb.Order //default asc } diff --git a/cmd/thor/sync_logdb.go b/cmd/thor/sync_logdb.go index 9fccf3127..edfb78793 100644 --- a/cmd/thor/sync_logdb.go +++ b/cmd/thor/sync_logdb.go @@ -285,6 +285,8 @@ func verifyLogDBPerBlock( n := block.Header().Number() id := block.Header().ID() ts := block.Header().Timestamp() + evCount := 0 + trCount := 0 var expectedEvLogs []*logdb.Event var expectedTrLogs []*logdb.Transfer @@ -292,6 +294,8 @@ func verifyLogDBPerBlock( for txIndex, r := range receipts { tx := txs[txIndex] origin, _ := tx.Origin() + evCount = 0 + trCount = 0 for clauseIndex, output := range r.Outputs { for _, ev := range output.Events { @@ -301,7 +305,7 @@ func verifyLogDBPerBlock( } expectedEvLogs = append(expectedEvLogs, &logdb.Event{ BlockNumber: n, - Index: uint32(len(expectedEvLogs)), + LogIndex: uint32(evCount), BlockID: id, BlockTime: ts, TxID: tx.ID(), @@ -310,12 +314,14 @@ func verifyLogDBPerBlock( Address: ev.Address, Topics: convertTopics(ev.Topics), Data: data, + TxIndex: uint32(txIndex), }) + evCount++ } for _, tr := range output.Transfers { expectedTrLogs = append(expectedTrLogs, &logdb.Transfer{ BlockNumber: n, - Index: uint32(len(expectedTrLogs)), + LogIndex: uint32(trCount), BlockID: id, BlockTime: ts, TxID: tx.ID(), @@ -324,7 +330,9 @@ func verifyLogDBPerBlock( Sender: tr.Sender, Recipient: tr.Recipient, Amount: tr.Amount, + TxIndex: uint32(txIndex), }) + trCount++ } } } diff --git a/logdb/logdb.go b/logdb/logdb.go index b1979813f..f172ebf1d 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -9,7 +9,6 @@ import ( "context" "database/sql" "fmt" - "math" "math/big" sqlite3 "github.com/mattn/go-sqlite3" @@ -118,10 +117,10 @@ FROM (%v) e if filter.Range != nil { subQuery += " AND seq >= ?" - args = append(args, newSequence(filter.Range.From, 0)) + args = append(args, newSequence(filter.Range.From, 0, 0)) if filter.Range.To >= filter.Range.From { subQuery += " AND seq <= ?" - args = append(args, newSequence(filter.Range.To, uint32(math.MaxInt32))) + args = append(args, newSequence(filter.Range.To, txIndexMask, logIndexMask)) } } @@ -184,10 +183,10 @@ FROM (%v) t if filter.Range != nil { subQuery += " AND seq >= ?" - args = append(args, newSequence(filter.Range.From, 0)) + args = append(args, newSequence(filter.Range.From, 0, 0)) if filter.Range.To >= filter.Range.From { subQuery += " AND seq <= ?" - args = append(args, newSequence(filter.Range.To, uint32(math.MaxInt32))) + args = append(args, newSequence(filter.Range.To, txIndexMask, logIndexMask)) } } @@ -272,10 +271,11 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac } event := &Event{ BlockNumber: seq.BlockNumber(), - Index: seq.Index(), + LogIndex: seq.LogIndex(), BlockID: thor.BytesToBytes32(blockID), BlockTime: blockTime, TxID: thor.BytesToBytes32(txID), + TxIndex: seq.TxIndex(), TxOrigin: thor.BytesToAddress(txOrigin), ClauseIndex: clauseIndex, Address: thor.BytesToAddress(address), @@ -334,10 +334,11 @@ func (db *LogDB) queryTransfers(ctx context.Context, query string, args ...inter } trans := &Transfer{ BlockNumber: seq.BlockNumber(), - Index: seq.Index(), + LogIndex: seq.LogIndex(), BlockID: thor.BytesToBytes32(blockID), BlockTime: blockTime, TxID: thor.BytesToBytes32(txID), + TxIndex: seq.TxIndex(), TxOrigin: thor.BytesToAddress(txOrigin), ClauseIndex: clauseIndex, Sender: thor.BytesToAddress(sender), @@ -376,7 +377,7 @@ func (db *LogDB) HasBlockID(id thor.Bytes32) (bool, error) { UNION SELECT * FROM (SELECT seq FROM event WHERE seq=? AND blockID=` + refIDQuery + ` LIMIT 1))` - seq := newSequence(block.Number(id), 0) + seq := newSequence(block.Number(id), 0, 0) row := db.stmtCache.MustPrepare(query).QueryRow(seq, id[:], seq, id[:]) var count int if err := row.Scan(&count); err != nil { @@ -426,7 +427,7 @@ type Writer struct { // Truncate truncates the database by deleting logs after blockNum (included). func (w *Writer) Truncate(blockNum uint32) error { - seq := newSequence(blockNum, 0) + seq := newSequence(blockNum, 0, 0) if err := w.exec("DELETE FROM event WHERE seq >= ?", seq); err != nil { return err } @@ -443,8 +444,6 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { blockNum = b.Header().Number() blockTimestamp = b.Header().Timestamp() txs = b.Transactions() - eventCount, - transferCount uint32 isReceiptEmpty = func(r *tx.Receipt) bool { for _, o := range r.Outputs { if len(o.Events) > 0 || len(o.Transfers) > 0 { @@ -453,20 +452,24 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { } return true } + blockIDInserted bool ) for i, r := range receipts { + eventCount, transferCount := uint32(0), uint32(0) + if isReceiptEmpty(r) { continue } - if eventCount == 0 && transferCount == 0 { + if !blockIDInserted { // block id is not yet inserted if err := w.exec( "INSERT OR IGNORE INTO ref(data) VALUES(?)", blockID[:]); err != nil { return err } + blockIDInserted = true } var ( @@ -478,6 +481,8 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { txID = tx.ID() txOrigin, _ = tx.Origin() } + + txIndex := i if err := w.exec( "INSERT OR IGNORE INTO ref(data) VALUES(?),(?)", txID[:], txOrigin[:]); err != nil { @@ -517,7 +522,7 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { if err := w.exec( query, - newSequence(blockNum, eventCount), + newSequence(blockNum, uint32(txIndex), eventCount), blockTimestamp, clauseIndex, eventData, @@ -552,7 +557,7 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { if err := w.exec( query, - newSequence(blockNum, transferCount), + newSequence(blockNum, uint32(txIndex), transferCount), blockTimestamp, clauseIndex, tr.Amount.Bytes(), diff --git a/logdb/logdb_test.go b/logdb/logdb_test.go index fc7c6af56..454d3a1e8 100644 --- a/logdb/logdb_test.go +++ b/logdb/logdb_test.go @@ -146,7 +146,8 @@ func TestEvents(t *testing.T) { origin, _ := tx.Origin() allEvents = append(allEvents, &Event{ BlockNumber: b.Header().Number(), - Index: uint32(j), + LogIndex: uint32(0), + TxIndex: uint32(j), BlockID: b.Header().ID(), BlockTime: b.Header().Timestamp(), TxID: tx.ID(), @@ -159,7 +160,8 @@ func TestEvents(t *testing.T) { allTransfers = append(allTransfers, &Transfer{ BlockNumber: b.Header().Number(), - Index: uint32(j), + LogIndex: uint32(0), + TxIndex: uint32(j), BlockID: b.Header().ID(), BlockTime: b.Header().Timestamp(), TxID: tx.ID(), diff --git a/logdb/sequence.go b/logdb/sequence.go index 52909ffe4..b76ad4821 100644 --- a/logdb/sequence.go +++ b/logdb/sequence.go @@ -5,21 +5,44 @@ package logdb -import "math" - type sequence int64 -func newSequence(blockNum uint32, index uint32) sequence { - if (index & math.MaxInt32) != index { - panic("index too large") +// Adjust these constants based on your bit allocation requirements +const ( + blockNumBits = 28 + txIndexBits = 15 + logIndexBits = 21 + // Max = 2^28 - 1 = 268,435,455 + blockNumMask = (1 << blockNumBits) - 1 + // Max = 2^15 - 1 = 32,767 + txIndexMask = (1 << txIndexBits) - 1 + // Max = 2^21 - 1 = 2,097,151 + logIndexMask = (1 << logIndexBits) - 1 +) + +func newSequence(blockNum uint32, txIndex uint32, logIndex uint32) sequence { + if blockNum > blockNumMask { + panic("block number too large") + } + if txIndex > txIndexMask { + panic("transaction index too large") } - return (sequence(blockNum) << 31) | sequence(index) + if logIndex > logIndexMask { + panic("log index too large") + } + return (sequence(blockNum) << (txIndexBits + logIndexBits)) | + (sequence(txIndex) << logIndexBits) | + sequence(logIndex) } func (s sequence) BlockNumber() uint32 { - return uint32(s >> 31) + return uint32(s>>(txIndexBits+logIndexBits)) & blockNumMask +} + +func (s sequence) TxIndex() uint32 { + return uint32((s >> logIndexBits) & txIndexMask) } -func (s sequence) Index() uint32 { - return uint32(s & math.MaxInt32) +func (s sequence) LogIndex() uint32 { + return uint32(s & logIndexMask) } diff --git a/logdb/sequence_test.go b/logdb/sequence_test.go index 9fa19fff0..b16e2d0da 100644 --- a/logdb/sequence_test.go +++ b/logdb/sequence_test.go @@ -6,33 +6,36 @@ package logdb import ( - "math" "testing" ) func TestSequence(t *testing.T) { type args struct { blockNum uint32 - index uint32 + txIndex uint32 + logIndex uint32 } tests := []struct { name string args args - want args }{ - {"regular", args{1, 2}, args{1, 2}}, - {"max bn", args{math.MaxUint32, 1}, args{math.MaxUint32, 1}}, - {"max index", args{5, math.MaxInt32}, args{5, math.MaxInt32}}, - {"both max", args{math.MaxUint32, math.MaxInt32}, args{math.MaxUint32, math.MaxInt32}}, + {"regular", args{1, 2, 3}}, + {"max bn", args{blockNumMask, 1, 2}}, + {"max tx index", args{5, txIndexMask, 4}}, + {"max log index", args{5, 4, logIndexMask}}, + {"both max", args{blockNumMask, txIndexMask, logIndexMask}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := newSequence(tt.args.blockNum, tt.args.index) - if bn := got.BlockNumber(); bn != tt.want.blockNum { - t.Errorf("seq.blockNum() = %v, want %v", bn, tt.want.blockNum) + got := newSequence(tt.args.blockNum, tt.args.txIndex, tt.args.logIndex) + if bn := got.BlockNumber(); bn != tt.args.blockNum { + t.Errorf("seq.blockNum() = %v, want %v", bn, tt.args.blockNum) } - if i := got.Index(); i != tt.want.index { - t.Errorf("seq.index() = %v, want %v", i, tt.want.index) + if ti := got.TxIndex(); ti != tt.args.txIndex { + t.Errorf("seq.txIndex() = %v, want %v", ti, tt.args.txIndex) + } + if i := got.LogIndex(); i != tt.args.logIndex { + t.Errorf("seq.index() = %v, want %v", i, tt.args.logIndex) } }) } @@ -42,5 +45,12 @@ func TestSequence(t *testing.T) { t.Errorf("newSequence should panic on 2nd arg > math.MaxInt32") } }() - newSequence(1, math.MaxInt32+1) + newSequence(1, txIndexMask+1, 5) + + defer func() { + if e := recover(); e == nil { + t.Errorf("newSequence should panic on 3rd arg > math.MaxInt32") + } + }() + newSequence(1, 5, logIndexMask+1) } diff --git a/logdb/types.go b/logdb/types.go index 7aa5ce990..8e772cc0c 100644 --- a/logdb/types.go +++ b/logdb/types.go @@ -15,10 +15,11 @@ import ( // Event represents tx.Event that can be stored in db. type Event struct { BlockNumber uint32 - Index uint32 + LogIndex uint32 BlockID thor.Bytes32 BlockTime uint64 TxID thor.Bytes32 + TxIndex uint32 TxOrigin thor.Address //contract caller ClauseIndex uint32 Address thor.Address // always a contract address @@ -29,10 +30,11 @@ type Event struct { // Transfer represents tx.Transfer that can be stored in db. type Transfer struct { BlockNumber uint32 - Index uint32 + LogIndex uint32 BlockID thor.Bytes32 BlockTime uint64 TxID thor.Bytes32 + TxIndex uint32 TxOrigin thor.Address ClauseIndex uint32 Sender thor.Address diff --git a/thor/params.go b/thor/params.go index 5912c46c9..3ec8462f8 100644 --- a/thor/params.go +++ b/thor/params.go @@ -12,6 +12,11 @@ import ( "github.com/ethereum/go-ethereum/params" ) +/* + NOTE: any changes to gas limit or block interval may affect how the txIndex and blockNumber are stored in logdb/sequence.go: + - an increase in gas limit may require more bits for txIndex; + - if block frequency is increased, blockNumber will increment faster, potentially exhausting the allocated bits sooner than expected. +*/ // Constants of block chain. const ( BlockInterval uint64 = 10 // time interval between two consecutive blocks.