From cbcbd0eb6763b57ea2ac8a6f8b8d6b79ebb0aaf5 Mon Sep 17 00:00:00 2001 From: Artem Poltorzhitskiy Date: Mon, 17 Jun 2024 11:49:09 +0200 Subject: [PATCH] Feature: add commitment filter to blob request (#216) --- cmd/api/docs/docs.go | 18 +++++ cmd/api/docs/swagger.json | 18 +++++ cmd/api/docs/swagger.yaml | 12 +++ cmd/api/handler/block.go | 16 +++- cmd/api/handler/block_test.go | 11 ++- cmd/api/handler/namespace.go | 70 ++++++++++++----- cmd/api/handler/namespace_test.go | 55 +++++++++++++ cmd/api/routes_test.go | 3 + internal/storage/blob_log.go | 11 ++- internal/storage/postgres/blob_log_test.go | 90 ++++++++++++++++++++++ internal/storage/postgres/scopes.go | 12 ++- 11 files changed, 288 insertions(+), 28 deletions(-) diff --git a/cmd/api/docs/docs.go b/cmd/api/docs/docs.go index 794a17c6..54669b3c 100644 --- a/cmd/api/docs/docs.go +++ b/cmd/api/docs/docs.go @@ -2359,6 +2359,24 @@ const docTemplate = `{ "description": "Sort field. If it's empty internal id is used", "name": "sort_by", "in": "query" + }, + { + "type": "string", + "description": "Commitment value in URLbase64 format", + "name": "commitment", + "in": "query" + }, + { + "type": "integer", + "description": "Time from in unix timestamp", + "name": "from", + "in": "query" + }, + { + "type": "integer", + "description": "Time to in unix timestamp", + "name": "to", + "in": "query" } ], "responses": { diff --git a/cmd/api/docs/swagger.json b/cmd/api/docs/swagger.json index 9574aeb2..2e921b7c 100644 --- a/cmd/api/docs/swagger.json +++ b/cmd/api/docs/swagger.json @@ -2349,6 +2349,24 @@ "description": "Sort field. If it's empty internal id is used", "name": "sort_by", "in": "query" + }, + { + "type": "string", + "description": "Commitment value in URLbase64 format", + "name": "commitment", + "in": "query" + }, + { + "type": "integer", + "description": "Time from in unix timestamp", + "name": "from", + "in": "query" + }, + { + "type": "integer", + "description": "Time to in unix timestamp", + "name": "to", + "in": "query" } ], "responses": { diff --git a/cmd/api/docs/swagger.yaml b/cmd/api/docs/swagger.yaml index 5c907d44..7dd88a58 100644 --- a/cmd/api/docs/swagger.yaml +++ b/cmd/api/docs/swagger.yaml @@ -3184,6 +3184,18 @@ paths: in: query name: sort_by type: string + - description: Commitment value in URLbase64 format + in: query + name: commitment + type: string + - description: Time from in unix timestamp + in: query + name: from + type: integer + - description: Time to in unix timestamp + in: query + name: to + type: integer produces: - application/json responses: diff --git a/cmd/api/handler/block.go b/cmd/api/handler/block.go index 93cb7845..db56b958 100644 --- a/cmd/api/handler/block.go +++ b/cmd/api/handler/block.go @@ -5,6 +5,7 @@ package handler import ( "net/http" + "time" "github.com/celenium-io/celestia-indexer/pkg/node" "github.com/celenium-io/celestia-indexer/pkg/types" @@ -404,6 +405,14 @@ func (handler *BlockHandler) Blobs(c echo.Context) error { } req.SetDefault() + blockTime, err := handler.block.Time(c.Request().Context(), req.Height) + if err != nil { + if handler.block.IsNoRows(err) { + return returnArray(c, []any{}) + } + return handleError(c, err, handler.block) + } + blobs, err := handler.blobLogs.ByHeight( c.Request().Context(), req.Height, @@ -412,6 +421,9 @@ func (handler *BlockHandler) Blobs(c echo.Context) error { Offset: int(req.Offset), Sort: pgSort(req.Sort), SortBy: req.SortBy, + // using time filters to take certain partition + From: blockTime, + To: blockTime.Add(time.Minute), }, ) if err != nil { @@ -469,12 +481,12 @@ func (handler *BlockHandler) BlockODS(c echo.Context) error { return badRequestError(c, err) } - b, err := handler.block.ByHeightWithStats(c.Request().Context(), req.Height) + blockStats, err := handler.blockStats.ByHeight(c.Request().Context(), req.Height) if err != nil { return handleError(c, err, handler.block) } - if b.Stats.TxCount == 0 { + if blockStats.TxCount == 0 { return c.JSON(http.StatusOK, responses.ODS{ Width: 0, Items: make([]responses.ODSItem, 0), diff --git a/cmd/api/handler/block_test.go b/cmd/api/handler/block_test.go index 5b6535ed..43b3e5e2 100644 --- a/cmd/api/handler/block_test.go +++ b/cmd/api/handler/block_test.go @@ -444,6 +444,11 @@ func (s *BlockTestSuite) TestBlobs() { c.SetParamNames("height") c.SetParamValues("100") + s.blocks.EXPECT(). + Time(gomock.Any(), pkgTypes.Level(100)). + Return(testBlock.Time, nil). + Times(1) + s.blobLogs.EXPECT(). ByHeight(gomock.Any(), pkgTypes.Level(100), gomock.Any()). Return([]storage.BlobLog{ @@ -590,9 +595,9 @@ func (s *BlockTestSuite) TestBlockODS() { c.SetParamNames("height") c.SetParamValues("100") - s.blocks.EXPECT(). - ByHeightWithStats(gomock.Any(), pkgTypes.Level(100)). - Return(testBlockWithStats, nil). + s.blockStats.EXPECT(). + ByHeight(gomock.Any(), pkgTypes.Level(100)). + Return(testBlockStats, nil). Times(1) rawTxs := []string{ diff --git a/cmd/api/handler/namespace.go b/cmd/api/handler/namespace.go index 2e21e738..16695fef 100644 --- a/cmd/api/handler/namespace.go +++ b/cmd/api/handler/namespace.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "encoding/hex" "net/http" + "time" "github.com/celenium-io/celestia-indexer/pkg/types" sdk "github.com/dipdup-net/indexer-sdk/pkg/storage" @@ -419,12 +420,27 @@ func (handler *NamespaceHandler) BlobMetadata(c echo.Context) error { } type getBlobLogsForNamespace struct { - Id string `param:"id" validate:"required,hexadecimal,len=56"` - Version byte `param:"version"` - Limit uint64 `query:"limit" validate:"omitempty,min=1,max=100"` - Offset uint64 `query:"offset" validate:"omitempty,min=0"` - Sort string `query:"sort" validate:"omitempty,oneof=asc desc"` - SortBy string `query:"sort_by" validate:"omitempty,oneof=time size"` + Id string `param:"id" validate:"required,hexadecimal,len=56"` + Version byte `param:"version"` + Limit uint64 `query:"limit" validate:"omitempty,min=1,max=100"` + Offset uint64 `query:"offset" validate:"omitempty,min=0"` + Sort string `query:"sort" validate:"omitempty,oneof=asc desc"` + SortBy string `query:"sort_by" validate:"omitempty,oneof=time size"` + Commitment string `query:"commitment" validate:"omitempty,base64url"` + + From int64 `example:"1692892095" query:"from" swaggertype:"integer" validate:"omitempty,min=1"` + To int64 `example:"1692892095" query:"to" swaggertype:"integer" validate:"omitempty,min=1"` +} + +func (req getBlobLogsForNamespace) getCommitment() (string, error) { + if req.Commitment == "" { + return "", nil + } + data, err := base64.URLEncoding.DecodeString(req.Commitment) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(data), nil } func (req *getBlobLogsForNamespace) SetDefault() { @@ -442,12 +458,15 @@ func (req *getBlobLogsForNamespace) SetDefault() { // @Description Returns blob changes for namespace // @Tags namespace // @ID get-blob-logs -// @Param id path string true "Namespace id in hexadecimal" minlength(56) maxlength(56) -// @Param version path integer true "Version of namespace" -// @Param limit query integer false "Count of requested entities" mininum(1) maximum(100) -// @Param offset query integer false "Offset" mininum(1) -// @Param sort query string false "Sort order. Default: desc" Enums(asc, desc) -// @Param sort_by query string false "Sort field. If it's empty internal id is used" Enums(time, size) +// @Param id path string true "Namespace id in hexadecimal" minlength(56) maxlength(56) +// @Param version path integer true "Version of namespace" +// @Param limit query integer false "Count of requested entities" mininum(1) maximum(100) +// @Param offset query integer false "Offset" mininum(1) +// @Param sort query string false "Sort order. Default: desc" Enums(asc, desc) +// @Param sort_by query string false "Sort field. If it's empty internal id is used" Enums(time, size) +// @Param commitment query string false "Commitment value in URLbase64 format" +// @Param from query integer false "Time from in unix timestamp" mininum(1) +// @Param to query integer false "Time to in unix timestamp" mininum(1) // @Produce json // @Success 200 {array} responses.BlobLog // @Failure 400 {object} Error @@ -460,6 +479,11 @@ func (handler *NamespaceHandler) GetBlobLogs(c echo.Context) error { } req.SetDefault() + cm, err := req.getCommitment() + if err != nil { + return badRequestError(c, err) + } + namespaceId, err := hex.DecodeString(req.Id) if err != nil { return badRequestError(c, err) @@ -470,15 +494,25 @@ func (handler *NamespaceHandler) GetBlobLogs(c echo.Context) error { return handleError(c, err, handler.namespace) } + fltrs := storage.BlobLogFilters{ + Limit: int(req.Limit), + Offset: int(req.Offset), + Sort: pgSort(req.Sort), + SortBy: req.SortBy, + Commitment: cm, + } + + if req.From > 0 { + fltrs.From = time.Unix(req.From, 0).UTC() + } + if req.To > 0 { + fltrs.To = time.Unix(req.To, 0).UTC() + } + logs, err := handler.blobLogs.ByNamespace( c.Request().Context(), ns.Id, - storage.BlobLogFilters{ - Limit: int(req.Limit), - Offset: int(req.Offset), - Sort: pgSort(req.Sort), - SortBy: req.SortBy, - }, + fltrs, ) if err != nil { return handleError(c, err, handler.namespace) diff --git a/cmd/api/handler/namespace_test.go b/cmd/api/handler/namespace_test.go index 57975d53..f48a8f48 100644 --- a/cmd/api/handler/namespace_test.go +++ b/cmd/api/handler/namespace_test.go @@ -531,6 +531,61 @@ func (s *NamespaceTestSuite) TestGetLogs() { s.Require().Nil(l.Namespace) } +func (s *NamespaceTestSuite) TestGetLogsWithCommitment() { + cm := "T1EPYi3jq6hC3ueLOZRtWB7LUsAC4DcnAX_oSwDopps=" + args := make(url.Values) + args.Set("commitment", cm) + + req := httptest.NewRequest(http.MethodGet, "/?"+args.Encode(), nil) + rec := httptest.NewRecorder() + c := s.echo.NewContext(req, rec) + c.SetPath("/namespace/:id/:version/logs") + c.SetParamNames("id", "version") + c.SetParamValues(testNamespaceId, "0") + + s.namespaces.EXPECT(). + ByNamespaceIdAndVersion(gomock.Any(), testNamespace.NamespaceID, byte(0)). + Return(testNamespace, nil) + + s.blobLogs.EXPECT(). + ByNamespace(gomock.Any(), testNamespace.Id, storage.BlobLogFilters{ + Limit: 10, + Sort: "desc", + Commitment: "T1EPYi3jq6hC3ueLOZRtWB7LUsAC4DcnAX/oSwDopps=", + }). + Return([]storage.BlobLog{ + { + NamespaceId: testNamespace.Id, + MsgId: 1, + TxId: 1, + SignerId: 1, + Signer: &storage.Address{ + Address: testAddress, + }, + Commitment: "T1EPYi3jq6hC3ueLOZRtWB7LUsAC4DcnAX/oSwDopps=", + Size: 1000, + Height: 10000, + Time: testTime, + }, + }, nil) + + s.Require().NoError(s.handler.GetBlobLogs(c)) + s.Require().Equal(http.StatusOK, rec.Code, rec.Body.String()) + + var logs []responses.BlobLog + err := json.NewDecoder(rec.Body).Decode(&logs) + s.Require().NoError(err) + s.Require().Len(logs, 1) + + l := logs[0] + s.Require().EqualValues(10000, l.Height) + s.Require().Equal(testTime, l.Time) + s.Require().Equal(testAddress, l.Signer) + s.Require().Equal("T1EPYi3jq6hC3ueLOZRtWB7LUsAC4DcnAX/oSwDopps=", l.Commitment) + s.Require().EqualValues(1000, l.Size) + s.Require().Nil(l.Namespace) +} + func (s *NamespaceTestSuite) TestRollups() { req := httptest.NewRequest(http.MethodGet, "/", nil) rec := httptest.NewRecorder() diff --git a/cmd/api/routes_test.go b/cmd/api/routes_test.go index 3ab52bb5..88feb25f 100644 --- a/cmd/api/routes_test.go +++ b/cmd/api/routes_test.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2024 PK Lab AG +// SPDX-License-Identifier: MIT + package main import ( diff --git a/internal/storage/blob_log.go b/internal/storage/blob_log.go index 336ace33..d9d1661e 100644 --- a/internal/storage/blob_log.go +++ b/internal/storage/blob_log.go @@ -16,10 +16,13 @@ import ( ) type BlobLogFilters struct { - Limit int - Offset int - Sort sdk.SortOrder - SortBy string + Limit int + Offset int + Sort sdk.SortOrder + SortBy string + From time.Time + To time.Time + Commitment string } //go:generate mockgen -source=$GOFILE -destination=mock/$GOFILE -package=mock -typed diff --git a/internal/storage/postgres/blob_log_test.go b/internal/storage/postgres/blob_log_test.go index b12bea53..d63e65e9 100644 --- a/internal/storage/postgres/blob_log_test.go +++ b/internal/storage/postgres/blob_log_test.go @@ -44,6 +44,96 @@ func (s *StorageTestSuite) TestBlobLogsByNamespace() { s.Require().EqualValues(4, log.Tx.Id) } +func (s *StorageTestSuite) TestBlobLogsByNamespaceAndCommitment() { + ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer ctxCancel() + + logs, err := s.storage.BlobLogs.ByNamespace(ctx, 2, storage.BlobLogFilters{ + Limit: 2, + Offset: 0, + Sort: sdk.SortOrderAsc, + SortBy: "size", + Commitment: "0CsLX630cjij9DR6nqoWfQcCH2pCQSoSuq63dTkd4Bw=", + }) + s.Require().NoError(err) + s.Require().Len(logs, 1) + + log := logs[0] + s.Require().EqualValues(3, log.Id) + s.Require().EqualValues(1000, log.Height) + s.Require().EqualValues("0CsLX630cjij9DR6nqoWfQcCH2pCQSoSuq63dTkd4Bw=", log.Commitment) + s.Require().EqualValues(12, log.Size) + s.Require().EqualValues(2, log.NamespaceId) + s.Require().EqualValues(2, log.SignerId) + s.Require().EqualValues(1, log.MsgId) + s.Require().EqualValues(2, log.TxId) + + s.Require().NotNil(log.Signer) + s.Require().EqualValues("celestia1jc92qdnty48pafummfr8ava2tjtuhfdw774w60", log.Signer.Address) + + s.Require().NotNil(log.Tx) +} + +func (s *StorageTestSuite) TestBlobLogsByNamespaceAndTime() { + ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer ctxCancel() + + logs, err := s.storage.BlobLogs.ByNamespace(ctx, 2, storage.BlobLogFilters{ + Limit: 1, + Offset: 0, + Sort: sdk.SortOrderAsc, + SortBy: "time", + From: time.Date(2023, 7, 3, 11, 0, 0, 0, time.UTC), + }) + s.Require().NoError(err) + s.Require().Len(logs, 1) + + log := logs[0] + s.Require().EqualValues(3, log.Id) + s.Require().EqualValues(1000, log.Height) + s.Require().EqualValues("0CsLX630cjij9DR6nqoWfQcCH2pCQSoSuq63dTkd4Bw=", log.Commitment) + s.Require().EqualValues(12, log.Size) + s.Require().EqualValues(2, log.NamespaceId) + s.Require().EqualValues(2, log.SignerId) + s.Require().EqualValues(1, log.MsgId) + s.Require().EqualValues(2, log.TxId) + + s.Require().NotNil(log.Signer) + s.Require().EqualValues("celestia1jc92qdnty48pafummfr8ava2tjtuhfdw774w60", log.Signer.Address) + + s.Require().NotNil(log.Tx) +} + +func (s *StorageTestSuite) TestBlobLogsByNamespaceAndToTime() { + ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer ctxCancel() + + logs, err := s.storage.BlobLogs.ByNamespace(ctx, 2, storage.BlobLogFilters{ + Limit: 1, + Offset: 0, + Sort: sdk.SortOrderDesc, + SortBy: "time", + To: time.Date(2023, 7, 4, 13, 0, 0, 0, time.UTC), + }) + s.Require().NoError(err) + s.Require().Len(logs, 1) + + log := logs[0] + s.Require().EqualValues(3, log.Id) + s.Require().EqualValues(1000, log.Height) + s.Require().EqualValues("0CsLX630cjij9DR6nqoWfQcCH2pCQSoSuq63dTkd4Bw=", log.Commitment) + s.Require().EqualValues(12, log.Size) + s.Require().EqualValues(2, log.NamespaceId) + s.Require().EqualValues(2, log.SignerId) + s.Require().EqualValues(1, log.MsgId) + s.Require().EqualValues(2, log.TxId) + + s.Require().NotNil(log.Signer) + s.Require().EqualValues("celestia1jc92qdnty48pafummfr8ava2tjtuhfdw774w60", log.Signer.Address) + + s.Require().NotNil(log.Tx) +} + func (s *StorageTestSuite) TestBlobLogsSigner() { ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Second) defer ctxCancel() diff --git a/internal/storage/postgres/scopes.go b/internal/storage/postgres/scopes.go index ad23e686..b994b8a4 100644 --- a/internal/storage/postgres/scopes.go +++ b/internal/storage/postgres/scopes.go @@ -119,7 +119,17 @@ func messagesFilter(query *bun.SelectQuery, fltrs storage.MessageListWithTxFilte func blobLogFilters(query *bun.SelectQuery, fltrs storage.BlobLogFilters) *bun.SelectQuery { if fltrs.Offset > 0 { - query.Offset(fltrs.Offset) + query = query.Offset(fltrs.Offset) + } + + if !fltrs.From.IsZero() { + query = query.Where("time >= ?", fltrs.From) + } + if !fltrs.To.IsZero() { + query = query.Where("time < ?", fltrs.To) + } + if fltrs.Commitment != "" { + query = query.Where("commitment = ?", fltrs.Commitment) } query = limitScope(query, fltrs.Limit)