Skip to content

Commit

Permalink
Merge pull request #4072 from onflow/v0.29.16-height-coordinated-upgrade
Browse files Browse the repository at this point in the history
v0.29.16 height coordinated upgrade
  • Loading branch information
peterargue authored Mar 21, 2023
2 parents 47858e0 + bbde31e commit 4a4ee2b
Show file tree
Hide file tree
Showing 34 changed files with 823 additions and 202 deletions.
34 changes: 16 additions & 18 deletions engine/access/rest/blocks.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,13 @@ import (
"fmt"
"net/http"

"github.com/onflow/flow-go/engine/access/rest/models"
"github.com/onflow/flow-go/engine/access/rest/request"

"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

"github.com/onflow/flow-go/model/flow"

"github.com/onflow/flow-go/access"
"github.com/onflow/flow-go/engine/access/rest/models"
"github.com/onflow/flow-go/engine/access/rest/request"
"github.com/onflow/flow-go/model/flow"
)

// GetBlocksByIDs gets blocks by provided ID or list of IDs.
Expand Down Expand Up @@ -99,7 +97,7 @@ func GetBlockPayloadByID(r *request.Request, backend access.API, _ models.LinkGe
}

blkProvider := NewBlockProvider(backend, forID(&req.ID))
blk, statusErr := blkProvider.getBlock(r.Context())
blk, _, statusErr := blkProvider.getBlock(r.Context())
if statusErr != nil {
return nil, statusErr
}
Expand All @@ -116,7 +114,7 @@ func GetBlockPayloadByID(r *request.Request, backend access.API, _ models.LinkGe
func getBlock(option blockProviderOption, req *request.Request, backend access.API, link models.LinkGenerator) (*models.Block, error) {
// lookup block
blkProvider := NewBlockProvider(backend, option)
blk, err := blkProvider.getBlock(req.Context())
blk, blockStatus, err := blkProvider.getBlock(req.Context())
if err != nil {
return nil, err
}
Expand All @@ -129,7 +127,7 @@ func getBlock(option blockProviderOption, req *request.Request, backend access.A
// handle case where execution result is not yet available
if se, ok := status.FromError(err); ok {
if se.Code() == codes.NotFound {
err := block.Build(blk, nil, link, req.ExpandFields)
err := block.Build(blk, nil, link, blockStatus, req.ExpandFields)
if err != nil {
return nil, err
}
Expand All @@ -139,7 +137,7 @@ func getBlock(option blockProviderOption, req *request.Request, backend access.A
return nil, err
}

err = block.Build(blk, executionResult, link, req.ExpandFields)
err = block.Build(blk, executionResult, link, blockStatus, req.ExpandFields)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -192,31 +190,31 @@ func NewBlockProvider(backend access.API, options ...blockProviderOption) *block
return blkProvider
}

func (blkProvider *blockProvider) getBlock(ctx context.Context) (*flow.Block, error) {
func (blkProvider *blockProvider) getBlock(ctx context.Context) (*flow.Block, flow.BlockStatus, error) {
if blkProvider.id != nil {
blk, _, err := blkProvider.backend.GetBlockByID(ctx, *blkProvider.id)
if err != nil { // unfortunately backend returns internal error status if not found
return nil, NewNotFoundError(
return nil, flow.BlockStatusUnknown, NewNotFoundError(
fmt.Sprintf("error looking up block with ID %s", blkProvider.id.String()), err,
)
}
return blk, nil
return blk, flow.BlockStatusUnknown, nil
}

if blkProvider.latest {
blk, _, err := blkProvider.backend.GetLatestBlock(ctx, blkProvider.sealed)
blk, status, err := blkProvider.backend.GetLatestBlock(ctx, blkProvider.sealed)
if err != nil {
// cannot be a 'not found' error since final and sealed block should always be found
return nil, NewRestError(http.StatusInternalServerError, "block lookup failed", err)
return nil, flow.BlockStatusUnknown, NewRestError(http.StatusInternalServerError, "block lookup failed", err)
}
return blk, nil
return blk, status, nil
}

blk, _, err := blkProvider.backend.GetBlockByHeight(ctx, blkProvider.height)
blk, status, err := blkProvider.backend.GetBlockByHeight(ctx, blkProvider.height)
if err != nil { // unfortunately backend returns internal error status if not found
return nil, NewNotFoundError(
return nil, flow.BlockStatusUnknown, NewNotFoundError(
fmt.Sprintf("error looking up block at height %d", blkProvider.height), err,
)
}
return blk, nil
return blk, status, nil
}
33 changes: 19 additions & 14 deletions engine/access/rest/blocks_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,13 @@ func TestGetBlocks(t *testing.T) {
blkCnt := 10
blockIDs, heights, blocks, executionResults := generateMocks(backend, blkCnt)

singleBlockExpandedResponse := expectedBlockResponsesExpanded(blocks[:1], executionResults[:1], true)
multipleBlockExpandedResponse := expectedBlockResponsesExpanded(blocks, executionResults, true)
singleBlockExpandedResponse := expectedBlockResponsesExpanded(blocks[:1], executionResults[:1], true, flow.BlockStatusUnknown)
singleSealedBlockExpandedResponse := expectedBlockResponsesExpanded(blocks[:1], executionResults[:1], true, flow.BlockStatusSealed)
multipleBlockExpandedResponse := expectedBlockResponsesExpanded(blocks, executionResults, true, flow.BlockStatusUnknown)
multipleSealedBlockExpandedResponse := expectedBlockResponsesExpanded(blocks, executionResults, true, flow.BlockStatusSealed)

singleBlockCondensedResponse := expectedBlockResponsesExpanded(blocks[:1], executionResults[:1], false)
multipleBlockCondensedResponse := expectedBlockResponsesExpanded(blocks, executionResults, false)
singleBlockCondensedResponse := expectedBlockResponsesExpanded(blocks[:1], executionResults[:1], false, flow.BlockStatusUnknown)
multipleBlockCondensedResponse := expectedBlockResponsesExpanded(blocks, executionResults, false, flow.BlockStatusUnknown)

invalidID := unittest.IdentifierFixture().String()
invalidHeight := fmt.Sprintf("%d", blkCnt+1)
Expand Down Expand Up @@ -78,19 +80,19 @@ func TestGetBlocks(t *testing.T) {
description: "Get single expanded block by height",
request: getByHeightsExpandedURL(t, heights[:1]...),
expectedStatus: http.StatusOK,
expectedResponse: singleBlockExpandedResponse,
expectedResponse: singleSealedBlockExpandedResponse,
},
{
description: "Get multiple expanded blocks by heights",
request: getByHeightsExpandedURL(t, heights...),
expectedStatus: http.StatusOK,
expectedResponse: multipleBlockExpandedResponse,
expectedResponse: multipleSealedBlockExpandedResponse,
},
{
description: "Get multiple expanded blocks by start and end height",
request: getByStartEndHeightExpandedURL(t, heights[0], heights[len(heights)-1]),
expectedStatus: http.StatusOK,
expectedResponse: multipleBlockExpandedResponse,
expectedResponse: multipleSealedBlockExpandedResponse,
},
{
description: "Get block by ID not found",
Expand Down Expand Up @@ -223,20 +225,21 @@ func generateMocks(backend *mock.API, count int) ([]string, []string, []*flow.Bl
return blockIDs, heights, blocks, executionResults
}

func expectedBlockResponsesExpanded(blocks []*flow.Block, execResult []*flow.ExecutionResult, expanded bool) string {
func expectedBlockResponsesExpanded(blocks []*flow.Block, execResult []*flow.ExecutionResult, expanded bool, status flow.BlockStatus) string {
blockResponses := make([]string, len(blocks))
for i, b := range blocks {
blockResponses[i] = expectedBlockResponse(b, execResult[i], expanded)
blockResponses[i] = expectedBlockResponse(b, execResult[i], expanded, status)
}
return fmt.Sprintf("[%s]", strings.Join(blockResponses, ","))
}

func expectedBlockResponse(block *flow.Block, execResult *flow.ExecutionResult, expanded bool) string {
func expectedBlockResponse(block *flow.Block, execResult *flow.ExecutionResult, expanded bool, status flow.BlockStatus) string {
id := block.ID().String()
execResultID := execResult.ID().String()
execLink := fmt.Sprintf("/v1/execution_results/%s", execResultID)
blockLink := fmt.Sprintf("/v1/blocks/%s", id)
payloadLink := fmt.Sprintf("/v1/blocks/%s/payload", id)
blockStatus := status.String()

timestamp := block.Header.Timestamp.Format(time.RFC3339Nano)

Expand All @@ -258,9 +261,10 @@ func expectedBlockResponse(block *flow.Block, execResult *flow.ExecutionResult,
"_expandable": {},
"_links": {
"_self": "%s"
}
},
"block_status": "%s"
}`, id, block.Header.ParentID.String(), block.Header.Height, timestamp,
util.ToBase64(block.Header.ParentVoterSigData), executionResultExpectedStr(execResult), blockLink)
util.ToBase64(block.Header.ParentVoterSigData), executionResultExpectedStr(execResult), blockLink, blockStatus)
}

return fmt.Sprintf(`
Expand All @@ -278,7 +282,8 @@ func expectedBlockResponse(block *flow.Block, execResult *flow.ExecutionResult,
},
"_links": {
"_self": "%s"
}
},
"block_status": "%s"
}`, id, block.Header.ParentID.String(), block.Header.Height, timestamp,
util.ToBase64(block.Header.ParentVoterSigData), payloadLink, execLink, blockLink)
util.ToBase64(block.Header.ParentVoterSigData), payloadLink, execLink, blockLink, blockStatus)
}
2 changes: 2 additions & 0 deletions engine/access/rest/models/block.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ func (b *Block) Build(
block *flow.Block,
execResult *flow.ExecutionResult,
link LinkGenerator,
blockStatus flow.BlockStatus,
expand map[string]bool,
) error {
self, err := SelfLink(block.ID(), link.BlockLink)
Expand Down Expand Up @@ -61,6 +62,7 @@ func (b *Block) Build(
}

b.Links = self
b.BlockStatus = blockStatus.String()
return nil
}

Expand Down
1 change: 1 addition & 0 deletions engine/access/rest/models/model_block.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,5 @@ type Block struct {
ExecutionResult *ExecutionResult `json:"execution_result,omitempty"`
Expandable *BlockExpandable `json:"_expandable"`
Links *Links `json:"_links,omitempty"`
BlockStatus string `json:"block_status"`
}
4 changes: 2 additions & 2 deletions engine/execution/computation/computer/computer.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,6 @@ func (e *blockComputer) ExecuteBlock(
return nil, fmt.Errorf("failed to execute transactions: %w", err)
}

// TODO: compute block fees & reward payments

return results, nil
}

Expand Down Expand Up @@ -315,6 +313,8 @@ func (e *blockComputer) executeBlock(
Hex("block_id", logging.Entity(block)).
Msg("all views committed")

e.metrics.ExecutionBlockCachedPrograms(derivedBlockData.CachedPrograms())

executionDataID, err := e.executionDataProvider.Provide(
ctx,
block.Height(),
Expand Down
53 changes: 50 additions & 3 deletions engine/execution/computation/computer/computer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,20 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) {

t.Run("single collection", func(t *testing.T) {

execCtx := fvm.NewContext()
execCtx := fvm.NewContext(
fvm.WithDerivedBlockData(derived.NewEmptyDerivedBlockData()),
)

vm := new(computermock.VirtualMachine)
vm.On("Run", mock.Anything, mock.Anything, mock.Anything).
Return(nil).
Run(func(args mock.Arguments) {
// ctx := args[0].(fvm.Context)
ctx := args[0].(fvm.Context)
tx := args[1].(*fvm.TransactionProcedure)

tx.Events = generateEvents(1, tx.TxIndex)

getSetAProgram(t, ctx.DerivedBlockData)
}).
Times(2 + 1) // 2 txs in collection + system chunk

Expand All @@ -96,6 +100,13 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) {
Return(nil).
Times(2 + 1) // 2 txs in collection + system chunk tx

expectedProgramsInCache := 1 // we set one program in the cache
exemetrics.On(
"ExecutionBlockCachedPrograms",
expectedProgramsInCache).
Return(nil).
Times(1) // 1 block

bservice := requesterunit.MockBlobService(blockstore.NewBlockstore(dssync.MutexWrap(datastore.NewMapDatastore())))
trackerStorage := new(mocktracker.Storage)
trackerStorage.On("Update", mock.Anything).Return(func(fn tracker.UpdateFn) error {
Expand Down Expand Up @@ -948,9 +959,12 @@ func Test_ExecutingSystemCollection(t *testing.T) {

noopCollector := metrics.NewNoopCollector()

metrics := new(modulemock.ExecutionMetrics)
expectedNumberOfEvents := 2
expectedEventSize := 911
// bootstrapping does not cache programs
expectedCachedPrograms := 0

metrics := new(modulemock.ExecutionMetrics)
metrics.On("ExecutionCollectionExecuted",
mock.Anything, // duration
mock.Anything). // stats
Expand All @@ -968,6 +982,12 @@ func Test_ExecutingSystemCollection(t *testing.T) {
Return(nil).
Times(1) // system chunk tx

metrics.On(
"ExecutionBlockCachedPrograms",
expectedCachedPrograms).
Return(nil).
Times(1) // block

bservice := requesterunit.MockBlobService(blockstore.NewBlockstore(dssync.MutexWrap(datastore.NewMapDatastore())))
trackerStorage := new(mocktracker.Storage)
trackerStorage.On("Update", mock.Anything).Return(func(fn tracker.UpdateFn) error {
Expand Down Expand Up @@ -1100,3 +1120,30 @@ func generateEvents(eventCount int, txIndex uint32) []flow.Event {
}
return events
}

func getSetAProgram(t *testing.T, derivedBlockData *derived.DerivedBlockData) {

derivedTxnData, err := derivedBlockData.NewDerivedTransactionData(
0,
0)
require.NoError(t, err)

loc := common.AddressLocation{
Name: "SomeContract",
Address: common.MustBytesToAddress([]byte{0x1}),
}
_, _, got := derivedTxnData.GetProgram(
loc,
)
if got {
return
}

derivedTxnData.SetProgram(
loc,
&derived.Program{},
&state.State{},
)
err = derivedTxnData.Commit()
require.NoError(t, err)
}
7 changes: 2 additions & 5 deletions engine/execution/computation/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,16 +117,13 @@ func New(
vm = fvm.NewVirtualMachine()
}

chainID := vmCtx.Chain.ChainID()

options := []fvm.Option{
fvm.WithReusableCadenceRuntimePool(
reusableRuntime.NewReusableCadenceRuntimePool(
ReusableCadenceRuntimePoolSize,
runtime.Config{
TracingEnabled: params.CadenceTracing,
// AccountLinking is enabled everywhere except on mainnet
AccountLinkingEnabled: chainID != flow.Mainnet,
TracingEnabled: params.CadenceTracing,
AccountLinkingEnabled: true,
},
),
),
Expand Down
Loading

0 comments on commit 4a4ee2b

Please sign in to comment.