diff --git a/README.md b/README.md index 9634fbed39..2cf8d688fd 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ and [Decrediton (GUI)](https://github.com/decred/decrediton). ## Minimum Recommended Specifications (dcrd only) * 12 GB disk space (as of April 2020, increases over time) -* 1GB memory (RAM) +* 2GB memory (RAM) * ~150MB/day download, ~1.5GB/day upload * Plus one-time initial download of the entire block chain * Windows 10 (server preferred), macOS, Linux diff --git a/blockchain/chain.go b/blockchain/chain.go index 92f1f40a96..3c22a9aef3 100644 --- a/blockchain/chain.go +++ b/blockchain/chain.go @@ -143,6 +143,7 @@ type BlockChain struct { sigCache *txscript.SigCache indexManager indexers.IndexManager interrupt <-chan struct{} + utxoCache *UtxoCache // subsidyCache is the cache that provides quick lookup of subsidy // values. @@ -619,14 +620,6 @@ func (b *BlockChain) connectBlock(node *blockNode, block, parent *dcrutil.Block, return err } - // Update the utxo set using the state of the utxo view. This - // entails removing all of the utxos spent and adding the new - // ones created by the block. - err = dbPutUtxoView(dbTx, view) - if err != nil { - return err - } - // Update the transaction spend journal by adding a record for // the block that contains all txos spent by it. err = dbPutSpendJournalEntry(dbTx, block.Hash(), stxos) @@ -676,9 +669,26 @@ func (b *BlockChain) connectBlock(node *blockNode, block, parent *dcrutil.Block, return err } - // Prune fully spent entries and mark all entries in the view unmodified - // now that the modifications have been committed to the database. - view.commit() + // Commit all entries in the view to the utxo cache. All entries in the view + // that are marked as modified and spent are removed from the view. + // Additionally, all entries that are added to the cache are removed from the + // view. + err = b.utxoCache.Commit(view) + if err != nil { + return err + } + + // Conditionally flush the utxo cache to the database. Force a flush if the + // chain believes it is current since blocks are connected infrequently at + // that point. Only log the flush when the chain is not current as it is + // mostly useful to see the flush details when many blocks are being connected + // (and subsequently flushed) in quick succession. + isCurrent := b.isCurrent(node) + err = b.utxoCache.MaybeFlush(&node.hash, uint32(node.height), isCurrent, + !isCurrent) + if err != nil { + return err + } // This node is now the end of the best chain. b.bestChain.SetTip(node) @@ -814,21 +824,6 @@ func (b *BlockChain) disconnectBlock(node *blockNode, block, parent *dcrutil.Blo return err } - // Update the utxo set using the state of the utxo view. This - // entails restoring all of the utxos spent and removing the new - // ones created by the block. - err = dbPutUtxoView(dbTx, view) - if err != nil { - return err - } - - // Update the transaction spend journal by removing the record - // that contains all txos spent by the block. - err = dbRemoveSpendJournalEntry(dbTx, block.Hash()) - if err != nil { - return err - } - err = stake.WriteDisconnectedBestNode(dbTx, parentStakeNode, node.parent.hash, childStakeNode.UndoData()) if err != nil { @@ -856,9 +851,35 @@ func (b *BlockChain) disconnectBlock(node *blockNode, block, parent *dcrutil.Blo return err } - // Prune fully spent entries and mark all entries in the view unmodified - // now that the modifications have been committed to the database. - view.commit() + // Commit all entries in the view to the utxo cache. All entries in the view + // that are marked as modified and spent are removed from the view. + // Additionally, all entries that are added to the cache are removed from the + // view. + err = b.utxoCache.Commit(view) + if err != nil { + return err + } + + // Force a utxo cache flush when blocks are being disconnected. A cache flush + // is forced here since the spend journal entry for the disconnected block + // will be removed below. + err = b.utxoCache.MaybeFlush(&node.parent.hash, uint32(node.parent.height), + true, false) + if err != nil { + return err + } + + // Update the transaction spend journal by removing the record that contains + // all txos spent by the block. This is intentionally done AFTER the utxo + // cache has been force flushed since the spend journal information will no + // longer be available for the cache to use for recovery purposes after being + // removed. + err = b.db.Update(func(dbTx database.Tx) error { + return dbRemoveSpendJournalEntry(dbTx, block.Hash()) + }) + if err != nil { + return err + } // This node's parent is now the end of the best chain. b.bestChain.SetTip(node.parent) @@ -993,7 +1014,7 @@ func (b *BlockChain) reorganizeChainInternal(target *blockNode) error { // using that information to unspend all of the spent txos and remove the // utxos created by the blocks. In addition, if a block votes against its // parent, the regular transactions are reconnected. - view := NewUtxoViewpoint() + view := NewUtxoViewpoint(b.utxoCache) view.SetBestHash(&tip.hash) var nextBlockToDetach *dcrutil.Block for tip != nil && tip != fork { @@ -1049,7 +1070,7 @@ func (b *BlockChain) reorganizeChainInternal(target *blockNode) error { // Update the view to unspend all of the spent txos and remove the utxos // created by the block. Also, if the block votes against its parent, // reconnect all of the regular transactions. - err = view.disconnectBlock(b.db, block, parent, stxos, isTreasuryEnabled) + err = view.disconnectBlock(block, parent, stxos, isTreasuryEnabled) if err != nil { return err } @@ -2125,6 +2146,13 @@ type Config struct { // This field can be nil if the caller does not wish to make use of an // index manager. IndexManager indexers.IndexManager + + // UtxoCache defines a utxo cache that sits on top of the utxo set database. + // All utxo reads and writes go through the cache, and never read or write to + // the database directly. + // + // This field is required. + UtxoCache *UtxoCache } // New returns a BlockChain instance using the provided configuration details. @@ -2190,6 +2218,7 @@ func New(ctx context.Context, config *Config) (*BlockChain, error) { calcPriorStakeVersionCache: make(map[[chainhash.HashSize]byte]uint32), calcVoterVersionIntervalCache: make(map[[chainhash.HashSize]byte]uint32), calcStakeVersionCache: make(map[[chainhash.HashSize]byte]uint32), + utxoCache: config.UtxoCache, } b.pruner = newChainPruner(&b) diff --git a/blockchain/chainio.go b/blockchain/chainio.go index 70ab0367ad..bd7fffaf24 100644 --- a/blockchain/chainio.go +++ b/blockchain/chainio.go @@ -1,5 +1,5 @@ // Copyright (c) 2015-2016 The btcsuite developers -// Copyright (c) 2016-2020 The Decred developers +// Copyright (c) 2016-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -80,6 +80,10 @@ var ( // transaction output set. utxoSetBucketName = []byte("utxosetv3") + // utxoSetStateKeyName is the name of the database key used to house the + // state of the unspent transaction output set. + utxoSetStateKeyName = []byte("utxosetstate") + // blockIndexBucketName is the name of the db bucket used to house the block // index which consists of metadata for all known blocks both in the main // chain and on side chains. @@ -993,16 +997,13 @@ func deserializeUtxoEntry(serialized []byte, txOutIndex uint32) (*UtxoEntry, err offset += bytesRead // Create a new utxo entry with the details deserialized above. - const spent = false - const modified = false entry := &UtxoEntry{ amount: amount, pkScript: script, blockHeight: uint32(blockHeight), blockIndex: uint32(blockIndex), scriptVersion: scriptVersion, - packedFlags: encodeUtxoFlags(isCoinBase, spent, modified, hasExpiry, - txType), + packedFlags: encodeUtxoFlags(isCoinBase, hasExpiry, txType), } // Copy the minimal outputs if this was a ticket submission output. @@ -1116,48 +1117,142 @@ func dbFetchUtxoStats(dbTx database.Tx) (*UtxoStats, error) { return &stats, nil } -// dbPutUtxoView uses an existing database transaction to update the utxo set in -// the database based on the provided utxo view contents and state. In -// particular, only the entries that have been marked as modified are written to -// the database. -func dbPutUtxoView(dbTx database.Tx, view *UtxoViewpoint) error { - utxoBucket := dbTx.Metadata().Bucket(utxoSetBucketName) - for outpoint, entry := range view.entries { - // No need to update the database if the entry was not modified. - if entry == nil || !entry.isModified() { - continue - } - - // Remove the utxo entry if it is spent. - if entry.IsSpent() { - key := outpointKey(outpoint) - err := utxoBucket.Delete(*key) - recycleOutpointKey(key) - if err != nil { - return err - } - - continue - } +// dbPutUtxoEntry uses an existing database transaction to update the utxo +// entry for the given outpoint based on the provided utxo entry state. In +// particular, the entry is only written to the database if it is marked as +// modified, and if the entry is marked as spent it is removed from the +// database. +func dbPutUtxoEntry(dbTx database.Tx, outpoint wire.OutPoint, entry *UtxoEntry) error { + // No need to update the database if the entry was not modified. + if entry == nil || !entry.isModified() { + return nil + } - // Serialize and store the utxo entry. - serialized, err := serializeUtxoEntry(entry) - if err != nil { - return err - } + // Remove the utxo entry if it is spent. + utxoBucket := dbTx.Metadata().Bucket(utxoSetBucketName) + if entry.IsSpent() { key := outpointKey(outpoint) - err = utxoBucket.Put(*key, serialized) - // NOTE: The key is intentionally not recycled here since the database - // interface contract prohibits modifications. It will be garbage collected - // normally when the database is done with it. + err := utxoBucket.Delete(*key) + recycleOutpointKey(key) if err != nil { return err } + + return nil + } + + // Serialize and store the utxo entry. + serialized, err := serializeUtxoEntry(entry) + if err != nil { + return err + } + key := outpointKey(outpoint) + err = utxoBucket.Put(*key, serialized) + // NOTE: The key is intentionally not recycled here since the database + // interface contract prohibits modifications. It will be garbage collected + // normally when the database is done with it. + if err != nil { + return err } return nil } +// ----------------------------------------------------------------------------- +// The utxo set state contains information regarding the current state of the +// utxo set. In particular, it tracks the block height and block hash of the +// last completed flush. +// +// The utxo set state is tracked in the database since at any given time, the +// utxo cache may not be consistent with the utxo set in the database. This is +// due to the fact that the utxo cache only flushes changes to the database +// periodically. Therefore, during initialization, the utxo set state is used +// to identify the last flushed state of the utxo set and it can be caught up +// to the current best state of the main chain. +// +// Note: The utxo set state MUST always be updated in the same database +// transaction that the utxo set is updated in to guarantee that they stay in +// sync in the database. +// +// The serialized format is: +// +// +// +// Field Type Size +// block height VLQ variable +// block hash chainhash.Hash chainhash.HashSize +// +// ----------------------------------------------------------------------------- + +// utxoSetState represents the current state of the utxo set. In particular, +// it tracks the block height and block hash of the last completed flush. +type utxoSetState struct { + lastFlushHeight uint32 + lastFlushHash chainhash.Hash +} + +// serializeUtxoSetState serializes the provided utxo set state. The format is +// described in detail above. +func serializeUtxoSetState(state *utxoSetState) []byte { + // Calculate the size needed to serialize the utxo set state. + size := serializeSizeVLQ(uint64(state.lastFlushHeight)) + chainhash.HashSize + + // Serialize the utxo set state and return it. + serialized := make([]byte, size) + offset := putVLQ(serialized, uint64(state.lastFlushHeight)) + copy(serialized[offset:], state.lastFlushHash[:]) + return serialized +} + +// deserializeUtxoSetState deserializes the passed serialized byte slice into +// the utxo set state. The format is described in detail above. +func deserializeUtxoSetState(serialized []byte) (*utxoSetState, error) { + // Deserialize the block height. + blockHeight, bytesRead := deserializeVLQ(serialized) + offset := bytesRead + if offset >= len(serialized) { + return nil, errDeserialize("unexpected end of data after height") + } + + // Deserialize the hash. + if len(serialized[offset:]) != chainhash.HashSize { + return nil, errDeserialize("unexpected length for serialized hash") + } + var hash chainhash.Hash + copy(hash[:], serialized[offset:offset+chainhash.HashSize]) + + // Create the utxo set state and return it. + return &utxoSetState{ + lastFlushHeight: uint32(blockHeight), + lastFlushHash: hash, + }, nil +} + +// dbPutUtxoSetState uses an existing database transaction to update the utxo +// set state in the database. +func dbPutUtxoSetState(dbTx database.Tx, state *utxoSetState) error { + // Serialize and store the utxo set state. + return dbTx.Metadata().Put(utxoSetStateKeyName, serializeUtxoSetState(state)) +} + +// dbFetchUtxoSetState uses an existing database transaction to fetch the utxo +// set state from the database. If the utxo set state does not exist in the +// database, nil is returned. +func dbFetchUtxoSetState(dbTx database.Tx) (*utxoSetState, error) { + // Fetch the serialized utxo set state from the database. + serialized := dbTx.Metadata().Get(utxoSetStateKeyName) + + // Return nil if the utxo set state does not exist in the database. This + // should only be the case when starting from a fresh database or a database + // that has not been run with the utxo cache yet. + if serialized == nil { + return nil, nil + } + + // Deserialize the utxo set state and return it. + return deserializeUtxoSetState(serialized) +} + // ----------------------------------------------------------------------------- // The GCS filter journal consists of an entry for each block connected to the // main chain (or has ever been connected to it) which consists of a serialized @@ -1735,6 +1830,7 @@ func (b *BlockChain) initChainState(ctx context.Context) error { } // Attempt to load the chain state from the database. + var tip *blockNode err = b.db.View(func(dbTx database.Tx) error { // Fetch the stored best chain state from the database. state, err := dbFetchBestState(dbTx) @@ -1753,7 +1849,7 @@ func (b *BlockChain) initChainState(ctx context.Context) error { } // Set the best chain to the stored best state. - tip := b.index.lookupNode(&state.hash) + tip = b.index.lookupNode(&state.hash) if tip == nil { return AssertError(fmt.Sprintf("initChainState: cannot find "+ "chain tip %s in block index", state.hash)) @@ -1828,7 +1924,14 @@ func (b *BlockChain) initChainState(ctx context.Context) error { } // Upgrade the database post block index load as needed. - return upgradeDBPostBlockIndexLoad(ctx, b) + err = upgradeDBPostBlockIndexLoad(ctx, b) + if err != nil { + return err + } + + // Initialize the utxo cache to ensure that the state of the utxo set is + // caught up to the tip of the best chain. + return b.InitUtxoCache(tip) } // dbFetchBlockByNode uses an existing database transaction to retrieve the raw diff --git a/blockchain/chainio_test.go b/blockchain/chainio_test.go index c83a66e611..37a4c3df88 100644 --- a/blockchain/chainio_test.go +++ b/blockchain/chainio_test.go @@ -1,5 +1,5 @@ // Copyright (c) 2015-2016 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -10,6 +10,8 @@ import ( "encoding/hex" "errors" "math/big" + "os" + "path/filepath" "reflect" "testing" "time" @@ -828,9 +830,6 @@ func TestUtxoSerialization(t *testing.T) { withCoinbase = true noExpiry = false withExpiry = true - unspent = false - spent = true - unmodified = false ) tests := []struct { @@ -851,8 +850,6 @@ func TestUtxoSerialization(t *testing.T) { scriptVersion: 0, packedFlags: encodeUtxoFlags( withCoinbase, - unspent, - unmodified, noExpiry, stake.TxTypeRegular, ), @@ -872,8 +869,6 @@ func TestUtxoSerialization(t *testing.T) { scriptVersion: 0, packedFlags: encodeUtxoFlags( withCoinbase, - unspent, - unmodified, noExpiry, stake.TxTypeRegular, ), @@ -892,8 +887,6 @@ func TestUtxoSerialization(t *testing.T) { scriptVersion: 0, packedFlags: encodeUtxoFlags( noCoinbase, - unspent, - unmodified, noExpiry, stake.TxTypeRegular, ), @@ -912,8 +905,6 @@ func TestUtxoSerialization(t *testing.T) { scriptVersion: 0, packedFlags: encodeUtxoFlags( noCoinbase, - unspent, - unmodified, withExpiry, stake.TxTypeSStx, ), @@ -940,8 +931,6 @@ func TestUtxoSerialization(t *testing.T) { scriptVersion: 0xffff, packedFlags: encodeUtxoFlags( withCoinbase, - unspent, - unmodified, noExpiry, stake.TxTypeRegular, ), @@ -960,8 +949,6 @@ func TestUtxoSerialization(t *testing.T) { scriptVersion: 0, packedFlags: encodeUtxoFlags( noCoinbase, - unspent, - unmodified, withExpiry, stake.TxTypeRegular, ), @@ -979,10 +966,9 @@ func TestUtxoSerialization(t *testing.T) { blockHeight: 33333, blockIndex: 3, scriptVersion: 0, + state: utxoStateModified | utxoStateSpent, packedFlags: encodeUtxoFlags( withCoinbase, - spent, - unmodified, withExpiry, stake.TxTypeRegular, ), @@ -1016,12 +1002,12 @@ func TestUtxoSerialization(t *testing.T) { // Ensure that the serialized bytes are decoded back to the expected utxo. gotUtxo, err := deserializeUtxoEntry(test.serialized, test.txOutIndex) if err != nil { - t.Errorf("serializeUtxoEntry #%d (%s): unexpected error: %v", i, + t.Errorf("deserializeUtxoEntry #%d (%s): unexpected error: %v", i, test.name, err) continue } if !reflect.DeepEqual(gotUtxo, test.entry) { - t.Errorf("serializeUtxoEntry #%d (%s):\nwant: %+v\n got: %+v\n", i, + t.Errorf("deserializeUtxoEntry #%d (%s):\nwant: %+v\n got: %+v\n", i, test.name, test.entry, gotUtxo) } } @@ -1139,6 +1125,163 @@ func TestUtxoEntryDeserializeErrors(t *testing.T) { } } +// TestUtxoSetStateSerialization ensures that serializing and deserializing +// the utxo set state works as expected. +func TestUtxoSetStateSerialization(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + state *utxoSetState + serialized []byte + }{{ + name: "last flush height and hash updated", + state: &utxoSetState{ + lastFlushHeight: 432100, + lastFlushHash: *mustParseHash("000000000000000023455b4328635d8e014dbeea" + + "99c6140aa715836cc7e55981"), + }, + serialized: hexToBytes("99ae648159e5c76c8315a70a14c699eabe4d018e5d6328435" + + "b45230000000000000000"), + }, { + name: "last flush height and hash are the genesis block", + state: &utxoSetState{ + lastFlushHeight: 0, + lastFlushHash: *mustParseHash("298e5cc3d985bfe7f81dc135f360abe089edd439" + + "6b86d2de66b0cef42b21d980"), + }, + serialized: hexToBytes("0080d9212bf4ceb066ded2866b39d4ed89e0ab60f335c11df" + + "8e7bf85d9c35c8e29"), + }} + + for _, test := range tests { + // Ensure the utxo set state serializes to the expected value. + gotBytes := serializeUtxoSetState(test.state) + if !bytes.Equal(gotBytes, test.serialized) { + t.Errorf("serializeUtxoSetState (%s): mismatched bytes - got %x, "+ + "want %x", test.name, gotBytes, test.serialized) + continue + } + + // Ensure that the serialized bytes are decoded back to the expected utxo + // set state. + gotUtxoSetState, err := deserializeUtxoSetState(test.serialized) + if err != nil { + t.Errorf("deserializeUtxoSetState (%s): unexpected error: %v", test.name, + err) + continue + } + if !reflect.DeepEqual(gotUtxoSetState, test.state) { + t.Errorf("deserializeUtxoSetState (%s):\nwant: %+v\n got: %+v\n", + test.name, test.state, gotUtxoSetState) + } + } +} + +// TestUtxoSetStateDeserializeErrors performs negative tests against +// deserializing the utxo set state to ensure error paths work as expected. +func TestUtxoSetStateDeserializeErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + serialized []byte + errType error + }{{ + // [EOF] + name: "nothing serialized (no last flush height)", + serialized: hexToBytes(""), + errType: errDeserialize(""), + }, { + // [] + name: "no data after last flush height", + serialized: hexToBytes("99ae64"), + errType: errDeserialize(""), + }, { + // [] + name: "truncated hash", + serialized: hexToBytes("99ae648159e5c76c8315a70a14c699"), + errType: errDeserialize(""), + }} + + for _, test := range tests { + // Ensure the expected error type is returned and the returned + // utxo set state is nil. + entry, err := deserializeUtxoSetState(test.serialized) + if !errors.As(err, &test.errType) { + t.Errorf("deserializeUtxoSetState (%s): expected error type does not "+ + "match - got %T, want %T", test.name, err, test.errType) + continue + } + if entry != nil { + t.Errorf("deserializeUtxoSetState (%s): returned utxo set state is not "+ + "nil", test.name) + continue + } + } +} + +// TestDbFetchUtxoSetState ensures that putting and fetching the utxo set state +// works as expected. +func TestDbFetchUtxoSetState(t *testing.T) { + t.Parallel() + + // Create a test database. + dbPath := filepath.Join(os.TempDir(), "test-dbfetchutxosetstate") + _ = os.RemoveAll(dbPath) + db, err := database.Create("ffldb", dbPath, wire.MainNet) + if err != nil { + t.Fatalf("error creating test database: %v", err) + } + defer os.RemoveAll(dbPath) + defer db.Close() + + tests := []struct { + name string + state *utxoSetState + }{{ + name: "fresh database (no utxo set state saved)", + state: nil, + }, { + name: "last flush saved in database", + state: &utxoSetState{ + lastFlushHeight: 432100, + lastFlushHash: *mustParseHash("000000000000000023455b4328635d8e014dbeea" + + "99c6140aa715836cc7e55981"), + }, + }} + + for _, test := range tests { + // Update the utxo set state in the database. + if test.state != nil { + err = db.Update(func(dbTx database.Tx) error { + return dbPutUtxoSetState(dbTx, test.state) + }) + if err != nil { + t.Fatalf("%q: error putting utxo set state: %v", test.name, err) + } + } + + // Fetch the utxo set state from the database. + err = db.View(func(dbTx database.Tx) error { + gotState, err := dbFetchUtxoSetState(dbTx) + if err != nil { + return err + } + + // Ensure that the fetched utxo set state matches the expected state. + if !reflect.DeepEqual(gotState, test.state) { + t.Errorf("%q: mismatched state:\nwant: %+v\n got: %+v\n", test.name, + test.state, gotState) + } + return nil + }) + if err != nil { + t.Fatalf("%q: error fetching utxo set state: %v", test.name, err) + } + } +} + // TestBestChainStateSerialization ensures serializing and deserializing the // best chain state works as expected. func TestBestChainStateSerialization(t *testing.T) { diff --git a/blockchain/common_test.go b/blockchain/common_test.go index 7c323dcc97..60954e44be 100644 --- a/blockchain/common_test.go +++ b/blockchain/common_test.go @@ -1,5 +1,5 @@ // Copyright (c) 2013-2016 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -117,6 +117,10 @@ func chainSetup(dbName string, params *chaincfg.Params) (*BlockChain, func(), er ChainParams: ¶msCopy, TimeSource: NewMedianTime(), SigCache: sigCache, + UtxoCache: NewUtxoCache(&UtxoCacheConfig{ + DB: db, + MaxSize: 100 * 1024 * 1024, // 100 MiB + }), }) if err != nil { diff --git a/blockchain/example_test.go b/blockchain/example_test.go index 54a27c4b1f..be9961154e 100644 --- a/blockchain/example_test.go +++ b/blockchain/example_test.go @@ -1,5 +1,5 @@ // Copyright (c) 2014-2016 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -52,6 +52,10 @@ func ExampleBlockChain_ProcessBlock() { DB: db, ChainParams: mainNetParams, TimeSource: blockchain.NewMedianTime(), + UtxoCache: blockchain.NewUtxoCache(&blockchain.UtxoCacheConfig{ + DB: db, + MaxSize: 100 * 1024 * 1024, // 100 MiB + }), }) if err != nil { fmt.Printf("Failed to create chain instance: %v\n", err) diff --git a/blockchain/fullblocks_test.go b/blockchain/fullblocks_test.go index 6d7a784ad6..f59ae7c188 100644 --- a/blockchain/fullblocks_test.go +++ b/blockchain/fullblocks_test.go @@ -1,5 +1,5 @@ // Copyright (c) 2016 The btcsuite developers -// Copyright (c) 2016-2020 The Decred developers +// Copyright (c) 2016-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -111,6 +111,10 @@ func chainSetup(dbName string, params *chaincfg.Params) (*blockchain.BlockChain, ChainParams: ¶msCopy, TimeSource: blockchain.NewMedianTime(), SigCache: sigCache, + UtxoCache: blockchain.NewUtxoCache(&blockchain.UtxoCacheConfig{ + DB: db, + MaxSize: 100 * 1024 * 1024, // 100 MiB + }), }) if err != nil { diff --git a/blockchain/go.mod b/blockchain/go.mod index cc3a211a48..fe0aca00e7 100644 --- a/blockchain/go.mod +++ b/blockchain/go.mod @@ -1,6 +1,6 @@ module github.com/decred/dcrd/blockchain/v4 -go 1.13 +go 1.14 require ( github.com/decred/dcrd/blockchain/stake/v4 v4.0.0-20210129192908-660d0518b4cf diff --git a/blockchain/headercmt.go b/blockchain/headercmt.go index 123901ef95..964073b90a 100644 --- a/blockchain/headercmt.go +++ b/blockchain/headercmt.go @@ -74,7 +74,7 @@ func (b *BlockChain) FetchUtxoViewParentTemplate(block *wire.MsgBlock) (*UtxoVie // Since the block template is building on the parent of the current tip, // undo the transactions and spend information for the tip block to reach // the point of view of the block template. - view := NewUtxoViewpoint() + view := NewUtxoViewpoint(b.utxoCache) view.SetBestHash(&tip.hash) tipBlock, err := b.fetchMainChainBlockByNode(tip) if err != nil { @@ -103,7 +103,7 @@ func (b *BlockChain) FetchUtxoViewParentTemplate(block *wire.MsgBlock) (*UtxoVie // Update the view to unspend all of the spent txos and remove the utxos // created by the tip block. Also, if the block votes against its parent, // reconnect all of the regular transactions. - err = view.disconnectBlock(b.db, tipBlock, parent, stxos, isTreasuryEnabled) + err = view.disconnectBlock(tipBlock, parent, stxos, isTreasuryEnabled) if err != nil { return nil, err } diff --git a/blockchain/sequencelock_test.go b/blockchain/sequencelock_test.go index 8746fe2a0d..befc82b512 100644 --- a/blockchain/sequencelock_test.go +++ b/blockchain/sequencelock_test.go @@ -55,7 +55,7 @@ func TestCalcSequenceLock(t *testing.T) { PkScript: nil, }}, }) - view := NewUtxoViewpoint() + view := NewUtxoViewpoint(nil) view.AddTxOuts(targetTx, int64(numBlocks)-4, 0, noTreasury) view.SetBestHash(&node.hash) diff --git a/blockchain/stakeext.go b/blockchain/stakeext.go index 244ba69447..19b7166a86 100644 --- a/blockchain/stakeext.go +++ b/blockchain/stakeext.go @@ -1,5 +1,5 @@ // Copyright (c) 2013-2014 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -112,7 +112,7 @@ func (b *BlockChain) TicketsWithAddress(address dcrutil.Address, isTreasuryEnabl err := b.db.View(func(dbTx database.Tx) error { for _, hash := range tickets { outpoint := wire.OutPoint{Hash: hash, Index: 0, Tree: wire.TxTreeStake} - utxo, err := dbFetchUtxoEntry(dbTx, outpoint) + utxo, err := b.utxoCache.FetchEntry(dbTx, outpoint) if err != nil { return err } @@ -224,7 +224,7 @@ func (b *BlockChain) TicketPoolValue() (dcrutil.Amount, error) { err := b.db.View(func(dbTx database.Tx) error { for _, hash := range sn.LiveTickets() { outpoint := wire.OutPoint{Hash: hash, Index: 0, Tree: wire.TxTreeStake} - utxo, err := dbFetchUtxoEntry(dbTx, outpoint) + utxo, err := b.utxoCache.FetchEntry(dbTx, outpoint) if err != nil { return err } diff --git a/blockchain/utxocache.go b/blockchain/utxocache.go new file mode 100644 index 0000000000..3cb048ba0c --- /dev/null +++ b/blockchain/utxocache.go @@ -0,0 +1,910 @@ +// Copyright (c) 2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package blockchain + +import ( + "fmt" + "math" + "sync" + "time" + + "github.com/decred/dcrd/chaincfg/chainhash" + "github.com/decred/dcrd/database/v2" + "github.com/decred/dcrd/dcrutil/v4" + "github.com/decred/dcrd/wire" +) + +const ( + // outpointSize is the size of an outpoint on a 64-bit platform. It is + // equivalent to what unsafe.Sizeof(wire.OutPoint{}) returns on a 64-bit + // platform. + outpointSize = 56 + + // pointerSize is the size of a pointer on a 64-bit platform. + pointerSize = 8 + + // p2pkhScriptLen is the length of a standard pay-to-pubkey-hash script. It + // is used in the calculation to approximate the average size of a utxo entry + // when setting the initial capacity of the cache. + p2pkhScriptLen = 25 + + // mapOverhead is the number of bytes per entry to use when approximating the + // memory overhead of the entries map itself (i.e. the memory usage due to + // internals of the map, such as the underlying buckets that are allocated). + // This number was determined by inspecting the true size of the map with + // various numbers of entries and comparing it with the total size of all + // entries in the map. The average overhead came out to 57 bytes per entry. + mapOverhead = 57 + + // evictionPercentage is the targeted percentage of entries to evict from the + // cache when its maximum size has been reached. + // + // A lower percentage will result in a higher overall hit ratio for the cache, + // and thus better performance, but will require eviction again sooner. This + // value was selected to keep the hit ratio of the cache as high as possible + // while still evicting a significant portion of the cache when it reaches its + // maximum allowed size. + evictionPercentage = 0.15 + + // periodicFlushInterval is the amount of time to wait before a periodic flush + // is required. + // + // The cache is flushed periodically during initial block download to avoid + // requiring a flush that would take a significant amount of time on shutdown + // (or, in the case of an unclean shutdown, a significant amount of time to + // initialize the cache when restarted). + periodicFlushInterval = time.Minute * 2 +) + +// UtxoCache is an unspent transaction output cache that sits on top of the +// utxo set database and provides significant runtime performance benefits at +// the cost of some additional memory usage. It drastically reduces the amount +// of reading and writing to disk, especially during initial block download when +// a very large number of blocks are being processed in quick succession. +// +// The UtxoCache is a read-through cache. All utxo reads go through the cache. +// When there is a cache miss, the cache loads the missing data from the +// database, caches it, and returns it to the caller. +// +// The UtxoCache is a write-back cache. Writes to the cache are acknowledged +// by the cache immediately but are only periodically flushed to the database. +// This allows intermediate steps to effectively be skipped. For example, a +// utxo that is created and then spent in between flushes never needs to be +// written to the utxo set in the database. +// +// Due to the write-back nature of the cache, at any given time the database +// may not be in sync with the cache, and therefore all utxo reads and writes +// MUST go through the cache, and never read or write to the database directly. +type UtxoCache struct { + // db is the database that contains the utxo set. It is set when the instance + // is created and is not changed afterward. + db database.DB + + // maxSize is the maximum allowed size of the utxo cache, in bytes. It is set + // when the instance is created and is not changed afterward. + maxSize uint64 + + // cacheLock protects access to the fields in the struct below this point. A + // standard mutex is used rather than a read-write mutex since the cache will + // often write when reads result in a cache miss, so it is generally not worth + // the additional overhead of using a read-write mutex. + cacheLock sync.Mutex + + // entries holds the cached utxo entries. + entries map[wire.OutPoint]*UtxoEntry + + // lastFlushHash is the block hash of the last flush. It is used to compare + // the state of the cache to the utxo set state in the database so that the + // utxo set can properly be initialized in the case that the latest utxo data + // had not been flushed to the database yet. + lastFlushHash chainhash.Hash + + // lastFlushTime is the last time that the cache was flushed to the database. + // It is used to determine when to periodically flush the cache to the + // database during initial block download even if the cache isn't full to + // minimize the amount of progress lost if an unclean shutdown occurs. + lastFlushTime time.Time + + // lastEvictionHeight is the block height of the last eviction. When the + // cache reaches the maximum allowed size, entries are evicted based on the + // height of the block that they are contained in, and last eviction height is + // updated to the current height. + lastEvictionHeight uint32 + + // totalEntrySize is the total size of all utxo entries in the cache, in + // bytes. It is updated whenever an entry is added or removed from the cache. + totalEntrySize uint64 + + // The following fields track the total number of cache hits and misses and + // are used to measure the overall cache hit ratio. + hits uint64 + misses uint64 +} + +// UtxoCacheConfig is a descriptor which specifies the utxo cache instance +// configuration. +type UtxoCacheConfig struct { + // DB defines the database which houses the utxo set. + // + // This field is required. + DB database.DB + + // MaxSize defines the maximum allowed size of the utxo cache, in bytes. + // + // This field is required. + MaxSize uint64 +} + +// NewUtxoCache returns a UtxoCache instance using the provided configuration +// details. +func NewUtxoCache(config *UtxoCacheConfig) *UtxoCache { + // Approximate the maximum number of entries allowed in the cache in order + // to set the initial capacity of the entries map. + avgEntrySize := mapOverhead + outpointSize + pointerSize + baseEntrySize + + p2pkhScriptLen + maxEntries := math.Ceil(float64(config.MaxSize) / float64(avgEntrySize)) + + return &UtxoCache{ + db: config.DB, + maxSize: config.MaxSize, + entries: make(map[wire.OutPoint]*UtxoEntry, uint64(maxEntries)), + lastFlushTime: time.Now(), + } +} + +// totalSize returns the total size of the cache on a 64-bit platform, in bytes. +// Note that this only takes the entries map into account, which represents the +// vast majoirty of the memory that the cache uses, and does not include the +// memory usage of other fields in the utxo cache struct. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) totalSize() uint64 { + numEntries := uint64(len(c.entries)) + return mapOverhead*numEntries + outpointSize*numEntries + + pointerSize*numEntries + c.totalEntrySize +} + +// hitRatio returns the percentage of cache lookups that resulted in a cache +// hit. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) hitRatio() float64 { + totalLookups := c.hits + c.misses + if totalLookups == 0 { + return 100 + } + + return float64(c.hits) / float64(totalLookups) * 100 +} + +// addEntry adds the specified output to the cache. The entry being added MUST +// NOT be mutated by the caller after being passed to this function. +// +// Note that this function does not check if the entry is unspendable and +// therefore the caller should ensure that the entry is spendable before adding +// it to the cache. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) addEntry(outpoint wire.OutPoint, entry *UtxoEntry) error { + // Attempt to get an existing entry from the cache. + cachedEntry := c.entries[outpoint] + + // If an existing entry does not exist, the added entry should be marked as + // modified and fresh. + if cachedEntry == nil { + entry.state |= utxoStateModified | utxoStateFresh + } + + // Add the entry to the cache. In the case that an entry already exists, the + // existing entry is overwritten. All fields are overwritten because it's + // possible (although extremely unlikely) that the existing entry is being + // replaced by a different transaction with the same hash. This is allowed so + // long as the previous transaction is fully spent. + c.entries[outpoint] = entry + + // Update the total entry size of the cache. + if cachedEntry != nil { + c.totalEntrySize -= cachedEntry.size() + } + c.totalEntrySize += entry.size() + + return nil +} + +// AddEntry adds the specified output to the cache. The entry being added MUST +// NOT be mutated by the caller after being passed to this function. +// +// Note that this function does not check if the entry is unspendable and +// therefore the caller should ensure that the entry is spendable before adding +// it to the cache. +// +// This function is safe for concurrent access. +func (c *UtxoCache) AddEntry(outpoint wire.OutPoint, entry *UtxoEntry) error { + c.cacheLock.Lock() + err := c.addEntry(outpoint, entry) + c.cacheLock.Unlock() + return err +} + +// spendEntry marks the specified output as spent. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) spendEntry(outpoint wire.OutPoint) { + // Attempt to get an existing entry from the cache. + cachedEntry := c.entries[outpoint] + + // If the entry is nil or already spent, return immediately. + if cachedEntry == nil || cachedEntry.IsSpent() { + return + } + + // If the entry is fresh, and is now being spent, it can safely be removed. + // This is an optimization to skip writing to the database for outputs that + // are added and spent in between flushes to the database. + if cachedEntry.isFresh() { + // The entry in the map is marked as nil rather than deleting it so that + // subsequent lookups for the outpoint will still result in a cache hit and + // avoid querying the database. + c.entries[outpoint] = nil + c.totalEntrySize -= cachedEntry.size() + return + } + + // Mark the output as spent and modified. + cachedEntry.Spend() +} + +// SpendEntry marks the specified output as spent. +// +// This function is safe for concurrent access. +func (c *UtxoCache) SpendEntry(outpoint wire.OutPoint) { + c.cacheLock.Lock() + c.spendEntry(outpoint) + c.cacheLock.Unlock() +} + +// fetchEntry returns the specified transaction output from the utxo set. If +// the output exists in the cache, it is returned immediately. Otherwise, it +// uses an existing database transaction to fetch the output from the database, +// cache it, and return it to the caller. A cloned copy of the entry is +// returned so it can safely be mutated by the caller without invalidating the +// cache. +// +// When there is no entry for the provided output, nil will be returned for both +// the entry and the error. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) fetchEntry(dbTx database.Tx, outpoint wire.OutPoint) (*UtxoEntry, error) { + // If the cache already has the entry, return it immediately. A cloned copy + // of the entry is returned so it can safely be mutated by the caller without + // invalidating the cache. + if entry, found := c.entries[outpoint]; found { + c.hits++ + return entry.Clone(), nil + } + + // Increment cache misses. + c.misses++ + + // Fetch the entry from the database. + // + // NOTE: Missing entries are not considered an error here and instead + // will result in nil entries in the view. This is intentionally done + // so other code can use the presence of an entry in the view as a way + // to unnecessarily avoid attempting to reload it from the database. + entry, err := dbFetchUtxoEntry(dbTx, outpoint) + if err != nil { + return nil, err + } + + // Update the total entry size of the cache. + if entry != nil { + c.totalEntrySize += entry.size() + } + + // Add the entry to the cache and return it. A cloned copy of the entry is + // returned so it can safely be mutated by the caller without invalidating the + // cache. + c.entries[outpoint] = entry + return entry.Clone(), nil +} + +// FetchEntry returns the specified transaction output from the utxo set. If +// the output exists in the cache, it is returned immediately. Otherwise, it +// uses an existing database transaction to fetch the output from the database, +// cache it, and return it to the caller. A cloned copy of the entry is +// returned so it can safely be mutated by the caller without invalidating the +// cache. +// +// When there is no entry for the provided output, nil will be returned for both +// the entry and the error. +// +// This function is safe for concurrent access. +func (c *UtxoCache) FetchEntry(dbTx database.Tx, outpoint wire.OutPoint) (*UtxoEntry, error) { + c.cacheLock.Lock() + entry, err := c.fetchEntry(dbTx, outpoint) + c.cacheLock.Unlock() + return entry, err +} + +// FetchEntries adds the requested transaction outputs to the provided view. It +// first checks the cache for each output, and if an output does not exist in +// the cache, it will fetch it from the database. +// +// Upon completion of this function, the view will contain an entry for each +// requested outpoint. Spent outputs, or those which otherwise don't exist, +// will result in a nil entry in the view. +// +// This function is safe for concurrent access. +func (c *UtxoCache) FetchEntries(filteredSet viewFilteredSet, view *UtxoViewpoint) error { + c.cacheLock.Lock() + err := c.db.View(func(dbTx database.Tx) error { + for outpoint := range filteredSet { + entry, err := c.fetchEntry(dbTx, outpoint) + if err != nil { + return err + } + + // NOTE: Missing entries are not considered an error here and instead + // will result in nil entries in the view. This is intentionally done + // so other code can use the presence of an entry in the view as a way + // to unnecessarily avoid attempting to reload it from the database. + view.entries[outpoint] = entry + } + + return nil + }) + c.cacheLock.Unlock() + + return err +} + +// Commit updates all entries in the cache based on the state of each entry in +// the provided view. +// +// All entries in the provided view that are marked as modified and spent are +// removed from the view. Additionally, all entries that are added to the cache +// are removed from the provided view. +// +// This function is safe for concurrent access. +func (c *UtxoCache) Commit(view *UtxoViewpoint) error { + c.cacheLock.Lock() + for outpoint, entry := range view.entries { + // If the entry is nil, delete it from the view and continue. + if entry == nil { + delete(view.entries, outpoint) + continue + } + + // If the entry is not modified and not fresh, continue as there is nothing + // to do. + if !entry.isModified() && !entry.isFresh() { + continue + } + + // If the entry is modified and spent, mark it as spent in the cache and + // then delete it from the view. + if entry.isModified() && entry.IsSpent() { + // Mark the entry as spent in the cache. + c.spendEntry(outpoint) + + // Delete the entry from the view. + delete(view.entries, outpoint) + continue + } + + // If we passed all of the conditions above, the entry is modified or fresh, + // but not spent, and should be added to the cache. + err := c.addEntry(outpoint, entry) + if err != nil { + c.cacheLock.Unlock() + return err + } + + // All entries that are added to the cache should be removed from the + // provided view. This is an optimization to allow the cache to take + // ownership of the entry from the view and avoid an additional allocation. + // It is removed from the view to ensure that it is not mutated by the + // caller after being added to the cache. + // + // This does cause the view to refetch the entry if it is requested again + // after being removed. However, this only really occurs during reorgs, + // whereas committing the view to the cache happens with every connected + // block, so this optimizes for the much more common case. + delete(view.entries, outpoint) + } + c.cacheLock.Unlock() + + return nil +} + +// calcEvictionHeight returns the eviction height based on the best height of +// the main chain and the last eviction height. All entries that are contained +// in a block at a height less than the eviction height will be evicted from the +// cache when the cache reaches its maximum allowed size. +// +// Eviction is based on height since the height of the block that an entry is +// contained in is a proxy for how old the utxo is. On average, recent utxos +// are much more likely to be spent in upcoming blocks than older utxos, so the +// strategy used is to evict the oldest utxos in order to maximize the hit ratio +// of the cache. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) calcEvictionHeight(bestHeight uint32) uint32 { + if bestHeight < c.lastEvictionHeight { + return bestHeight + } + + lastEvictionDepth := bestHeight - c.lastEvictionHeight + numBlocksToEvict := math.Ceil(float64(lastEvictionDepth) * evictionPercentage) + return c.lastEvictionHeight + uint32(numBlocksToEvict) +} + +// shouldFlush returns whether or not a flush should be performed. +// +// If the maximum size of the cache has been reached, or if the periodic flush +// interval has been reached, then a flush is required. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) shouldFlush(bestHash *chainhash.Hash) bool { + // No need to flush if the cache has already been flushed through the best + // hash. + if c.lastFlushHash == *bestHash { + return false + } + + // Flush if the max size of the cache has been reached. + if c.totalSize() >= c.maxSize { + return true + } + + // Flush if the periodic flush interval has been reached. + return time.Since(c.lastFlushTime) >= periodicFlushInterval +} + +// flush commits all modified entries to the database and conditionally evicts +// entries. +// +// Entries that are nil or spent are always evicted since they are +// unlikely to be accessed again. Additionally, if the cache has reached its +// maximum size, entries are evicted based on the height of the block that they +// are contained in. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) flush(bestHash *chainhash.Hash, bestHeight uint32, logFlush bool) error { + // If the maximum allowed size of the cache has been reached, determine the + // eviction height. + var evictionHeight uint32 + memUsage := c.totalSize() + if memUsage >= c.maxSize { + evictionHeight = c.calcEvictionHeight(bestHeight) + } + + // Log that a flush is starting and indicate the current memory usage, hit + // ratio, and eviction height. + var hitRatio float64 + var evictionLog string + var preFlushNumEntries int + if logFlush { + preFlushNumEntries = len(c.entries) + memUsageMiB := float64(memUsage) / 1024 / 1024 + memUsagePercent := float64(memUsage) / float64(c.maxSize) * 100 + hitRatio = c.hitRatio() + if evictionHeight != 0 { + evictionLog = fmt.Sprintf(", eviction height: %d", evictionHeight) + } + log.Debugf("UTXO cache flush starting (%d entries, %.2f MiB (%.2f%%), "+ + "%.2f%% hit ratio, height: %d%s)", preFlushNumEntries, memUsageMiB, + memUsagePercent, hitRatio, bestHeight, evictionLog) + } + + // Flush the entries in the cache to the database and update the utxo set + // state in the database. + // + // It is important that the utxo set state is always updated in the same + // database transaction as the utxo set itself so that it is always in sync. + err := c.db.Update(func(dbTx database.Tx) error { + for outpoint, entry := range c.entries { + // Write the entry to the database. + err := dbPutUtxoEntry(dbTx, outpoint, entry) + if err != nil { + return err + } + } + + // Update the utxo set state in the database. + return dbPutUtxoSetState(dbTx, &utxoSetState{ + lastFlushHeight: bestHeight, + lastFlushHash: *bestHash, + }) + }) + if err != nil { + return err + } + + // Update the entries in the cache after flushing to the database. This is + // done after the updates to the database have been successfully committed to + // ensure that an unexpected database error would not leave the cache in an + // inconsistent state. + for outpoint, entry := range c.entries { + // Conditionally evict entries from the cache. Entries that are nil or + // spent are always evicted since they are unlikely to be accessed again. + // Additionally, entries that are contained in a block with a height less + // than the eviction height are evicted. + if entry == nil || entry.IsSpent() || + entry.BlockHeight() < int64(evictionHeight) { + + // Remove the entry from the cache. + delete(c.entries, outpoint) + + // Update the total entry size of the cache. + if entry != nil { + c.totalEntrySize -= entry.size() + } + + continue + } + + // If the entry wasn't removed from the cache, clear the modified and + // fresh flags since it has been updated in the database. + entry.state &^= utxoStateModified + entry.state &^= utxoStateFresh + } + + // Update the last flush on the cache instance now that the flush has been + // completed. + c.lastFlushHash = *bestHash + c.lastFlushTime = time.Now() + + // Update the last eviction height on the cache instance if we evicted just + // now. + if evictionHeight != 0 { + c.lastEvictionHeight = evictionHeight + } + + // Log that the flush has been completed and indicate the updated memory usage + // as it will be reduced due to evicting entries above. + if logFlush { + remainingEntries := len(c.entries) + flushedEntries := preFlushNumEntries - remainingEntries + memUsage = c.totalSize() + memUsageMiB := float64(memUsage) / 1024 / 1024 + memUsagePercent := float64(memUsage) / float64(c.maxSize) * 100 + log.Debugf("UTXO cache flush completed (%d entries flushed, %d entries "+ + "remaining, %.2f MiB (%.2f%%))", flushedEntries, remainingEntries, + memUsageMiB, memUsagePercent) + } + + return nil +} + +// MaybeFlush conditionally flushes the cache to the database. +// +// If the maximum size of the cache has been reached, or if the periodic flush +// interval has been reached, then a flush is required. Additionally, a flush +// can be forced by setting the force flush parameter. +// +// This function is safe for concurrent access. +func (c *UtxoCache) MaybeFlush(bestHash *chainhash.Hash, bestHeight uint32, + forceFlush bool, logFlush bool) error { + + c.cacheLock.Lock() + if forceFlush || c.shouldFlush(bestHash) { + err := c.flush(bestHash, bestHeight, logFlush) + c.cacheLock.Unlock() + return err + } + + c.cacheLock.Unlock() + return nil +} + +// InitUtxoCache initializes the utxo cache by ensuring that the utxo set is +// caught up to the tip of the best chain. +// +// Since the cache is only flushed to the database periodically, the utxo set +// may not be caught up to the tip of the best chain. This function catches the +// utxo set up by replaying all blocks from the block after the block that was +// last flushed to the tip block through the cache. +// +// This function should only be called during initialization. +func (b *BlockChain) InitUtxoCache(tip *blockNode) error { + log.Infof("UTXO cache initializing (max size: %d MiB)...", + b.utxoCache.maxSize/1024/1024) + + // Fetch the utxo set state from the database. + var state *utxoSetState + err := b.db.View(func(dbTx database.Tx) error { + var err error + state, err = dbFetchUtxoSetState(dbTx) + return err + }) + if err != nil { + return err + } + + // If the state is nil, update the state to the tip. This should only be the + // case when starting from a fresh database or a database that has not been + // run with the utxo cache yet. + if state == nil { + state = &utxoSetState{ + lastFlushHeight: uint32(tip.height), + lastFlushHash: tip.hash, + } + err := b.db.Update(func(dbTx database.Tx) error { + return dbPutUtxoSetState(dbTx, state) + }) + if err != nil { + return err + } + } + + // Set the last flush hash and the last eviction height from the saved state + // since that is where we are starting from. + b.utxoCache.lastFlushHash = state.lastFlushHash + b.utxoCache.lastEvictionHeight = state.lastFlushHeight + + // If state is already caught up to the tip, return as there is nothing to do. + if state.lastFlushHash == tip.hash { + log.Info("UTXO cache initialization completed") + return nil + } + + // Find the fork point between the current tip and the last flushed block. + lastFlushedNode := b.index.LookupNode(&state.lastFlushHash) + if lastFlushedNode == nil { + // panic if the last flushed block node does not exist. This should never + // happen unless the database is corrupted. + panicf("last flushed block node hash %v (height %v) does not exist", + state.lastFlushHash, state.lastFlushHeight) + } + fork := b.bestChain.FindFork(lastFlushedNode) + + // Disconnect all of the blocks back to the point of the fork. This entails + // loading the blocks and their associated spent txos from the database and + // using that information to unspend all of the spent txos and remove the + // utxos created by the blocks. In addition, if a block votes against its + // parent, the regular transactions are reconnected. + // + // Note that blocks will only need to be disconnected during initialization + // if an unclean shutdown occurred between a block being disconnected and the + // cache being flushed. Since the cache is always flushed immediately after + // disconnecting a block, this will occur very infrequently. In the typical + // catchup case, the fork node will be the last flushed node itself and this + // loop will be skipped. + view := NewUtxoViewpoint(b.utxoCache) + view.SetBestHash(&tip.hash) + var nextBlockToDetach *dcrutil.Block + n := lastFlushedNode + for n != nil && n != fork { + select { + case <-b.interrupt: + return errInterruptRequested + default: + } + + // Grab the block to detach based on the node. Use the fact that the + // blocks are being detached in reverse order, so the parent of the + // current block being detached is the next one being detached. + block := nextBlockToDetach + if block == nil { + var err error + block, err = b.fetchBlockByNode(n) + if err != nil { + return err + } + } + if n.hash != *block.Hash() { + panicf("detach block node hash %v (height %v) does not match "+ + "previous parent block hash %v", &n.hash, n.height, + block.Hash()) + } + + // Grab the parent of the current block and also save a reference to it + // as the next block to detach so it doesn't need to be loaded again on + // the next iteration. + parent, err := b.fetchBlockByNode(n.parent) + if err != nil { + return err + } + nextBlockToDetach = parent + + // Determine if treasury agenda is active. + isTreasuryEnabled, err := b.isTreasuryAgendaActive(n.parent) + if err != nil { + return err + } + + // Load all of the spent txos for the block from the spend journal. + var stxos []spentTxOut + err = b.db.View(func(dbTx database.Tx) error { + stxos, err = dbFetchSpendJournalEntry(dbTx, block, isTreasuryEnabled) + return err + }) + if err != nil { + return err + } + + // Update the view to unspend all of the spent txos and remove the utxos + // created by the block. Also, if the block votes against its parent, + // reconnect all of the regular transactions. + err = view.disconnectBlock(block, parent, stxos, isTreasuryEnabled) + if err != nil { + return err + } + + // Commit all entries in the view to the utxo cache. All entries in the + // view that are marked as modified and spent are removed from the view. + // Additionally, all entries that are added to the cache are removed from + // the view. + err = b.utxoCache.Commit(view) + if err != nil { + return err + } + + // Conditionally flush the utxo cache to the database. Don't force flush + // since many blocks may be disconnected and connected in quick succession + // when initializing. + err = b.utxoCache.MaybeFlush(&n.hash, uint32(n.height), false, true) + if err != nil { + return err + } + + n = n.parent + } + + // Determine the blocks to attach after the fork point. Each block is added + // to the slice from back to front so they are attached in the appropriate + // order when iterating the slice below. + replayNodes := make([]*blockNode, tip.height-fork.height) + for n := tip; n != nil && n != fork; n = n.parent { + replayNodes[n.height-fork.height-1] = n + } + + // Replay all of the blocks through the cache. + var prevBlockAttached *dcrutil.Block + for i, n := range replayNodes { + select { + case <-b.interrupt: + return errInterruptRequested + default: + } + + // Grab the block to attach based on the node. The parent of the block is + // the previous one that was attached except for the first node being + // attached, which needs to be fetched. + block, err := b.fetchBlockByNode(n) + if err != nil { + return err + } + parent := prevBlockAttached + if i == 0 { + parent, err = b.fetchBlockByNode(n.parent) + if err != nil { + return err + } + } + if n.parent.hash != *parent.Hash() { + panicf("attach block node hash %v (height %v) parent hash %v does "+ + "not match previous parent block hash %v", &n.hash, n.height, + &n.parent.hash, parent.Hash()) + } + + // Store the loaded block as parent of the block in the next iteration. + prevBlockAttached = block + + // Determine if treasury agenda is active. + isTreasuryEnabled, err := b.isTreasuryAgendaActive(n.parent) + if err != nil { + return err + } + + // Update the view to mark all utxos referenced by the block as + // spent and add all transactions being created by this block to it. + // In the case the block votes against the parent, also disconnect + // all of the regular transactions in the parent block. + err = view.connectBlock(b.db, block, parent, nil, isTreasuryEnabled) + if err != nil { + return err + } + + // Commit all entries in the view to the utxo cache. All entries in the + // view that are marked as modified and spent are removed from the view. + // Additionally, all entries that are added to the cache are removed from + // the view. + err = b.utxoCache.Commit(view) + if err != nil { + return err + } + + // Conditionally flush the utxo cache to the database. Don't force flush + // since many blocks may be connected in quick succession when initializing. + err = b.utxoCache.MaybeFlush(&n.hash, uint32(n.height), false, true) + if err != nil { + return err + } + } + + log.Info("UTXO cache initialization completed") + return nil +} + +// ShutdownUtxoCache flushes the utxo cache to the database on shutdown. Since +// the cache is flushed periodically during initial block download and flushed +// after every block is connected after initial block download is complete, +// this flush that occurs during shutdown should finish relatively quickly. +// +// Note that if an unclean shutdown occurs, the cache will still be initialized +// properly when restarted as during initialization it will replay blocks to +// catch up to the tip block if it was not fully flushed before shutting down. +// However, it is still preferred to flush when shutting down versus always +// recovering on startup since it is faster. +// +// This function should only be called during shutdown. +func (b *BlockChain) ShutdownUtxoCache() { + b.chainLock.RLock() + defer b.chainLock.RUnlock() + + tip := b.bestChain.Tip() + + // Force a cache flush and log the flush details. + b.utxoCache.MaybeFlush(&tip.hash, uint32(tip.height), true, true) +} + +// FetchUtxoEntry loads and returns the requested unspent transaction output +// from the point of view of the the main chain tip. +// +// NOTE: Requesting an output for which there is no data will NOT return an +// error. Instead both the entry and the error will be nil. This is done to +// allow pruning of spent transaction outputs. In practice this means the +// caller must check if the returned entry is nil before invoking methods on it. +// +// This function is safe for concurrent access however the returned entry (if +// any) is NOT. +func (b *BlockChain) FetchUtxoEntry(outpoint wire.OutPoint) (*UtxoEntry, error) { + b.chainLock.RLock() + defer b.chainLock.RUnlock() + + var entry *UtxoEntry + err := b.db.View(func(dbTx database.Tx) error { + var err error + entry, err = b.utxoCache.FetchEntry(dbTx, outpoint) + return err + }) + if err != nil { + return nil, err + } + + return entry, nil +} + +// UtxoStats represents unspent output statistics on the current utxo set. +type UtxoStats struct { + Utxos int64 + Transactions int64 + Size int64 + Total int64 + SerializedHash chainhash.Hash +} + +// FetchUtxoStats returns statistics on the current utxo set. +// +// NOTE: During initial block download the utxo stats will lag behind the best +// block that is currently synced since the utxo cache is only flushed to the +// database periodically. After initial block download the utxo stats will +// always be in sync with the best block. +func (b *BlockChain) FetchUtxoStats() (*UtxoStats, error) { + var stats *UtxoStats + err := b.db.View(func(dbTx database.Tx) error { + var err error + stats, err = dbFetchUtxoStats(dbTx) + return err + }) + if err != nil { + return nil, err + } + + return stats, nil +} diff --git a/blockchain/utxocache_test.go b/blockchain/utxocache_test.go new file mode 100644 index 0000000000..4dfddc369f --- /dev/null +++ b/blockchain/utxocache_test.go @@ -0,0 +1,1061 @@ +// Copyright (c) 2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package blockchain + +import ( + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/decred/dcrd/blockchain/stake/v4" + "github.com/decred/dcrd/chaincfg/chainhash" + "github.com/decred/dcrd/database/v2" + "github.com/decred/dcrd/wire" +) + +// Define constants for indicating flags throughout the tests. +const ( + noCoinbase = false + withCoinbase = true + noExpiry = false + withExpiry = true +) + +// outpoint299 returns a test outpoint from block height 299 that can be used +// throughout the tests. +func outpoint299() wire.OutPoint { + return wire.OutPoint{ + Hash: *mustParseHash("e299d2cc5deb5b39d230ad2a6046ff9cc164064f431a2893eb6" + + "28b467d018452"), + Index: 0, + Tree: wire.TxTreeRegular, + } +} + +// entry299 returns a utxo entry from block height 299 that can be used +// throughout the tests. +func entry299() *UtxoEntry { + return &UtxoEntry{ + amount: 58795424, + pkScript: hexToBytes("76a914454017705ab80470d089c7f644e39cc9e0fd308e" + + "88ac"), + blockHeight: 299, + blockIndex: 1, + scriptVersion: 0, + packedFlags: encodeUtxoFlags( + noCoinbase, + noExpiry, + stake.TxTypeRegular, + ), + } +} + +// outpoint1100 returns a test outpoint from block height 1100 that can be used +// throughout the tests. +func outpoint1100() wire.OutPoint { + return wire.OutPoint{ + Hash: *mustParseHash("ce1d0f74440c391d15516015224755a8661e56e796ac25490f3" + + "0ad1081c5d638"), + Index: 1, + Tree: wire.TxTreeRegular, + } +} + +// entry1100 returns a utxo entry from block height 1100 that can be used +// throughout the tests. +func entry1100() *UtxoEntry { + return &UtxoEntry{ + amount: 52454022, + pkScript: hexToBytes("76a9146b65f16ebca9b848158701d5a2eb5124547a2144" + + "88ac"), + blockHeight: 1100, + blockIndex: 1, + scriptVersion: 0, + packedFlags: encodeUtxoFlags( + noCoinbase, + noExpiry, + stake.TxTypeRegular, + ), + } +} + +// outpoint1200 returns a test outpoint from block height 1200 that can be used +// throughout the tests. +func outpoint1200() wire.OutPoint { + return wire.OutPoint{ + Hash: *mustParseHash("72914cae2d4bc75f7777373b7c085c4b92d59f3e059fc7fd39d" + + "ef71c9fe188b5"), + Index: 2, + Tree: wire.TxTreeRegular, + } +} + +// entry1200 returns a utxo entry from block height 1200 that can be used +// throughout the tests. +func entry1200() *UtxoEntry { + return &UtxoEntry{ + amount: 1871749598, + pkScript: hexToBytes("76a9142ec5027abadede723c47b6acdbace3be10b7e937" + + "88ac"), + blockHeight: 1200, + blockIndex: 0, + scriptVersion: 0, + packedFlags: encodeUtxoFlags( + withCoinbase, + noExpiry, + stake.TxTypeRegular, + ), + } +} + +// outpoint85314 returns a test outpoint from block height 85314 that can be +// used throughout the tests. +func outpoint85314() wire.OutPoint { + return wire.OutPoint{ + Hash: *mustParseHash("d3bce77da2747baa85fb7ca4f6f8e123f31cd15ac691b2f8254" + + "3780158587d3a"), + Index: 0, + Tree: wire.TxTreeStake, + } +} + +// entry85314 returns a utxo entry from block height 85314 that can be used +// throughout the tests. +func entry85314() *UtxoEntry { + return &UtxoEntry{ + amount: 4294959555, + pkScript: hexToBytes("ba76a914a13afb81d54c9f8bb0c5e082d56fd563ab9b359" + + "688ac"), + blockHeight: 85314, + blockIndex: 6, + scriptVersion: 0, + packedFlags: encodeUtxoFlags( + noCoinbase, + withExpiry, + stake.TxTypeSStx, + ), + ticketMinOuts: &ticketMinimalOutputs{ + data: hexToBytes("03808efefade57001aba76a914a13afb81d54c9f8bb0c5e08" + + "2d56fd563ab9b359688ac0000206a1e9ac39159847e259c9162405b5f6c8135d" + + "2c7eaf1a375040001000000005800001abd76a91400000000000000000000000" + + "0000000000000000088ac"), + }, + } +} + +// createTestUtxoDatabase creates a test database with the utxo set bucket. +func createTestUtxoDatabase(t *testing.T) database.DB { + t.Helper() + + // Create a test database. + dbPath := filepath.Join(os.TempDir(), t.Name()) + _ = os.RemoveAll(dbPath) + db, err := database.Create("ffldb", dbPath, wire.MainNet) + if err != nil { + t.Fatalf("error creating test database: %v", err) + } + t.Cleanup(func() { + os.RemoveAll(dbPath) + }) + t.Cleanup(func() { + db.Close() + }) + + // Create the utxo set bucket. + err = db.Update(func(dbTx database.Tx) error { + _, err := dbTx.Metadata().CreateBucketIfNotExists(utxoSetBucketName) + return err + }) + if err != nil { + t.Fatalf("error creating utxo bucket: %v", err) + } + + return db +} + +// createTestUtxoCache creates a test utxo cache with the specified entries. +func createTestUtxoCache(t *testing.T, entries map[wire.OutPoint]*UtxoEntry) *UtxoCache { + t.Helper() + + utxoCache := NewUtxoCache(&UtxoCacheConfig{}) + for outpoint, entry := range entries { + // Add the entry to the cache. The entry is cloned before being added so + // that any modifications that the cache makes to the entry are not + // reflected in the provided test entry. + err := utxoCache.AddEntry(outpoint, entry.Clone()) + if err != nil { + t.Fatalf("unexpected error when adding entry: %v", err) + } + + // Set the state of the cached entries based on the provided entries. This + // is allowed for tests to easily simulate entries in the cache that are not + // fresh without having to fetch them from the database. + cachedEntry := utxoCache.entries[outpoint] + if cachedEntry != nil { + cachedEntry.state = entry.state + } + } + return utxoCache +} + +// TestTotalSize validates that the correct number of bytes is returned for the +// size of the utxo cache. +func TestTotalSize(t *testing.T) { + t.Parallel() + + // Create test entries to be used throughout the tests. + outpointRegular := outpoint1200() + entryRegular := entry1200() + outpointTicket := outpoint85314() + entryTicket := entry85314() + + tests := []struct { + name string + entries map[wire.OutPoint]*UtxoEntry + want uint64 + }{{ + name: "without any entries", + entries: map[wire.OutPoint]*UtxoEntry{}, + want: 0, + }, { + name: "with entries", + entries: map[wire.OutPoint]*UtxoEntry{ + outpointRegular: entryRegular, + outpointTicket: entryTicket, + }, + // mapOverhead*numEntries + outpointSize*numEntries + + // pointerSize*numEntries + (first entry: base entry size + len(pkScript)) + + // (second entry: base entry size + len(pkScript) + len(ticketMinOuts.data)) + want: mapOverhead*2 + outpointSize*2 + pointerSize*2 + + (baseEntrySize + 25) + (baseEntrySize + 26 + 99), + }} + + for _, test := range tests { + // Create a utxo cache with the entries specified by the test. + utxoCache := createTestUtxoCache(t, test.entries) + + // Validate that total size returns the expected value. + got := utxoCache.totalSize() + if got != test.want { + t.Errorf("%q: unexpected result -- got %d, want %d", test.name, got, + test.want) + } + } +} + +// TestHitRatio validates that the correct hit ratio is returned based on the +// number of cache hits and misses. +func TestHitRatio(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + hits uint64 + misses uint64 + want float64 + }{{ + name: "no hits or misses", + want: 100, + }, { + name: "all hits, no misses", + hits: 50, + want: 100, + }, { + name: "98.5% hit ratio", + hits: 197, + misses: 3, + want: 98.5, + }} + + for _, test := range tests { + // Create a utxo cache with hits and misses as specified by the test. + utxoCache := NewUtxoCache(&UtxoCacheConfig{}) + utxoCache.hits = test.hits + utxoCache.misses = test.misses + + // Validate that hit ratio returns the expected value. + got := utxoCache.hitRatio() + if got != test.want { + t.Errorf("%q: unexpected result -- got %f, want %f", test.name, got, + test.want) + } + } +} + +// TestAddEntry validates that entries are added to the cache properly under a +// variety of conditions. +func TestAddEntry(t *testing.T) { + t.Parallel() + + // Create test entries to be used throughout the tests. + outpoint := outpoint299() + entry := entry299() + entryModified := entry.Clone() + entryModified.amount++ + entryModified.state |= utxoStateModified + entryFresh := entry.Clone() + entryFresh.state |= utxoStateModified | utxoStateFresh + + tests := []struct { + name string + existingEntries map[wire.OutPoint]*UtxoEntry + outpoint wire.OutPoint + entry *UtxoEntry + wantEntry *UtxoEntry + }{{ + name: "add an entry that does not already exist in the cache", + outpoint: outpoint, + entry: entry, + wantEntry: entryFresh, + }, { + name: "add an entry that overwrites an existing entry", + existingEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint: entry, + }, + outpoint: outpoint, + entry: entryModified, + wantEntry: entryModified, + }} + + for _, test := range tests { + // Create a utxo cache with the existing entries specified by the test. + utxoCache := createTestUtxoCache(t, test.existingEntries) + wantTotalEntrySize := utxoCache.totalEntrySize + + // Attempt to get an existing entry from the cache. If it exists, subtract + // its size from the expected total entry size since it will be overwritten. + existingEntry := utxoCache.entries[test.outpoint] + if existingEntry != nil { + wantTotalEntrySize -= test.entry.size() + } + + // Add the entry specified by the test. + err := utxoCache.AddEntry(test.outpoint, test.entry) + if err != nil { + t.Fatalf("%q: unexpected error when adding entry: %v", test.name, err) + } + wantTotalEntrySize += test.entry.size() + + // Attempt to get the added entry from the cache. + cachedEntry := utxoCache.entries[test.outpoint] + + // Validate that the added entry exists in the cache. + if cachedEntry == nil { + t.Fatalf("%q: expected entry for outpoint %v to exist in the cache", + test.name, test.outpoint) + } + + // Validate that the entry is marked as modified. + if !cachedEntry.isModified() { + t.Fatalf("%q: unexpected modified flag -- got false, want true", + test.name) + } + + // Validate that the cached entry matches the expected entry. + if !reflect.DeepEqual(cachedEntry, test.wantEntry) { + t.Fatalf("%q: mismatched cached entry:\nwant: %+v\n got: %+v\n", + test.name, test.wantEntry, cachedEntry) + } + + // Validate that the total entry size was updated as expected. + if utxoCache.totalEntrySize != wantTotalEntrySize { + t.Fatalf("%q: unexpected total entry size -- got %v, want %v", test.name, + utxoCache.totalEntrySize, wantTotalEntrySize) + } + } +} + +// TestSpendEntry validates that entries in the cache are properly updated when +// being spent under a variety of conditions. +func TestSpendEntry(t *testing.T) { + t.Parallel() + + // Create test entries to be used throughout the tests. + outpoint := outpoint299() + entry := entry299() + entryFresh := entry.Clone() + entryFresh.state |= utxoStateModified | utxoStateFresh + entrySpent := entry.Clone() + entrySpent.Spend() + + tests := []struct { + name string + existingEntries map[wire.OutPoint]*UtxoEntry + outpoint wire.OutPoint + entry *UtxoEntry + }{{ + name: "spend an entry that does not exist in the cache", + outpoint: outpoint, + entry: entry, + }, { + name: "spend an entry that exists in the cache but is already spent", + existingEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint: entrySpent, + }, + outpoint: outpoint, + entry: entrySpent, + }, { + name: "spend an entry that exists in the cache and is fresh", + existingEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint: entryFresh, + }, + outpoint: outpoint, + entry: entryFresh, + }, { + name: "spend an entry that exists in the cache and is not fresh", + existingEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint: entry, + }, + outpoint: outpoint, + entry: entry, + }} + + for _, test := range tests { + // Create a utxo cache with the existing entries specified by the test. + utxoCache := createTestUtxoCache(t, test.existingEntries) + wantTotalEntrySize := utxoCache.totalEntrySize + + // Attempt to get an existing entry from the cache. + entry := utxoCache.entries[test.outpoint] + var entryAlreadySpent bool + if entry != nil { + entryAlreadySpent = entry.IsSpent() + } + + // Spend the entry specified by the test. + utxoCache.SpendEntry(test.outpoint) + + // If the existing entry was nil or spent, continue as there is nothing + // else to validate. + if entry == nil || entryAlreadySpent { + continue + } + + // If the entry is fresh, validate that it was removed from the cache when + // spent. + if entry.isFresh() { + wantTotalEntrySize -= test.entry.size() + if utxoCache.entries[test.outpoint] != nil { + t.Fatalf("%q: entry for outpoint %v was not removed from the cache", + test.name, test.outpoint) + } + } + + // Validate that the total entry size was updated as expected. + if utxoCache.totalEntrySize != wantTotalEntrySize { + t.Fatalf("%q: unexpected total entry size -- got %v, want %v", test.name, + utxoCache.totalEntrySize, wantTotalEntrySize) + } + + // If entry is not fresh, validate that it still exists in the cache and is + // now marked as spent. + if !entry.isFresh() { + cachedEntry := utxoCache.entries[test.outpoint] + if cachedEntry == nil || !cachedEntry.IsSpent() { + t.Fatalf("%q: expected entry for outpoint %v to exist in the cache "+ + "and be marked spent", test.name, test.outpoint) + } + } + } +} + +// TestFetchEntry validates that fetch entry returns the correct entry under a +// variety of conditions. +func TestFetchEntry(t *testing.T) { + t.Parallel() + + // Create a test database. + db := createTestUtxoDatabase(t) + + // Create test entries to be used throughout the tests. + outpoint := outpoint299() + entry := entry299() + entryModified := entry.Clone() + entryModified.state |= utxoStateModified + + tests := []struct { + name string + cachedEntries map[wire.OutPoint]*UtxoEntry + dbEntries map[wire.OutPoint]*UtxoEntry + outpoint wire.OutPoint + cacheHit bool + wantEntry *UtxoEntry + }{{ + name: "entry is not in the cache or the database", + outpoint: outpoint, + }, { + name: "entry is in the cache", + cachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint: entry, + }, + outpoint: outpoint, + cacheHit: true, + wantEntry: entry, + }, { + name: "entry is not in the cache but is in the database", + dbEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint: entryModified, + }, + outpoint: outpoint, + wantEntry: entry, + }} + + for _, test := range tests { + // Create a utxo cache with the cached entries specified by the test. + utxoCache := createTestUtxoCache(t, test.cachedEntries) + wantTotalEntrySize := utxoCache.totalEntrySize + + // Add entries specified by the test to the test database. + err := db.Update(func(dbTx database.Tx) error { + for outpoint, entry := range test.dbEntries { + err := dbPutUtxoEntry(dbTx, outpoint, entry) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatalf("%q: unexpected error adding entries to test db: %v", test.name, + err) + } + + // Attempt to fetch the entry for the outpoint specified by the test. + var entry *UtxoEntry + err = db.View(func(dbTx database.Tx) error { + var err error + entry, err = utxoCache.FetchEntry(dbTx, test.outpoint) + return err + }) + if err != nil { + t.Fatalf("%q: unexpected error fetching entry: %v", test.name, err) + } + + // Ensure that the fetched entry matches the expected entry. + if !reflect.DeepEqual(entry, test.wantEntry) { + t.Fatalf("%q: mismatched entry:\nwant: %+v\n got: %+v\n", test.name, + test.wantEntry, entry) + } + + // Ensure that the entry is now cached. + cachedEntry := utxoCache.entries[test.outpoint] + if !reflect.DeepEqual(cachedEntry, test.wantEntry) { + t.Fatalf("%q: mismatched cached entry:\nwant: %+v\n got: %+v\n", + test.name, test.wantEntry, cachedEntry) + } + + // Validate the cache hits and misses counts. + if test.cacheHit && utxoCache.hits != 1 { + t.Fatalf("%q: unexpected cache hits -- got %v, want 1", test.name, + utxoCache.hits) + } + if !test.cacheHit && utxoCache.misses != 1 { + t.Fatalf("%q: unexpected cache misses -- got %v, want 1", test.name, + utxoCache.misses) + } + + // Validate that the total entry size was updated as expected. + if !test.cacheHit && cachedEntry != nil { + wantTotalEntrySize += cachedEntry.size() + } + if utxoCache.totalEntrySize != wantTotalEntrySize { + t.Fatalf("%q: unexpected total entry size -- got %v, want %v", test.name, + utxoCache.totalEntrySize, wantTotalEntrySize) + } + } +} + +// TestFetchEntries validates that the provided view is populated with the +// requested entries as expecetd. +func TestFetchEntries(t *testing.T) { + t.Parallel() + + // Create a test database. + db := createTestUtxoDatabase(t) + + // Create test entries to be used throughout the tests. + outpoint299 := outpoint299() + outpoint1100 := outpoint1100() + entry1100 := entry1100() + outpoint1200 := outpoint1200() + entry1200 := entry1200() + entry1200Modified := entry1200.Clone() + entry1200Modified.state |= utxoStateModified + + tests := []struct { + name string + cachedEntries map[wire.OutPoint]*UtxoEntry + dbEntries map[wire.OutPoint]*UtxoEntry + filteredSet viewFilteredSet + wantEntries map[wire.OutPoint]*UtxoEntry + }{{ + name: "entries are fetched from the cache and the database", + cachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1100: entry1100, + }, + dbEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1200: entry1200Modified, + }, + filteredSet: viewFilteredSet{ + outpoint299: struct{}{}, + outpoint1100: struct{}{}, + outpoint1200: struct{}{}, + }, + wantEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint299: nil, + outpoint1100: entry1100, + outpoint1200: entry1200, + }, + }} + + for _, test := range tests { + // Create a utxo cache with the cached entries specified by the test. + utxoCache := createTestUtxoCache(t, test.cachedEntries) + utxoCache.db = db + + // Add entries specified by the test to the test database. + err := db.Update(func(dbTx database.Tx) error { + for outpoint, entry := range test.dbEntries { + err := dbPutUtxoEntry(dbTx, outpoint, entry) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatalf("%q: unexpected error adding entries to test db: %v", test.name, + err) + } + + // Fetch the entries requested by the test and add them to a view. + view := NewUtxoViewpoint(utxoCache) + err = utxoCache.FetchEntries(test.filteredSet, view) + if err != nil { + t.Fatalf("%q: unexpected error fetching entries for view: %v", test.name, + err) + } + + // Ensure that the fetched entries match the expected entries. + if !reflect.DeepEqual(view.entries, test.wantEntries) { + t.Fatalf("%q: mismatched entries:\nwant: %+v\n got: %+v\n", test.name, + test.wantEntries, view.entries) + } + } +} + +// TestCommit validates that all entries in both the cache and the provided view +// are updated appropriately when committing the provided view to the cache. +func TestCommit(t *testing.T) { + t.Parallel() + + // Create test entries to be used throughout the tests. + outpoint299 := outpoint299() + outpoint1100 := outpoint1100() + entry1100Unmodified := entry1100() + outpoint1200 := outpoint1200() + entry1200 := entry1200() + entry1200Spent := entry1200.Clone() + entry1200Spent.Spend() + outpoint85314 := outpoint85314() + entry85314Modified := entry85314() + entry85314Modified.state |= utxoStateModified + + tests := []struct { + name string + viewEntries map[wire.OutPoint]*UtxoEntry + cachedEntries map[wire.OutPoint]*UtxoEntry + wantViewEntries map[wire.OutPoint]*UtxoEntry + wantCachedEntries map[wire.OutPoint]*UtxoEntry + }{{ + name: "view contains nil, unmodified, spent, and modified entries", + viewEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint299: nil, + outpoint1100: entry1100Unmodified, + outpoint1200: entry1200Spent, + outpoint85314: entry85314Modified, + }, + cachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1200: entry1200, + }, + // outpoint299 is removed from the view since the entry is nil. + // entry1100Unmodified remains in the view since it is unmodified. + // entry1200Spent is removed from the view since the entry is spent. + // entry85314Modified is removed from the view since it is modified and + // added to the cache. + wantViewEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1100: entry1100Unmodified, + }, + // entry1200Spent remains in the cache but is now spent. + // entry85314Modified is added to the cache. + wantCachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1200: entry1200Spent, + outpoint85314: entry85314Modified, + }, + }} + + for _, test := range tests { + // Create a utxo cache with the cached entries specified by the test. + utxoCache := createTestUtxoCache(t, test.cachedEntries) + + // Create a utxo cache with the view entries specified by the test. + view := &UtxoViewpoint{ + cache: utxoCache, + entries: test.viewEntries, + } + + // Commit the view to the cache. + err := utxoCache.Commit(view) + if err != nil { + t.Fatalf("%q: unexpected error committing view to the cache: %v", + test.name, err) + } + + // Validate the cached entries after committing. + if !reflect.DeepEqual(utxoCache.entries, test.wantCachedEntries) { + t.Fatalf("%q: mismatched cached entries:\nwant: %+v\n got: %+v\n", + test.name, test.wantCachedEntries, utxoCache.entries) + } + + // Validate the view entries after committing. + if !reflect.DeepEqual(view.entries, test.wantViewEntries) { + t.Fatalf("%q: mismatched view entries:\nwant: %+v\n got: %+v\n", + test.name, test.wantViewEntries, view.entries) + } + } +} + +// TestCalcEvictionHeight validates that the correct eviction height is returned +// based on the provided best height and the last eviction height. +func TestCalcEvictionHeight(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + lastEvictionHeight uint32 + bestHeight uint32 + want uint32 + }{{ + name: "no last eviction", + bestHeight: 100, + want: 15, + }, { + name: "best height less than last eviction height", + lastEvictionHeight: 101, + bestHeight: 100, + want: 100, + }, { + name: "best height greater than last eviction height", + lastEvictionHeight: 99, + bestHeight: 200, + want: 115, + }} + + for _, test := range tests { + // Create a utxo cache with the last eviction height as specified by the + // test. + utxoCache := NewUtxoCache(&UtxoCacheConfig{}) + utxoCache.lastEvictionHeight = test.lastEvictionHeight + + // Validate that calc eviction height returns the expected value. + got := utxoCache.calcEvictionHeight(test.bestHeight) + if got != test.want { + t.Errorf("%q: unexpected result -- got %d, want %d", test.name, got, + test.want) + } + } +} + +// TestShouldFlush validates that it is correctly determined whether or not a +// flush should be performed given various conditions. +func TestShouldFlush(t *testing.T) { + t.Parallel() + + // Create test hashes to be used throughout the tests. + block1000Hash := mustParseHash("0000000000004740ad140c86753f9295e09f9cc81b1" + + "bb75d7f5552aeeedb7012") + block2000Hash := mustParseHash("0000000000000c8a886e3f7c32b1bb08422066dcfd0" + + "08de596471f11a5aff475") + + tests := []struct { + name string + totalEntrySize uint64 + maxSize uint64 + lastFlushTime time.Time + lastFlushHash *chainhash.Hash + bestHash *chainhash.Hash + want bool + }{{ + name: "already flushed through the best hash", + totalEntrySize: 100, + maxSize: 1000, + lastFlushTime: time.Now(), + lastFlushHash: block1000Hash, + bestHash: block1000Hash, + want: false, + }, { + name: "less than max size and periodic duration not reached", + totalEntrySize: 100, + maxSize: 1000, + lastFlushTime: time.Now(), + lastFlushHash: block1000Hash, + bestHash: block2000Hash, + want: false, + }, { + name: "equal to max size", + totalEntrySize: 1000, + maxSize: 1000, + lastFlushTime: time.Now(), + lastFlushHash: block1000Hash, + bestHash: block2000Hash, + want: true, + }, { + name: "greater than max size", + totalEntrySize: 1001, + maxSize: 1000, + lastFlushTime: time.Now(), + lastFlushHash: block1000Hash, + bestHash: block2000Hash, + want: true, + }, { + name: "less than max size but periodic duration reached", + totalEntrySize: 100, + maxSize: 1000, + lastFlushTime: time.Now().Add(periodicFlushInterval * -1), + lastFlushHash: block1000Hash, + bestHash: block2000Hash, + want: true, + }} + + for _, test := range tests { + // Create a utxo cache and set the field values as specified by the test. + utxoCache := NewUtxoCache(&UtxoCacheConfig{ + MaxSize: test.maxSize, + }) + utxoCache.totalEntrySize = test.totalEntrySize + utxoCache.lastFlushTime = test.lastFlushTime + utxoCache.lastFlushHash = *test.lastFlushHash + + // Validate that should flush returns the expected value. + got := utxoCache.shouldFlush(test.bestHash) + if got != test.want { + t.Errorf("%q: unexpected result -- got %v, want %v", test.name, got, + test.want) + } + } +} + +// TestMaybeFlush validates that the cache is properly flushed to the database +// under a variety of conditions. +func TestMaybeFlush(t *testing.T) { + t.Parallel() + + // Create a test database. + db := createTestUtxoDatabase(t) + + // Create test hashes to be used throughout the tests. + block1000Hash := mustParseHash("0000000000004740ad140c86753f9295e09f9cc81b1" + + "bb75d7f5552aeeedb7012") + block2000Hash := mustParseHash("0000000000000c8a886e3f7c32b1bb08422066dcfd0" + + "08de596471f11a5aff475") + + // entry299Fresh is from block height 299 and is modified and fresh. + outpoint299 := outpoint299() + entry299Fresh := entry299() + entry299Fresh.state |= utxoStateModified | utxoStateFresh + + // entry299Unmodified is from block height 299 and is unmodified. + entry299Unmodified := entry299() + + // entry1100Spent is from block height 1100 and is modified and spent. + outpoint1100 := outpoint1100() + entry1100Spent := entry1100() + entry1100Spent.Spend() + + // entry1100Modified is from block height 1100 and is modified and unspent. + entry1100Modified := entry1100() + entry1100Modified.state |= utxoStateModified + + // entry1100Unmodified is from block height 1100 and is unspent and + // unmodified. + entry1100Unmodified := entry1100() + + // entry1200Fresh is from block height 1200 and is modified and fresh. + outpoint1200 := outpoint1200() + entry1200Fresh := entry1200() + entry1200Fresh.state |= utxoStateModified | utxoStateFresh + + // entry1200Unmodified is from block height 1200 and is unmodified. + entry1200Unmodified := entry1200() + + tests := []struct { + name string + maxSize uint64 + lastEvictionHeight uint32 + lastFlushHash *chainhash.Hash + bestHash *chainhash.Hash + bestHeight uint32 + forceFlush bool + cachedEntries map[wire.OutPoint]*UtxoEntry + dbEntries map[wire.OutPoint]*UtxoEntry + wantCachedEntries map[wire.OutPoint]*UtxoEntry + wantDbEntries map[wire.OutPoint]*UtxoEntry + wantLastEvictionHeight uint32 + wantLastFlushHash *chainhash.Hash + wantUpdatedLastFlushTime bool + }{{ + name: "flush not required", + maxSize: 1000, + lastEvictionHeight: 0, + lastFlushHash: block1000Hash, + bestHash: block2000Hash, + bestHeight: 2000, + cachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint299: entry299Fresh, + outpoint1100: entry1100Spent, + outpoint1200: entry1200Fresh, + }, + dbEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1100: entry1100Modified, + }, + // The cache should remain unchanged since a flush is not required. + wantCachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint299: entry299Fresh, + outpoint1100: entry1100Spent, + outpoint1200: entry1200Fresh, + }, + // The db should remain unchanged since a flush is not required. + wantDbEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1100: entry1100Unmodified, + }, + wantLastEvictionHeight: 0, + wantLastFlushHash: block1000Hash, + wantUpdatedLastFlushTime: false, + }, { + name: "all entries flushed, some entries evicted", + maxSize: 0, + lastEvictionHeight: 0, + lastFlushHash: block1000Hash, + bestHash: block2000Hash, + bestHeight: 2000, + cachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint299: entry299Fresh, + outpoint1100: entry1100Spent, + outpoint1200: entry1200Fresh, + }, + dbEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1100: entry1100Modified, + }, + // entry299Fresh should be evicted from the cache due to its height. + // entry1100Spent should be evicted since it is spent. + // entry1200Fresh should remain in the cache but should now be unmodified. + wantCachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1200: entry1200Unmodified, + }, + // entry299Unmodified should be added to the db during the flush. + // entry1100Unmodified should be removed from the db since it now spent. + // entry1200Unmodified should be added to the db during the flush. + wantDbEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint299: entry299Unmodified, + outpoint1200: entry1200Unmodified, + }, + wantLastEvictionHeight: 300, + wantLastFlushHash: block2000Hash, + wantUpdatedLastFlushTime: true, + }} + + for _, test := range tests { + // Create a utxo cache with the cached entries specified by the test. + utxoCache := createTestUtxoCache(t, test.cachedEntries) + utxoCache.db = db + utxoCache.maxSize = test.maxSize + utxoCache.lastEvictionHeight = test.lastEvictionHeight + utxoCache.lastFlushHash = *test.lastFlushHash + origLastFlushTime := utxoCache.lastFlushTime + + // Add entries specified by the test to the test database. + err := db.Update(func(dbTx database.Tx) error { + for outpoint, entry := range test.dbEntries { + err := dbPutUtxoEntry(dbTx, outpoint, entry) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatalf("%q: unexpected error adding entries to test db: %v", test.name, + err) + } + + // Conditionally flush the cache based on the test parameters. + err = utxoCache.MaybeFlush(test.bestHash, test.bestHeight, test.forceFlush, + false) + if err != nil { + t.Fatalf("%q: unexpected error flushing cache: %v", test.name, err) + } + + // Validate that the cached entries match the expected entries after + // eviction. + if !reflect.DeepEqual(utxoCache.entries, test.wantCachedEntries) { + t.Fatalf("%q: mismatched cached entries:\nwant: %+v\n got: %+v\n", + test.name, test.wantCachedEntries, utxoCache.entries) + } + + // Validate that the db entries match the expected entries after flushing + // the cache. + dbEntries := make(map[wire.OutPoint]*UtxoEntry) + err = db.View(func(dbTx database.Tx) error { + for outpoint := range test.cachedEntries { + entry, err := dbFetchUtxoEntry(dbTx, outpoint) + if err != nil { + return err + } + + if entry != nil { + dbEntries[outpoint] = entry + } + } + return nil + }) + if err != nil { + t.Fatalf("%q: unexpected error fetching entries from test db: %v", + test.name, err) + } + if !reflect.DeepEqual(dbEntries, test.wantDbEntries) { + t.Fatalf("%q: mismatched db entries:\nwant: %+v\n got: %+v\n", test.name, + test.wantDbEntries, dbEntries) + } + + // Validate that the last flush hash and time have been updated as expexted. + if utxoCache.lastFlushHash != *test.wantLastFlushHash { + t.Fatalf("%q: unexpected last flush hash -- got %x, want %x", test.name, + utxoCache.lastFlushHash, *test.wantLastFlushHash) + } + updatedLastFlushTime := utxoCache.lastFlushTime != origLastFlushTime + if updatedLastFlushTime != test.wantUpdatedLastFlushTime { + t.Fatalf("%q: unexpected updated last flush time -- got %v, want %v", + test.name, updatedLastFlushTime, test.wantUpdatedLastFlushTime) + } + + // Validate the updated last eviction height. + if utxoCache.lastEvictionHeight != test.wantLastEvictionHeight { + t.Fatalf("%q: unexpected last eviction height -- got %d, want %d", + test.name, utxoCache.lastEvictionHeight, test.wantLastEvictionHeight) + } + + // Validate the updated total entry size of the cache. + wantTotalEntrySize := uint64(0) + for _, entry := range test.wantCachedEntries { + wantTotalEntrySize += entry.size() + } + if utxoCache.totalEntrySize != wantTotalEntrySize { + t.Fatalf("%q: unexpected total entry size -- got %v, want %v", test.name, + utxoCache.totalEntrySize, wantTotalEntrySize) + } + } +} diff --git a/blockchain/utxoentry.go b/blockchain/utxoentry.go index 98b1a6645f..ac33452946 100644 --- a/blockchain/utxoentry.go +++ b/blockchain/utxoentry.go @@ -4,57 +4,73 @@ package blockchain -import "github.com/decred/dcrd/blockchain/stake/v4" +import ( + "github.com/decred/dcrd/blockchain/stake/v4" +) + +const ( + // baseEntrySize is the base size of a utxo entry on a 64-bit platform, + // excluding the contents of the script and ticket minimal outputs. It is + // equivalent to what unsafe.Sizeof(UtxoEntry{}) returns on a 64-bit platform. + baseEntrySize = 56 +) -// utxoFlags defines additional information and state for a transaction output -// in a utxo view. The bit representation is: +// utxoState defines the in-memory state of a utxo entry. +// +// The bit representation is: +// bit 0 - transaction output has been spent +// bit 1 - transaction output has been modified since it was loaded +// bit 2 - transaction output is fresh +// bits 3-7 - unused +type utxoState uint8 + +const ( + // utxoStateSpent indicates that a txout is spent. + utxoStateSpent utxoState = 1 << iota + + // utxoStateModified indicates that a txout has been modified since it was + // loaded. + utxoStateModified + + // utxoStateFresh indicates that a txout is fresh, which means that it exists + // in the utxo cache but does not exist in the underlying database. + utxoStateFresh +) + +// utxoFlags defines additional information for the containing transaction of a +// utxo entry. +// +// The bit representation is: // bit 0 - containing transaction is a coinbase -// bit 1 - transaction output has been spent -// bit 2 - transaction output has been modified since it was loaded -// bit 3 - containing transaction has an expiry -// bits 4-7 - transaction type +// bit 1 - containing transaction has an expiry +// bits 2-5 - transaction type type utxoFlags uint8 const ( // utxoFlagCoinBase indicates that a txout was contained in a coinbase tx. utxoFlagCoinBase utxoFlags = 1 << iota - // utxoFlagSpent indicates that a txout is spent. - utxoFlagSpent - - // utxoFlagModified indicates that a txout has been modified since it was - // loaded. - utxoFlagModified - // utxoFlagHasExpiry indicates that a txout was contained in a tx that // included an expiry. utxoFlagHasExpiry ) const ( - // utxoFlagTxTypeBitmask describes the bitmask that yields bits 4-7 from + // utxoFlagTxTypeBitmask describes the bitmask that yields bits 2-5 from // utxoFlags. - utxoFlagTxTypeBitmask = 0xf0 + utxoFlagTxTypeBitmask = 0x3c // utxoFlagTxTypeShift is the number of bits to shift utxoFlags to the right // to yield the correct integer value after applying the bitmask with AND. - utxoFlagTxTypeShift = 4 + utxoFlagTxTypeShift = 2 ) // encodeUtxoFlags returns utxoFlags representing the passed parameters. -func encodeUtxoFlags(coinbase bool, spent bool, modified bool, hasExpiry bool, - txType stake.TxType) utxoFlags { - +func encodeUtxoFlags(coinbase bool, hasExpiry bool, txType stake.TxType) utxoFlags { packedFlags := utxoFlags(txType) << utxoFlagTxTypeShift if coinbase { packedFlags |= utxoFlagCoinBase } - if spent { - packedFlags |= utxoFlagSpent - } - if modified { - packedFlags |= utxoFlagModified - } if hasExpiry { packedFlags |= utxoFlagHasExpiry } @@ -100,16 +116,34 @@ type UtxoEntry struct { blockIndex uint32 scriptVersion uint16 - // packedFlags contains additional info about the output as defined by - // utxoFlags. This approach is used in order to reduce memory usage since - // there will be a lot of these in memory. + // state contains info for the in-memory state of the output as defined by + // utxoState. + state utxoState + + // packedFlags contains additional info for the containing transaction of the + // output as defined by utxoFlags. This approach is used in order to reduce + // memory usage since there will be a lot of these in memory. packedFlags utxoFlags } +// size returns the number of bytes that the entry uses on a 64-bit platform. +func (entry *UtxoEntry) size() uint64 { + size := baseEntrySize + len(entry.pkScript) + if entry.ticketMinOuts != nil { + size += len(entry.ticketMinOuts.data) + } + return uint64(size) +} + // isModified returns whether or not the output has been modified since it was // loaded. func (entry *UtxoEntry) isModified() bool { - return entry.packedFlags&utxoFlagModified == utxoFlagModified + return entry.state&utxoStateModified == utxoStateModified +} + +// isFresh returns whether or not the output is fresh. +func (entry *UtxoEntry) isFresh() bool { + return entry.state&utxoStateFresh == utxoStateFresh } // IsCoinBase returns whether or not the output was contained in a coinbase @@ -121,7 +155,7 @@ func (entry *UtxoEntry) IsCoinBase() bool { // IsSpent returns whether or not the output has been spent based upon the // current state of the unspent transaction output view it was obtained from. func (entry *UtxoEntry) IsSpent() bool { - return entry.packedFlags&utxoFlagSpent == utxoFlagSpent + return entry.state&utxoStateSpent == utxoStateSpent } // HasExpiry returns whether or not the output was contained in a transaction @@ -157,7 +191,7 @@ func (entry *UtxoEntry) Spend() { } // Mark the output as spent and modified. - entry.packedFlags |= utxoFlagSpent | utxoFlagModified + entry.state |= utxoStateSpent | utxoStateModified } // Amount returns the amount of the output. @@ -203,6 +237,7 @@ func (entry *UtxoEntry) Clone() *UtxoEntry { blockHeight: entry.blockHeight, blockIndex: entry.blockIndex, scriptVersion: entry.scriptVersion, + state: entry.state, packedFlags: entry.packedFlags, } diff --git a/blockchain/utxoentry_test.go b/blockchain/utxoentry_test.go index 2184d9d8d1..94968b69cc 100644 --- a/blockchain/utxoentry_test.go +++ b/blockchain/utxoentry_test.go @@ -20,40 +20,31 @@ func TestEncodeUtxoFlags(t *testing.T) { tests := []struct { name string coinbase bool - spent bool - modified bool hasExpiry bool txType stake.TxType want utxoFlags }{{ name: "no flags set, regular tx", coinbase: false, - spent: false, - modified: false, hasExpiry: false, txType: stake.TxTypeRegular, want: 0x00, }, { name: "coinbase, has expiry, vote tx", coinbase: true, - spent: false, - modified: false, hasExpiry: true, txType: stake.TxTypeSSGen, - want: 0x29, + want: 0x0b, }, { - name: "spent, modified, has expiry, ticket tx", + name: "has expiry, ticket tx", coinbase: false, - spent: true, - modified: true, hasExpiry: true, txType: stake.TxTypeSStx, - want: 0x1e, + want: 0x06, }} for _, test := range tests { - got := encodeUtxoFlags(test.coinbase, test.spent, test.modified, - test.hasExpiry, test.txType) + got := encodeUtxoFlags(test.coinbase, test.hasExpiry, test.txType) if got != test.want { t.Errorf("%q: unexpected result -- got %x, want %x", test.name, got, test.want) @@ -104,9 +95,10 @@ func TestUtxoEntry(t *testing.T) { tests := []struct { name string - coinbase bool spent bool modified bool + fresh bool + coinbase bool expiry bool txType stake.TxType amount int64 @@ -116,6 +108,7 @@ func TestUtxoEntry(t *testing.T) { scriptVersion uint16 ticketMinOuts *ticketMinimalOutputs deserializedTicketMinOuts []*stake.MinimalOutput + size uint64 }{{ name: "coinbase output", coinbase: true, @@ -126,8 +119,11 @@ func TestUtxoEntry(t *testing.T) { blockHeight: 54321, blockIndex: 0, scriptVersion: 0, + // baseEntrySize + len(pkScript). + size: baseEntrySize + 25, }, { name: "ticket submission output", + fresh: true, expiry: true, txType: stake.TxTypeSStx, amount: 4294959555, @@ -158,6 +154,8 @@ func TestUtxoEntry(t *testing.T) { Value: 0, Version: 0, }}, + // baseEntrySize + len(pkScript) + len(ticketMinOuts.data). + size: baseEntrySize + 26 + 99, }} for _, test := range tests { @@ -171,117 +169,143 @@ func TestUtxoEntry(t *testing.T) { scriptVersion: test.scriptVersion, packedFlags: encodeUtxoFlags( test.coinbase, - test.spent, - test.modified, test.expiry, test.txType, ), } + // Set state flags given the parameters for the current test. + if test.spent { + entry.state |= utxoStateSpent + } + if test.modified { + entry.state |= utxoStateModified + } + if test.fresh { + entry.state |= utxoStateFresh + } + + // Validate the size of the entry. + size := entry.size() + if size != test.size { + t.Fatalf("%q: unexpected size -- got %v, want %v", test.name, size, + test.size) + } + + // Validate the spent flag. + isSpent := entry.IsSpent() + if isSpent != test.spent { + t.Fatalf("%q: unexpected spent flag -- got %v, want %v", test.name, + isSpent, test.spent) + } + // Validate the modified flag. isModified := entry.isModified() if isModified != test.modified { - t.Fatalf("unexpected modified flag -- got %v, want %v", isModified, - test.modified) + t.Fatalf("%q: unexpected modified flag -- got %v, want %v", test.name, + isModified, test.modified) + } + + // Validate the fresh flag. + isFresh := entry.isFresh() + if isFresh != test.fresh { + t.Fatalf("%q: unexpected fresh flag -- got %v, want %v", test.name, + isFresh, test.fresh) } // Validate the coinbase flag. isCoinBase := entry.IsCoinBase() if isCoinBase != test.coinbase { - t.Fatalf("unexpected coinbase flag -- got %v, want %v", isCoinBase, - test.coinbase) - } - - // Validate the spent flag. - isSpent := entry.IsSpent() - if isSpent != test.spent { - t.Fatalf("unexpected spent flag -- got %v, want %v", isSpent, test.spent) + t.Fatalf("%q: unexpected coinbase flag -- got %v, want %v", test.name, + isCoinBase, test.coinbase) } // Validate the expiry flag. hasExpiry := entry.HasExpiry() if hasExpiry != test.expiry { - t.Fatalf("unexpected expiry flag -- got %v, want %v", hasExpiry, - test.expiry) + t.Fatalf("%q: unexpected expiry flag -- got %v, want %v", test.name, + hasExpiry, test.expiry) + } + + // Validate the type of the transaction that the output is contained in. + gotTxType := entry.TransactionType() + if gotTxType != test.txType { + t.Fatalf("%q: unexpected transaction type -- got %v, want %v", test.name, + gotTxType, test.txType) } // Validate the height of the block containing the output. gotBlockHeight := entry.BlockHeight() if gotBlockHeight != int64(test.blockHeight) { - t.Fatalf("unexpected block height -- got %v, want %v", gotBlockHeight, - int64(test.blockHeight)) + t.Fatalf("%q: unexpected block height -- got %v, want %v", test.name, + gotBlockHeight, int64(test.blockHeight)) } // Validate the index of the transaction that the output is contained in. gotBlockIndex := entry.BlockIndex() if gotBlockIndex != test.blockIndex { - t.Fatalf("unexpected block index -- got %v, want %v", gotBlockIndex, - test.blockIndex) - } - - // Validate the type of the transaction that the output is contained in. - gotTxType := entry.TransactionType() - if gotTxType != test.txType { - t.Fatalf("unexpected transaction type -- got %v, want %v", gotTxType, - test.txType) + t.Fatalf("%q: unexpected block index -- got %v, want %v", test.name, + gotBlockIndex, test.blockIndex) } // Validate the amount of the output. gotAmount := entry.Amount() if gotAmount != test.amount { - t.Fatalf("unexpected amount -- got %v, want %v", gotAmount, test.amount) + t.Fatalf("%q: unexpected amount -- got %v, want %v", test.name, gotAmount, + test.amount) } // Validate the script of the output. gotScript := entry.PkScript() if !bytes.Equal(gotScript, test.pkScript) { - t.Fatalf("unexpected script -- got %v, want %v", gotScript, test.pkScript) + t.Fatalf("%q: unexpected script -- got %v, want %v", test.name, gotScript, + test.pkScript) } // Validate the script version of the output. gotScriptVersion := entry.ScriptVersion() if gotScriptVersion != test.scriptVersion { - t.Fatalf("unexpected script version -- got %v, want %v", gotScriptVersion, - test.scriptVersion) + t.Fatalf("%q: unexpected script version -- got %v, want %v", test.name, + gotScriptVersion, test.scriptVersion) } // Spend the entry. Validate that it is marked as spent and modified. entry.Spend() if !entry.IsSpent() { - t.Fatal("expected entry to be spent") + t.Fatalf("%q: expected entry to be spent", test.name) } if !entry.isModified() { - t.Fatal("expected entry to be modified") + t.Fatalf("%q: expected entry to be modified", test.name) } // Validate that if spend is called again the entry is still marked as spent // and modified. entry.Spend() if !entry.IsSpent() { - t.Fatal("expected entry to still be marked as spent") + t.Fatalf("%q: expected entry to still be marked as spent", test.name) } if !entry.isModified() { - t.Fatal("expected entry to still be marked as modified") + t.Fatalf("%q: expected entry to still be marked as modified", test.name) } // Validate the ticket minimal outputs. ticketMinOutsResult := entry.TicketMinimalOutputs() if !reflect.DeepEqual(ticketMinOutsResult, test.deserializedTicketMinOuts) { - t.Fatalf("unexpected ticket min outs -- got %v, want %v", + t.Fatalf("%q: unexpected ticket min outs -- got %v, want %v", test.name, ticketMinOutsResult, test.deserializedTicketMinOuts) } // Clone the entry and validate that all values are deep equal. clonedEntry := entry.Clone() if !reflect.DeepEqual(clonedEntry, entry) { - t.Fatalf("expected entry to be equal to cloned entry -- got %v, want %v", - clonedEntry, entry) + t.Fatalf("%q: expected entry to be equal to cloned entry -- got %v, "+ + "want %v", test.name, clonedEntry, entry) } // Validate that clone returns nil when called on a nil entry. var nilEntry *UtxoEntry if nilEntry.Clone() != nil { - t.Fatal("expected nil when calling clone on a nil entry") + t.Fatalf("%q: expected nil when calling clone on a nil entry", test.name) } } } diff --git a/blockchain/utxoviewpoint.go b/blockchain/utxoviewpoint.go index caa767c053..4c7c4783b9 100644 --- a/blockchain/utxoviewpoint.go +++ b/blockchain/utxoviewpoint.go @@ -25,6 +25,7 @@ import ( // The unspent outputs are needed by other transactions for things such as // script validation and double spend prevention. type UtxoViewpoint struct { + cache *UtxoCache entries map[wire.OutPoint]*UtxoEntry bestHash chainhash.Hash } @@ -73,12 +74,27 @@ func (view *UtxoViewpoint) addTxOut(outpoint wire.OutPoint, txOut *wire.TxOut, } entry.amount = txOut.Value - entry.pkScript = txOut.PkScript entry.blockHeight = uint32(blockHeight) entry.blockIndex = blockIndex entry.scriptVersion = txOut.Version entry.packedFlags = packedFlags entry.ticketMinOuts = ticketMinOuts + + // The referenced transaction output should always be marked as unspent and + // modified when being added to the view. + entry.state &^= utxoStateSpent + entry.state |= utxoStateModified + + // Deep copy the script. This is required since the tx out script is a + // subslice of the overall contiguous buffer that the msg tx houses for all + // scripts within the tx. It is deep copied here since this entry may be + // added to the utxo cache, and we don't want the utxo cache holding the entry + // to prevent all of the other tx scripts from getting garbage collected. + scriptLen := len(txOut.PkScript) + if scriptLen != 0 { + entry.pkScript = make([]byte, scriptLen) + copy(entry.pkScript, txOut.PkScript) + } } // AddTxOut adds the specified output of the passed transaction to the view if @@ -96,15 +112,13 @@ func (view *UtxoViewpoint) AddTxOut(tx *dcrutil.Tx, txOutIdx uint32, // Set encoded flags for the transaction. isCoinBase := standalone.IsCoinBaseTx(msgTx, isTreasuryEnabled) - const spent = false hasExpiry := msgTx.Expiry != wire.NoExpiryValue - const modified = true txType := stake.DetermineTxType(msgTx, isTreasuryEnabled) tree := wire.TxTreeRegular if txType != stake.TxTypeRegular { tree = wire.TxTreeStake } - flags := encodeUtxoFlags(isCoinBase, spent, modified, hasExpiry, txType) + flags := encodeUtxoFlags(isCoinBase, hasExpiry, txType) // Update existing entries. All fields are updated because it's possible // (although extremely unlikely) that the existing entry is being replaced by @@ -131,15 +145,13 @@ func (view *UtxoViewpoint) AddTxOuts(tx *dcrutil.Tx, blockHeight int64, blockInd // Set encoded flags for the transaction. isCoinBase := standalone.IsCoinBaseTx(msgTx, isTreasuryEnabled) - const spent = false hasExpiry := msgTx.Expiry != wire.NoExpiryValue - const modified = true txType := stake.DetermineTxType(msgTx, isTreasuryEnabled) tree := wire.TxTreeRegular if txType != stake.TxTypeRegular { tree = wire.TxTreeStake } - flags := encodeUtxoFlags(isCoinBase, spent, modified, hasExpiry, txType) + flags := encodeUtxoFlags(isCoinBase, hasExpiry, txType) // Loop through all of the transaction outputs and add those which are not // provably unspendable. @@ -325,16 +337,25 @@ func (view *UtxoViewpoint) disconnectTransactions(block *dcrutil.Block, stxos [] outpoint.Index = uint32(txOutIdx) entry := view.entries[outpoint] if entry == nil { - const spent = false - const modified = true entry = &UtxoEntry{ amount: txOut.Value, - pkScript: txOut.PkScript, blockHeight: uint32(block.Height()), blockIndex: uint32(txIdx), scriptVersion: txOut.Version, - packedFlags: encodeUtxoFlags(isCoinBase, spent, modified, hasExpiry, - txType), + state: utxoStateModified, + packedFlags: encodeUtxoFlags(isCoinBase, hasExpiry, txType), + } + + // Deep copy the script. This is required since the tx out script is a + // subslice of the overall contiguous buffer that the msg tx houses for + // all scripts within the tx. It is deep copied here since this entry + // may be added to the utxo cache, and we don't want the utxo cache + // holding the entry to prevent all of the other tx scripts from getting + // garbage collected. + scriptLen := len(txOut.PkScript) + if scriptLen != 0 { + entry.pkScript = make([]byte, scriptLen) + copy(entry.pkScript, txOut.PkScript) } if isTicketSubmissionOutput(txType, uint32(txOutIdx)) { @@ -374,8 +395,6 @@ func (view *UtxoViewpoint) disconnectTransactions(block *dcrutil.Block, stxos [] txIn := msgTx.TxIn[txInIdx] entry := view.entries[txIn.PreviousOutPoint] if entry == nil { - const spent = false - const modified = true entry = &UtxoEntry{ amount: txIn.ValueIn, pkScript: stxo.pkScript, @@ -383,8 +402,9 @@ func (view *UtxoViewpoint) disconnectTransactions(block *dcrutil.Block, stxos [] blockHeight: stxo.blockHeight, blockIndex: stxo.blockIndex, scriptVersion: stxo.scriptVersion, - packedFlags: encodeUtxoFlags(stxo.IsCoinBase(), spent, modified, - stxo.HasExpiry(), stxo.TransactionType()), + state: utxoStateModified, + packedFlags: encodeUtxoFlags(stxo.IsCoinBase(), stxo.HasExpiry(), + stxo.TransactionType()), } view.entries[txIn.PreviousOutPoint] = entry @@ -392,8 +412,8 @@ func (view *UtxoViewpoint) disconnectTransactions(block *dcrutil.Block, stxos [] // Mark the existing referenced transaction output as unspent and // modified. - entry.packedFlags &^= utxoFlagSpent - entry.packedFlags |= utxoFlagModified + entry.state &^= utxoStateSpent + entry.state |= utxoStateModified } } @@ -443,7 +463,7 @@ func (view *UtxoViewpoint) disconnectDisapprovedBlock(db database.DB, block *dcr // Load all of the utxos referenced by the inputs for all transactions in // the block that don't already exist in the utxo view from the database. - err = view.fetchRegularInputUtxos(db, block, isTreasuryEnabled) + err = view.fetchRegularInputUtxos(block, isTreasuryEnabled) if err != nil { return err } @@ -486,7 +506,7 @@ func (view *UtxoViewpoint) connectBlock(db database.DB, block, parent *dcrutil.B // Load all of the utxos referenced by the inputs for all transactions in // the block that don't already exist in the utxo view from the database. - err := view.fetchInputUtxos(db, block, isTreasuryEnabled) + err := view.fetchInputUtxos(block, isTreasuryEnabled) if err != nil { return err } @@ -534,7 +554,7 @@ func (view *UtxoViewpoint) connectBlock(db database.DB, block, parent *dcrutil.B // Note that, unlike block connection, the spent transaction output (stxo) // information is required and failure to provide it will result in an assertion // panic. -func (view *UtxoViewpoint) disconnectBlock(db database.DB, block, parent *dcrutil.Block, stxos []spentTxOut, isTreasuryEnabled bool) error { +func (view *UtxoViewpoint) disconnectBlock(block, parent *dcrutil.Block, stxos []spentTxOut, isTreasuryEnabled bool) error { // Sanity check the correct number of stxos are provided. if len(stxos) != countSpentOutputs(block, isTreasuryEnabled) { panicf("provided %v stxos for block %v (height %v) which spends %v "+ @@ -544,7 +564,7 @@ func (view *UtxoViewpoint) disconnectBlock(db database.DB, block, parent *dcruti // Load all of the utxos referenced by the inputs for all transactions in // the block don't already exist in the utxo view from the database. - err := view.fetchInputUtxos(db, block, isTreasuryEnabled) + err := view.fetchInputUtxos(block, isTreasuryEnabled) if err != nil { return err } @@ -567,7 +587,7 @@ func (view *UtxoViewpoint) disconnectBlock(db database.DB, block, parent *dcruti // Load all of the utxos referenced by the inputs for all transactions // in the regular tree of the parent block that don't already exist in // the utxo view from the database. - err := view.fetchRegularInputUtxos(db, parent, isTreasuryEnabled) + err := view.fetchRegularInputUtxos(parent, isTreasuryEnabled) if err != nil { return err } @@ -592,19 +612,6 @@ func (view *UtxoViewpoint) Entries() map[wire.OutPoint]*UtxoEntry { return view.entries } -// commit prunes all entries marked modified that are now spent and marks all -// entries as unmodified. -func (view *UtxoViewpoint) commit() { - for outpoint, entry := range view.entries { - if entry == nil || (entry.isModified() && entry.IsSpent()) { - delete(view.entries, outpoint) - continue - } - - entry.packedFlags &^= utxoFlagModified - } -} - // viewFilteredSet represents a set of utxos to fetch from the database that are // not already in a view. type viewFilteredSet map[wire.OutPoint]struct{} @@ -624,31 +631,13 @@ func (set viewFilteredSet) add(view *UtxoViewpoint, outpoint *wire.OutPoint) { // Upon completion of this function, the view will contain an entry for each // requested outpoint. Spent outputs, or those which otherwise don't exist, // will result in a nil entry in the view. -func (view *UtxoViewpoint) fetchUtxosMain(db database.DB, filteredSet viewFilteredSet) error { +func (view *UtxoViewpoint) fetchUtxosMain(filteredSet viewFilteredSet) error { // Nothing to do if there are no requested outputs. if len(filteredSet) == 0 { return nil } - // Load the requested set of unspent transaction outputs from the point - // of view of the end of the main chain. - // - // NOTE: Missing entries are not considered an error here and instead - // will result in nil entries in the view. This is intentionally done - // so other code can use the presence of an entry in the view as a way - // to unnecessarily avoid attempting to reload it from the database. - return db.View(func(dbTx database.Tx) error { - for outpoint := range filteredSet { - entry, err := dbFetchUtxoEntry(dbTx, outpoint) - if err != nil { - return err - } - - view.entries[outpoint] = entry - } - - return nil - }) + return view.cache.FetchEntries(filteredSet, view) } // addRegularInputUtxos adds any outputs of transactions in the regular tree of @@ -706,12 +695,12 @@ func (view *UtxoViewpoint) addRegularInputUtxos(block *dcrutil.Block, isTreasury // the view from the database as needed. In particular, referenced entries that // are earlier in the block are added to the view and entries that are already // in the view are not modified. -func (view *UtxoViewpoint) fetchRegularInputUtxos(db database.DB, block *dcrutil.Block, isTreasuryEnabled bool) error { +func (view *UtxoViewpoint) fetchRegularInputUtxos(block *dcrutil.Block, isTreasuryEnabled bool) error { // Add any outputs of transactions in the regular tree of the block that are // referenced by inputs of transactions that are located later in the tree // and fetch any inputs that are not already in the view from the database. filteredSet := view.addRegularInputUtxos(block, isTreasuryEnabled) - return view.fetchUtxosMain(db, filteredSet) + return view.fetchUtxosMain(filteredSet) } // fetchInputUtxos loads the unspent transaction outputs for the inputs @@ -720,7 +709,7 @@ func (view *UtxoViewpoint) fetchRegularInputUtxos(db database.DB, block *dcrutil // regular tree, referenced entries that are earlier in the regular tree of the // block are added to the view. In all cases, entries that are already in the // view are not modified. -func (view *UtxoViewpoint) fetchInputUtxos(db database.DB, block *dcrutil.Block, isTreasuryEnabled bool) error { +func (view *UtxoViewpoint) fetchInputUtxos(block *dcrutil.Block, isTreasuryEnabled bool) error { // Add any outputs of transactions in the regular tree of the block that are // referenced by inputs of transactions that are located later in the tree // and, while doing so, determine which inputs are not already in the view @@ -749,12 +738,13 @@ func (view *UtxoViewpoint) fetchInputUtxos(db database.DB, block *dcrutil.Block, } // Request the input utxos from the database. - return view.fetchUtxosMain(db, filteredSet) + return view.fetchUtxosMain(filteredSet) } // clone returns a deep copy of the view. func (view *UtxoViewpoint) clone() *UtxoViewpoint { clonedView := &UtxoViewpoint{ + cache: view.cache, entries: make(map[wire.OutPoint]*UtxoEntry), bestHash: view.bestHash, } @@ -767,8 +757,9 @@ func (view *UtxoViewpoint) clone() *UtxoViewpoint { } // NewUtxoViewpoint returns a new empty unspent transaction output view. -func NewUtxoViewpoint() *UtxoViewpoint { +func NewUtxoViewpoint(cache *UtxoCache) *UtxoViewpoint { return &UtxoViewpoint{ + cache: cache, entries: make(map[wire.OutPoint]*UtxoEntry), } } @@ -790,7 +781,7 @@ func (b *BlockChain) FetchUtxoView(tx *dcrutil.Tx, includeRegularTxns bool) (*Ut // because the code below requires the parent block and the genesis // block doesn't have one. tip := b.bestChain.Tip() - view := NewUtxoViewpoint() + view := NewUtxoViewpoint(b.utxoCache) view.SetBestHash(&tip.hash) if tip.height == 0 { return view, nil @@ -860,57 +851,6 @@ func (b *BlockChain) FetchUtxoView(tx *dcrutil.Tx, includeRegularTxns bool) (*Ut } } - err = view.fetchUtxosMain(b.db, filteredSet) + err = view.fetchUtxosMain(filteredSet) return view, err } - -// FetchUtxoEntry loads and returns the requested unspent transaction output -// from the point of view of the the main chain tip. -// -// NOTE: Requesting an output for which there is no data will NOT return an -// error. Instead both the entry and the error will be nil. This is done to -// allow pruning of spent transaction outputs. In practice this means the -// caller must check if the returned entry is nil before invoking methods on it. -// -// This function is safe for concurrent access however the returned entry (if -// any) is NOT. -func (b *BlockChain) FetchUtxoEntry(outpoint wire.OutPoint) (*UtxoEntry, error) { - b.chainLock.RLock() - defer b.chainLock.RUnlock() - - var entry *UtxoEntry - err := b.db.View(func(dbTx database.Tx) error { - var err error - entry, err = dbFetchUtxoEntry(dbTx, outpoint) - return err - }) - if err != nil { - return nil, err - } - - return entry, nil -} - -// UtxoStats represents unspent output statistics on the current utxo set. -type UtxoStats struct { - Utxos int64 - Transactions int64 - Size int64 - Total int64 - SerializedHash chainhash.Hash -} - -// FetchUtxoStats returns statistics on the current utxo set. -func (b *BlockChain) FetchUtxoStats() (*UtxoStats, error) { - var stats *UtxoStats - err := b.db.View(func(dbTx database.Tx) error { - var err error - stats, err = dbFetchUtxoStats(dbTx) - return err - }) - if err != nil { - return nil, err - } - - return stats, nil -} diff --git a/blockchain/validate.go b/blockchain/validate.go index ee41de7015..2f152518db 100644 --- a/blockchain/validate.go +++ b/blockchain/validate.go @@ -1854,7 +1854,7 @@ func (b *BlockChain) checkDupTxs(txSet []*dcrutil.Tx, view *UtxoViewpoint, tree filteredSet.add(view, &outpoint) } } - err := view.fetchUtxosMain(b.db, filteredSet) + err := view.fetchUtxosMain(filteredSet) if err != nil { return err } @@ -3557,7 +3557,7 @@ func (b *BlockChain) checkConnectBlock(node *blockNode, block, parent *dcrutil.B // // These utxo entries are needed for verification of things such as // transaction inputs, counting pay-to-script-hashes, and scripts. - err = view.fetchInputUtxos(b.db, block, isTreasuryEnabled) + err = view.fetchInputUtxos(block, isTreasuryEnabled) if err != nil { return err } @@ -3791,7 +3791,7 @@ func (b *BlockChain) CheckConnectBlockTemplate(block *dcrutil.Block) error { return ruleError(ErrMissingParent, err.Error()) } - view := NewUtxoViewpoint() + view := NewUtxoViewpoint(b.utxoCache) view.SetBestHash(&tip.hash) return b.checkConnectBlock(newNode, block, parent, view, nil, nil) @@ -3801,7 +3801,7 @@ func (b *BlockChain) CheckConnectBlockTemplate(block *dcrutil.Block) error { // current tip due to the previous checks, so undo the transactions and // spend information for the tip block to reach the point of view of the // block template. - view := NewUtxoViewpoint() + view := NewUtxoViewpoint(b.utxoCache) view.SetBestHash(&tip.hash) tipBlock, err := b.fetchMainChainBlockByNode(tip) if err != nil { @@ -3829,7 +3829,7 @@ func (b *BlockChain) CheckConnectBlockTemplate(block *dcrutil.Block) error { // Update the view to unspend all of the spent txos and remove the utxos // created by the tip block. Also, if the block votes against its parent, // reconnect all of the regular transactions. - err = view.disconnectBlock(b.db, tipBlock, parent, stxos, isTreasuryEnabled) + err = view.disconnectBlock(tipBlock, parent, stxos, isTreasuryEnabled) if err != nil { return err } diff --git a/blockchain/validate_test.go b/blockchain/validate_test.go index 6154b25209..e008542948 100644 --- a/blockchain/validate_test.go +++ b/blockchain/validate_test.go @@ -1,5 +1,5 @@ // Copyright (c) 2013-2016 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -290,6 +290,10 @@ func TestCheckBlockHeaderContext(t *testing.T) { DB: db, ChainParams: params, TimeSource: NewMedianTime(), + UtxoCache: NewUtxoCache(&UtxoCacheConfig{ + DB: db, + MaxSize: 100 * 1024 * 1024, // 100 MiB + }), }) if err != nil { t.Fatalf("Failed to create chain instance: %v\n", err) diff --git a/config.go b/config.go index cc026d88b7..477684c326 100644 --- a/config.go +++ b/config.go @@ -1,5 +1,5 @@ // Copyright (c) 2013-2016 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -38,13 +38,16 @@ import ( const ( // Defaults for general application behavior options. - defaultConfigFilename = "dcrd.conf" - defaultDataDirname = "data" - defaultLogDirname = "logs" - defaultLogFilename = "dcrd.log" - defaultDbType = "ffldb" - defaultLogLevel = "info" - defaultSigCacheMaxSize = 100000 + defaultConfigFilename = "dcrd.conf" + defaultDataDirname = "data" + defaultLogDirname = "logs" + defaultLogFilename = "dcrd.log" + defaultDbType = "ffldb" + defaultLogLevel = "info" + defaultSigCacheMaxSize = 100000 + defaultUtxoCacheMaxSize = 150 + minUtxoCacheMaxSize = 25 + maxUtxoCacheMaxSize = 32768 // 32 GiB // Defaults for RPC server options and policy. defaultTLSCurve = "P-256" @@ -118,21 +121,22 @@ func minUint32(a, b uint32) uint32 { // See loadConfig for details on the configuration load process. type config struct { // General application behavior. - ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` - HomeDir string `short:"A" long:"appdata" description:"Path to application home directory"` - ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` - DataDir string `short:"b" long:"datadir" description:"Directory to store data"` - LogDir string `long:"logdir" description:"Directory to log output"` - NoFileLogging bool `long:"nofilelogging" description:"Disable file logging"` - DbType string `long:"dbtype" description:"Database backend to use for the block chain"` - Profile string `long:"profile" description:"Enable HTTP profiling on given [addr:]port -- NOTE port must be between 1024 and 65536"` - CPUProfile string `long:"cpuprofile" description:"Write CPU profile to the specified file"` - MemProfile string `long:"memprofile" description:"Write mem profile to the specified file"` - TestNet bool `long:"testnet" description:"Use the test network"` - SimNet bool `long:"simnet" description:"Use the simulation test network"` - RegNet bool `long:"regnet" description:"Use the regression test network"` - DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify =,=,... to set the log level for individual subsystems -- Use show to list available subsystems"` - SigCacheMaxSize uint `long:"sigcachemaxsize" description:"The maximum number of entries in the signature verification cache"` + ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` + HomeDir string `short:"A" long:"appdata" description:"Path to application home directory"` + ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` + DataDir string `short:"b" long:"datadir" description:"Directory to store data"` + LogDir string `long:"logdir" description:"Directory to log output"` + NoFileLogging bool `long:"nofilelogging" description:"Disable file logging"` + DbType string `long:"dbtype" description:"Database backend to use for the block chain"` + Profile string `long:"profile" description:"Enable HTTP profiling on given [addr:]port -- NOTE port must be between 1024 and 65536"` + CPUProfile string `long:"cpuprofile" description:"Write CPU profile to the specified file"` + MemProfile string `long:"memprofile" description:"Write mem profile to the specified file"` + TestNet bool `long:"testnet" description:"Use the test network"` + SimNet bool `long:"simnet" description:"Use the simulation test network"` + RegNet bool `long:"regnet" description:"Use the regression test network"` + DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify =,=,... to set the log level for individual subsystems -- Use show to list available subsystems"` + SigCacheMaxSize uint `long:"sigcachemaxsize" description:"The maximum number of entries in the signature verification cache"` + UtxoCacheMaxSize uint `long:"utxocachemaxsize" description:"The maximum size in MiB of the utxo cache"` // RPC server options and policy. DisableRPC bool `long:"norpc" description:"Disable built-in RPC server -- NOTE: The RPC server is disabled by default if no rpcuser/rpcpass or rpclimituser/rpclimitpass is specified"` @@ -560,13 +564,14 @@ func loadConfig(appName string) (*config, []string, error) { // Default config. cfg := config{ // General application behavior. - HomeDir: defaultHomeDir, - ConfigFile: defaultConfigFile, - DataDir: defaultDataDir, - LogDir: defaultLogDir, - DbType: defaultDbType, - DebugLevel: defaultLogLevel, - SigCacheMaxSize: defaultSigCacheMaxSize, + HomeDir: defaultHomeDir, + ConfigFile: defaultConfigFile, + DataDir: defaultDataDir, + LogDir: defaultLogDir, + DbType: defaultDbType, + DebugLevel: defaultLogLevel, + SigCacheMaxSize: defaultSigCacheMaxSize, + UtxoCacheMaxSize: defaultUtxoCacheMaxSize, // RPC server options and policy. RPCCert: defaultRPCCertFile, @@ -846,6 +851,13 @@ func loadConfig(appName string) (*config, []string, error) { return nil, nil, err } + // Enforce the minimum and maximum utxo cache max size. + if cfg.UtxoCacheMaxSize < minUtxoCacheMaxSize { + cfg.UtxoCacheMaxSize = minUtxoCacheMaxSize + } else if cfg.UtxoCacheMaxSize > maxUtxoCacheMaxSize { + cfg.UtxoCacheMaxSize = maxUtxoCacheMaxSize + } + // Validate format of profile, can be an address:port, or just a port. if cfg.Profile != "" { // if profile is just a number, then add a default host of "127.0.0.1" such that Profile is a valid tcp address diff --git a/doc.go b/doc.go index 9642ab1c93..60ab5178eb 100644 --- a/doc.go +++ b/doc.go @@ -1,5 +1,5 @@ // Copyright (c) 2013-2016 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -45,6 +45,8 @@ Application Options: Use show to list available subsystems (info) --sigcachemaxsize= The maximum number of entries in the signature verification cache (default: 100000) + --utxocachemaxsize= The maximum size in MiB of the utxo cache + (default: 150, minimum: 25, maximum: 32768) --norpc Disable built-in RPC server -- NOTE: The RPC server is disabled by default if no rpcuser/rpcpass or rpclimituser/rpclimitpass is diff --git a/internal/mempool/mempool_test.go b/internal/mempool/mempool_test.go index aed020e85d..124a298ee9 100644 --- a/internal/mempool/mempool_test.go +++ b/internal/mempool/mempool_test.go @@ -87,7 +87,7 @@ func (s *fakeChain) FetchUtxoView(tx *dcrutil.Tx, treeValid bool) (*blockchain.U // Add entries for the outputs of the tx to the new view. msgTx := tx.MsgTx() - viewpoint := blockchain.NewUtxoViewpoint() + viewpoint := blockchain.NewUtxoViewpoint(nil) outpoint := wire.OutPoint{Hash: *tx.Hash(), Tree: tx.Tree()} for txOutIdx := range msgTx.TxOut { outpoint.Index = uint32(txOutIdx) @@ -754,7 +754,7 @@ func newPoolHarness(chainParams *chaincfg.Params) (*poolHarness, []spendableOutp // Create a new fake chain and harness bound to it. subsidyCache := standalone.NewSubsidyCache(chainParams) chain := &fakeChain{ - utxos: blockchain.NewUtxoViewpoint(), + utxos: blockchain.NewUtxoViewpoint(nil), utxoTimes: make(map[wire.OutPoint]int64), blocks: make(map[chainhash.Hash]*dcrutil.Block), scriptFlags: BaseStandardVerifyFlags, diff --git a/internal/mining/mining_harness_test.go b/internal/mining/mining_harness_test.go index f95cc1f90a..c92697cad8 100644 --- a/internal/mining/mining_harness_test.go +++ b/internal/mining/mining_harness_test.go @@ -144,7 +144,7 @@ func (c *fakeChain) FetchUtxoView(tx *dcrutil.Tx, treeValid bool) (*blockchain.U // Add entries for the outputs of the tx to the new view. msgTx := tx.MsgTx() - viewpoint := blockchain.NewUtxoViewpoint() + viewpoint := blockchain.NewUtxoViewpoint(nil) prevOut := wire.OutPoint{Hash: *tx.Hash(), Tree: tx.Tree()} for txOutIdx := range msgTx.TxOut { prevOut.Index = uint32(txOutIdx) @@ -195,7 +195,7 @@ func (c *fakeChain) MaxTreasuryExpenditure(preTVIBlock *chainhash.Hash) (int64, // NewUtxoViewpoint returns a new empty unspent transaction output view. func (c *fakeChain) NewUtxoViewpoint() *blockchain.UtxoViewpoint { - return blockchain.NewUtxoViewpoint() + return blockchain.NewUtxoViewpoint(nil) } // TipGeneration returns a mocked entire generation of blocks stemming from the @@ -1296,8 +1296,8 @@ func newMiningHarness(chainParams *chaincfg.Params) (*miningHarness, []spendable blocks: make(map[chainhash.Hash]*dcrutil.Block), isHeaderCommitmentsAgendaActive: true, isTreasuryAgendaActive: true, - parentUtxos: blockchain.NewUtxoViewpoint(), - utxos: blockchain.NewUtxoViewpoint(), + parentUtxos: blockchain.NewUtxoViewpoint(nil), + utxos: blockchain.NewUtxoViewpoint(nil), } // Set the proof of work limit and next required difficulty very high by diff --git a/internal/rpcserver/interface.go b/internal/rpcserver/interface.go index 16d8e5ecc2..5b8cc0800d 100644 --- a/internal/rpcserver/interface.go +++ b/internal/rpcserver/interface.go @@ -281,6 +281,11 @@ type Chain interface { FetchUtxoEntry(outpoint wire.OutPoint) (UtxoEntry, error) // FetchUtxoStats returns statistics on the current utxo set. + // + // NOTE: During initial block download the utxo stats will lag behind the best + // block that is currently synced since the utxo cache is only flushed to the + // database periodically. After initial block download the utxo stats will + // always be in sync with the best block. FetchUtxoStats() (*blockchain.UtxoStats, error) // GetStakeVersions returns a cooked array of StakeVersions. We do this in diff --git a/sampleconfig/sampleconfig.go b/sampleconfig/sampleconfig.go index 78e1bcb323..951135528b 100644 --- a/sampleconfig/sampleconfig.go +++ b/sampleconfig/sampleconfig.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -292,6 +292,12 @@ const fileContents = `[Application Options] ; Limit the signature cache to a max of 50000 entries. ; sigcachemaxsize=50000 +; ------------------------------------------------------------------------------ +; Unspent Transaction Output (UTXO) Cache +; ------------------------------------------------------------------------------ + +; Limit the utxo cache to a max of 100 MiB. +; utxocachemaxsize=150 ; ------------------------------------------------------------------------------ ; Coin Generation (Mining) Settings - The following options control the diff --git a/server.go b/server.go index 14df8de989..bc6e60c4b3 100644 --- a/server.go +++ b/server.go @@ -3017,6 +3017,8 @@ func (s *server) Run(ctx context.Context) { s.feeEstimator.Close() + s.chain.ShutdownUtxoCache() + s.wg.Wait() srvrLog.Trace("Server stopped") } @@ -3367,6 +3369,10 @@ func newServer(ctx context.Context, listenAddrs []string, db database.DB, chainP } // Create a new block chain instance with the appropriate configuration. + utxoCache := blockchain.NewUtxoCache(&blockchain.UtxoCacheConfig{ + DB: s.db, + MaxSize: uint64(cfg.UtxoCacheMaxSize) * 1024 * 1024, + }) s.chain, err = blockchain.New(ctx, &blockchain.Config{ DB: s.db, @@ -3377,6 +3383,7 @@ func newServer(ctx context.Context, listenAddrs []string, db database.DB, chainP SigCache: s.sigCache, SubsidyCache: s.subsidyCache, IndexManager: indexManager, + UtxoCache: utxoCache, }) if err != nil { return nil, err @@ -3526,7 +3533,7 @@ func newServer(ctx context.Context, listenAddrs []string, db database.DB, chainP IsTreasuryAgendaActive: s.chain.IsTreasuryAgendaActive, MaxTreasuryExpenditure: s.chain.MaxTreasuryExpenditure, NewUtxoViewpoint: func() *blockchain.UtxoViewpoint { - return blockchain.NewUtxoViewpoint() + return blockchain.NewUtxoViewpoint(utxoCache) }, TipGeneration: s.chain.TipGeneration, ValidateTransactionScripts: func(tx *dcrutil.Tx,