From 896c1591ba63496924597ec1408c350d044ca13c Mon Sep 17 00:00:00 2001 From: Ryan Staudt Date: Fri, 22 Jan 2021 09:02:31 -0600 Subject: [PATCH 01/10] blockchain: Add test name to TestUtxoEntry errors. --- blockchain/utxoentry_test.go | 53 +++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/blockchain/utxoentry_test.go b/blockchain/utxoentry_test.go index 2184d9d8d1..77ff357057 100644 --- a/blockchain/utxoentry_test.go +++ b/blockchain/utxoentry_test.go @@ -181,107 +181,110 @@ func TestUtxoEntry(t *testing.T) { // Validate the modified flag. isModified := entry.isModified() if isModified != test.modified { - t.Fatalf("unexpected modified flag -- got %v, want %v", isModified, - test.modified) + t.Fatalf("%q: unexpected modified flag -- got %v, want %v", test.name, + isModified, test.modified) } // Validate the coinbase flag. isCoinBase := entry.IsCoinBase() if isCoinBase != test.coinbase { - t.Fatalf("unexpected coinbase flag -- got %v, want %v", isCoinBase, - test.coinbase) + t.Fatalf("%q: unexpected coinbase flag -- got %v, want %v", test.name, + isCoinBase, test.coinbase) } // Validate the spent flag. isSpent := entry.IsSpent() if isSpent != test.spent { - t.Fatalf("unexpected spent flag -- got %v, want %v", isSpent, test.spent) + t.Fatalf("%q: unexpected spent flag -- got %v, want %v", test.name, + isSpent, test.spent) } // Validate the expiry flag. hasExpiry := entry.HasExpiry() if hasExpiry != test.expiry { - t.Fatalf("unexpected expiry flag -- got %v, want %v", hasExpiry, - test.expiry) + t.Fatalf("%q: unexpected expiry flag -- got %v, want %v", test.name, + hasExpiry, test.expiry) } // Validate the height of the block containing the output. gotBlockHeight := entry.BlockHeight() if gotBlockHeight != int64(test.blockHeight) { - t.Fatalf("unexpected block height -- got %v, want %v", gotBlockHeight, - int64(test.blockHeight)) + t.Fatalf("%q: unexpected block height -- got %v, want %v", test.name, + gotBlockHeight, int64(test.blockHeight)) } // Validate the index of the transaction that the output is contained in. gotBlockIndex := entry.BlockIndex() if gotBlockIndex != test.blockIndex { - t.Fatalf("unexpected block index -- got %v, want %v", gotBlockIndex, - test.blockIndex) + t.Fatalf("%q: unexpected block index -- got %v, want %v", test.name, + gotBlockIndex, test.blockIndex) } // Validate the type of the transaction that the output is contained in. gotTxType := entry.TransactionType() if gotTxType != test.txType { - t.Fatalf("unexpected transaction type -- got %v, want %v", gotTxType, - test.txType) + t.Fatalf("%q: unexpected transaction type -- got %v, want %v", test.name, + gotTxType, test.txType) } // Validate the amount of the output. gotAmount := entry.Amount() if gotAmount != test.amount { - t.Fatalf("unexpected amount -- got %v, want %v", gotAmount, test.amount) + t.Fatalf("%q: unexpected amount -- got %v, want %v", test.name, gotAmount, + test.amount) } // Validate the script of the output. gotScript := entry.PkScript() if !bytes.Equal(gotScript, test.pkScript) { - t.Fatalf("unexpected script -- got %v, want %v", gotScript, test.pkScript) + t.Fatalf("%q: unexpected script -- got %v, want %v", test.name, gotScript, + test.pkScript) } // Validate the script version of the output. gotScriptVersion := entry.ScriptVersion() if gotScriptVersion != test.scriptVersion { - t.Fatalf("unexpected script version -- got %v, want %v", gotScriptVersion, - test.scriptVersion) + t.Fatalf("%q: unexpected script version -- got %v, want %v", test.name, + gotScriptVersion, test.scriptVersion) } // Spend the entry. Validate that it is marked as spent and modified. entry.Spend() if !entry.IsSpent() { - t.Fatal("expected entry to be spent") + t.Fatalf("%q: expected entry to be spent", test.name) } if !entry.isModified() { - t.Fatal("expected entry to be modified") + t.Fatalf("%q: expected entry to be modified", test.name) } // Validate that if spend is called again the entry is still marked as spent // and modified. entry.Spend() if !entry.IsSpent() { - t.Fatal("expected entry to still be marked as spent") + t.Fatalf("%q: expected entry to still be marked as spent", test.name) } if !entry.isModified() { - t.Fatal("expected entry to still be marked as modified") + t.Fatalf("%q: expected entry to still be marked as modified", test.name) } // Validate the ticket minimal outputs. ticketMinOutsResult := entry.TicketMinimalOutputs() if !reflect.DeepEqual(ticketMinOutsResult, test.deserializedTicketMinOuts) { - t.Fatalf("unexpected ticket min outs -- got %v, want %v", + t.Fatalf("%q: unexpected ticket min outs -- got %v, want %v", test.name, ticketMinOutsResult, test.deserializedTicketMinOuts) } // Clone the entry and validate that all values are deep equal. clonedEntry := entry.Clone() if !reflect.DeepEqual(clonedEntry, entry) { - t.Fatalf("expected entry to be equal to cloned entry -- got %v, want %v", - clonedEntry, entry) + t.Fatalf("%q: expected entry to be equal to cloned entry -- got %v, "+ + "want %v", test.name, clonedEntry, entry) } // Validate that clone returns nil when called on a nil entry. var nilEntry *UtxoEntry if nilEntry.Clone() != nil { - t.Fatal("expected nil when calling clone on a nil entry") + t.Fatalf("%q: expected nil when calling clone on a nil entry", test.name) } } } From e55207b21d48329df72616285e043cfd5194e248 Mon Sep 17 00:00:00 2001 From: Ryan Staudt Date: Thu, 21 Jan 2021 06:35:59 -0600 Subject: [PATCH 02/10] blockchain: Separate utxo state from tx flags. This splits the utxo packed flags into two separate types, utxoState and utxoFlags. The reasoning is that: - This cleanly separates the purpose of the flags. utxoState defines the in-memory state of a utxo entry, whereas utxoFlags defines additional information for the containing transaction of a utxo entry. - This makes room for an additional state that is required for the utxo cache, namely whether or not a utxo entry is fresh (does not exist as an unspent transaction output in the database). --- blockchain/chainio.go | 7 ++-- blockchain/chainio_test.go | 20 ++--------- blockchain/utxoentry.go | 69 ++++++++++++++++++++---------------- blockchain/utxoentry_test.go | 57 ++++++++++++++--------------- blockchain/utxoviewpoint.go | 32 ++++++++--------- 5 files changed, 84 insertions(+), 101 deletions(-) diff --git a/blockchain/chainio.go b/blockchain/chainio.go index 70ab0367ad..3dbe9ddf70 100644 --- a/blockchain/chainio.go +++ b/blockchain/chainio.go @@ -1,5 +1,5 @@ // Copyright (c) 2015-2016 The btcsuite developers -// Copyright (c) 2016-2020 The Decred developers +// Copyright (c) 2016-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -993,16 +993,13 @@ func deserializeUtxoEntry(serialized []byte, txOutIndex uint32) (*UtxoEntry, err offset += bytesRead // Create a new utxo entry with the details deserialized above. - const spent = false - const modified = false entry := &UtxoEntry{ amount: amount, pkScript: script, blockHeight: uint32(blockHeight), blockIndex: uint32(blockIndex), scriptVersion: scriptVersion, - packedFlags: encodeUtxoFlags(isCoinBase, spent, modified, hasExpiry, - txType), + packedFlags: encodeUtxoFlags(isCoinBase, hasExpiry, txType), } // Copy the minimal outputs if this was a ticket submission output. diff --git a/blockchain/chainio_test.go b/blockchain/chainio_test.go index c83a66e611..8dffdde0df 100644 --- a/blockchain/chainio_test.go +++ b/blockchain/chainio_test.go @@ -1,5 +1,5 @@ // Copyright (c) 2015-2016 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -828,9 +828,6 @@ func TestUtxoSerialization(t *testing.T) { withCoinbase = true noExpiry = false withExpiry = true - unspent = false - spent = true - unmodified = false ) tests := []struct { @@ -851,8 +848,6 @@ func TestUtxoSerialization(t *testing.T) { scriptVersion: 0, packedFlags: encodeUtxoFlags( withCoinbase, - unspent, - unmodified, noExpiry, stake.TxTypeRegular, ), @@ -872,8 +867,6 @@ func TestUtxoSerialization(t *testing.T) { scriptVersion: 0, packedFlags: encodeUtxoFlags( withCoinbase, - unspent, - unmodified, noExpiry, stake.TxTypeRegular, ), @@ -892,8 +885,6 @@ func TestUtxoSerialization(t *testing.T) { scriptVersion: 0, packedFlags: encodeUtxoFlags( noCoinbase, - unspent, - unmodified, noExpiry, stake.TxTypeRegular, ), @@ -912,8 +903,6 @@ func TestUtxoSerialization(t *testing.T) { scriptVersion: 0, packedFlags: encodeUtxoFlags( noCoinbase, - unspent, - unmodified, withExpiry, stake.TxTypeSStx, ), @@ -940,8 +929,6 @@ func TestUtxoSerialization(t *testing.T) { scriptVersion: 0xffff, packedFlags: encodeUtxoFlags( withCoinbase, - unspent, - unmodified, noExpiry, stake.TxTypeRegular, ), @@ -960,8 +947,6 @@ func TestUtxoSerialization(t *testing.T) { scriptVersion: 0, packedFlags: encodeUtxoFlags( noCoinbase, - unspent, - unmodified, withExpiry, stake.TxTypeRegular, ), @@ -979,10 +964,9 @@ func TestUtxoSerialization(t *testing.T) { blockHeight: 33333, blockIndex: 3, scriptVersion: 0, + state: utxoStateModified | utxoStateSpent, packedFlags: encodeUtxoFlags( withCoinbase, - spent, - unmodified, withExpiry, stake.TxTypeRegular, ), diff --git a/blockchain/utxoentry.go b/blockchain/utxoentry.go index 98b1a6645f..43ecb394da 100644 --- a/blockchain/utxoentry.go +++ b/blockchain/utxoentry.go @@ -6,55 +6,57 @@ package blockchain import "github.com/decred/dcrd/blockchain/stake/v4" -// utxoFlags defines additional information and state for a transaction output -// in a utxo view. The bit representation is: +// utxoState defines the in-memory state of a utxo entry. +// +// The bit representation is: +// bit 0 - transaction output has been spent +// bit 1 - transaction output has been modified since it was loaded +// bits 2-7 - unused +type utxoState uint8 + +const ( + // utxoStateSpent indicates that a txout is spent. + utxoStateSpent utxoState = 1 << iota + + // utxoStateModified indicates that a txout has been modified since it was + // loaded. + utxoStateModified +) + +// utxoFlags defines additional information for the containing transaction of a +// utxo entry. +// +// The bit representation is: // bit 0 - containing transaction is a coinbase -// bit 1 - transaction output has been spent -// bit 2 - transaction output has been modified since it was loaded -// bit 3 - containing transaction has an expiry -// bits 4-7 - transaction type +// bit 1 - containing transaction has an expiry +// bits 2-5 - transaction type type utxoFlags uint8 const ( // utxoFlagCoinBase indicates that a txout was contained in a coinbase tx. utxoFlagCoinBase utxoFlags = 1 << iota - // utxoFlagSpent indicates that a txout is spent. - utxoFlagSpent - - // utxoFlagModified indicates that a txout has been modified since it was - // loaded. - utxoFlagModified - // utxoFlagHasExpiry indicates that a txout was contained in a tx that // included an expiry. utxoFlagHasExpiry ) const ( - // utxoFlagTxTypeBitmask describes the bitmask that yields bits 4-7 from + // utxoFlagTxTypeBitmask describes the bitmask that yields bits 2-5 from // utxoFlags. - utxoFlagTxTypeBitmask = 0xf0 + utxoFlagTxTypeBitmask = 0x3c // utxoFlagTxTypeShift is the number of bits to shift utxoFlags to the right // to yield the correct integer value after applying the bitmask with AND. - utxoFlagTxTypeShift = 4 + utxoFlagTxTypeShift = 2 ) // encodeUtxoFlags returns utxoFlags representing the passed parameters. -func encodeUtxoFlags(coinbase bool, spent bool, modified bool, hasExpiry bool, - txType stake.TxType) utxoFlags { - +func encodeUtxoFlags(coinbase bool, hasExpiry bool, txType stake.TxType) utxoFlags { packedFlags := utxoFlags(txType) << utxoFlagTxTypeShift if coinbase { packedFlags |= utxoFlagCoinBase } - if spent { - packedFlags |= utxoFlagSpent - } - if modified { - packedFlags |= utxoFlagModified - } if hasExpiry { packedFlags |= utxoFlagHasExpiry } @@ -100,16 +102,20 @@ type UtxoEntry struct { blockIndex uint32 scriptVersion uint16 - // packedFlags contains additional info about the output as defined by - // utxoFlags. This approach is used in order to reduce memory usage since - // there will be a lot of these in memory. + // state contains info for the in-memory state of the output as defined by + // utxoState. + state utxoState + + // packedFlags contains additional info for the containing transaction of the + // output as defined by utxoFlags. This approach is used in order to reduce + // memory usage since there will be a lot of these in memory. packedFlags utxoFlags } // isModified returns whether or not the output has been modified since it was // loaded. func (entry *UtxoEntry) isModified() bool { - return entry.packedFlags&utxoFlagModified == utxoFlagModified + return entry.state&utxoStateModified == utxoStateModified } // IsCoinBase returns whether or not the output was contained in a coinbase @@ -121,7 +127,7 @@ func (entry *UtxoEntry) IsCoinBase() bool { // IsSpent returns whether or not the output has been spent based upon the // current state of the unspent transaction output view it was obtained from. func (entry *UtxoEntry) IsSpent() bool { - return entry.packedFlags&utxoFlagSpent == utxoFlagSpent + return entry.state&utxoStateSpent == utxoStateSpent } // HasExpiry returns whether or not the output was contained in a transaction @@ -157,7 +163,7 @@ func (entry *UtxoEntry) Spend() { } // Mark the output as spent and modified. - entry.packedFlags |= utxoFlagSpent | utxoFlagModified + entry.state |= utxoStateSpent | utxoStateModified } // Amount returns the amount of the output. @@ -203,6 +209,7 @@ func (entry *UtxoEntry) Clone() *UtxoEntry { blockHeight: entry.blockHeight, blockIndex: entry.blockIndex, scriptVersion: entry.scriptVersion, + state: entry.state, packedFlags: entry.packedFlags, } diff --git a/blockchain/utxoentry_test.go b/blockchain/utxoentry_test.go index 77ff357057..053149cb4f 100644 --- a/blockchain/utxoentry_test.go +++ b/blockchain/utxoentry_test.go @@ -20,40 +20,31 @@ func TestEncodeUtxoFlags(t *testing.T) { tests := []struct { name string coinbase bool - spent bool - modified bool hasExpiry bool txType stake.TxType want utxoFlags }{{ name: "no flags set, regular tx", coinbase: false, - spent: false, - modified: false, hasExpiry: false, txType: stake.TxTypeRegular, want: 0x00, }, { name: "coinbase, has expiry, vote tx", coinbase: true, - spent: false, - modified: false, hasExpiry: true, txType: stake.TxTypeSSGen, - want: 0x29, + want: 0x0b, }, { - name: "spent, modified, has expiry, ticket tx", + name: "has expiry, ticket tx", coinbase: false, - spent: true, - modified: true, hasExpiry: true, txType: stake.TxTypeSStx, - want: 0x1e, + want: 0x06, }} for _, test := range tests { - got := encodeUtxoFlags(test.coinbase, test.spent, test.modified, - test.hasExpiry, test.txType) + got := encodeUtxoFlags(test.coinbase, test.hasExpiry, test.txType) if got != test.want { t.Errorf("%q: unexpected result -- got %x, want %x", test.name, got, test.want) @@ -104,9 +95,9 @@ func TestUtxoEntry(t *testing.T) { tests := []struct { name string - coinbase bool spent bool modified bool + coinbase bool expiry bool txType stake.TxType amount int64 @@ -171,13 +162,26 @@ func TestUtxoEntry(t *testing.T) { scriptVersion: test.scriptVersion, packedFlags: encodeUtxoFlags( test.coinbase, - test.spent, - test.modified, test.expiry, test.txType, ), } + // Set state flags given the parameters for the current test. + if test.spent { + entry.state |= utxoStateSpent + } + if test.modified { + entry.state |= utxoStateModified + } + + // Validate the spent flag. + isSpent := entry.IsSpent() + if isSpent != test.spent { + t.Fatalf("%q: unexpected spent flag -- got %v, want %v", test.name, + isSpent, test.spent) + } + // Validate the modified flag. isModified := entry.isModified() if isModified != test.modified { @@ -192,13 +196,6 @@ func TestUtxoEntry(t *testing.T) { isCoinBase, test.coinbase) } - // Validate the spent flag. - isSpent := entry.IsSpent() - if isSpent != test.spent { - t.Fatalf("%q: unexpected spent flag -- got %v, want %v", test.name, - isSpent, test.spent) - } - // Validate the expiry flag. hasExpiry := entry.HasExpiry() if hasExpiry != test.expiry { @@ -206,6 +203,13 @@ func TestUtxoEntry(t *testing.T) { hasExpiry, test.expiry) } + // Validate the type of the transaction that the output is contained in. + gotTxType := entry.TransactionType() + if gotTxType != test.txType { + t.Fatalf("%q: unexpected transaction type -- got %v, want %v", test.name, + gotTxType, test.txType) + } + // Validate the height of the block containing the output. gotBlockHeight := entry.BlockHeight() if gotBlockHeight != int64(test.blockHeight) { @@ -220,13 +224,6 @@ func TestUtxoEntry(t *testing.T) { gotBlockIndex, test.blockIndex) } - // Validate the type of the transaction that the output is contained in. - gotTxType := entry.TransactionType() - if gotTxType != test.txType { - t.Fatalf("%q: unexpected transaction type -- got %v, want %v", test.name, - gotTxType, test.txType) - } - // Validate the amount of the output. gotAmount := entry.Amount() if gotAmount != test.amount { diff --git a/blockchain/utxoviewpoint.go b/blockchain/utxoviewpoint.go index caa767c053..629dda5f4a 100644 --- a/blockchain/utxoviewpoint.go +++ b/blockchain/utxoviewpoint.go @@ -79,6 +79,11 @@ func (view *UtxoViewpoint) addTxOut(outpoint wire.OutPoint, txOut *wire.TxOut, entry.scriptVersion = txOut.Version entry.packedFlags = packedFlags entry.ticketMinOuts = ticketMinOuts + + // The referenced transaction output should always be marked as unspent and + // modified when being added to the view. + entry.state &^= utxoStateSpent + entry.state |= utxoStateModified } // AddTxOut adds the specified output of the passed transaction to the view if @@ -96,15 +101,13 @@ func (view *UtxoViewpoint) AddTxOut(tx *dcrutil.Tx, txOutIdx uint32, // Set encoded flags for the transaction. isCoinBase := standalone.IsCoinBaseTx(msgTx, isTreasuryEnabled) - const spent = false hasExpiry := msgTx.Expiry != wire.NoExpiryValue - const modified = true txType := stake.DetermineTxType(msgTx, isTreasuryEnabled) tree := wire.TxTreeRegular if txType != stake.TxTypeRegular { tree = wire.TxTreeStake } - flags := encodeUtxoFlags(isCoinBase, spent, modified, hasExpiry, txType) + flags := encodeUtxoFlags(isCoinBase, hasExpiry, txType) // Update existing entries. All fields are updated because it's possible // (although extremely unlikely) that the existing entry is being replaced by @@ -131,15 +134,13 @@ func (view *UtxoViewpoint) AddTxOuts(tx *dcrutil.Tx, blockHeight int64, blockInd // Set encoded flags for the transaction. isCoinBase := standalone.IsCoinBaseTx(msgTx, isTreasuryEnabled) - const spent = false hasExpiry := msgTx.Expiry != wire.NoExpiryValue - const modified = true txType := stake.DetermineTxType(msgTx, isTreasuryEnabled) tree := wire.TxTreeRegular if txType != stake.TxTypeRegular { tree = wire.TxTreeStake } - flags := encodeUtxoFlags(isCoinBase, spent, modified, hasExpiry, txType) + flags := encodeUtxoFlags(isCoinBase, hasExpiry, txType) // Loop through all of the transaction outputs and add those which are not // provably unspendable. @@ -325,16 +326,14 @@ func (view *UtxoViewpoint) disconnectTransactions(block *dcrutil.Block, stxos [] outpoint.Index = uint32(txOutIdx) entry := view.entries[outpoint] if entry == nil { - const spent = false - const modified = true entry = &UtxoEntry{ amount: txOut.Value, pkScript: txOut.PkScript, blockHeight: uint32(block.Height()), blockIndex: uint32(txIdx), scriptVersion: txOut.Version, - packedFlags: encodeUtxoFlags(isCoinBase, spent, modified, hasExpiry, - txType), + state: utxoStateModified, + packedFlags: encodeUtxoFlags(isCoinBase, hasExpiry, txType), } if isTicketSubmissionOutput(txType, uint32(txOutIdx)) { @@ -374,8 +373,6 @@ func (view *UtxoViewpoint) disconnectTransactions(block *dcrutil.Block, stxos [] txIn := msgTx.TxIn[txInIdx] entry := view.entries[txIn.PreviousOutPoint] if entry == nil { - const spent = false - const modified = true entry = &UtxoEntry{ amount: txIn.ValueIn, pkScript: stxo.pkScript, @@ -383,8 +380,9 @@ func (view *UtxoViewpoint) disconnectTransactions(block *dcrutil.Block, stxos [] blockHeight: stxo.blockHeight, blockIndex: stxo.blockIndex, scriptVersion: stxo.scriptVersion, - packedFlags: encodeUtxoFlags(stxo.IsCoinBase(), spent, modified, - stxo.HasExpiry(), stxo.TransactionType()), + state: utxoStateModified, + packedFlags: encodeUtxoFlags(stxo.IsCoinBase(), stxo.HasExpiry(), + stxo.TransactionType()), } view.entries[txIn.PreviousOutPoint] = entry @@ -392,8 +390,8 @@ func (view *UtxoViewpoint) disconnectTransactions(block *dcrutil.Block, stxos [] // Mark the existing referenced transaction output as unspent and // modified. - entry.packedFlags &^= utxoFlagSpent - entry.packedFlags |= utxoFlagModified + entry.state &^= utxoStateSpent + entry.state |= utxoStateModified } } @@ -601,7 +599,7 @@ func (view *UtxoViewpoint) commit() { continue } - entry.packedFlags &^= utxoFlagModified + entry.state &^= utxoStateModified } } From e23c565d3d3ba7d2de43bab3675b53fc0b914789 Mon Sep 17 00:00:00 2001 From: Ryan Staudt Date: Thu, 4 Feb 2021 05:45:19 -0600 Subject: [PATCH 03/10] blockchain: Add utxoStateFresh to UtxoEntry. This adds utxoStateFresh to UtxoEntry to indicate that a txout is fresh, which means that it exists in the utxo cache but does not exist in the underlying database. The utxo cache will use the fresh flag as an optimization to skip writing to the database for outputs that are added and spent in between flushes to the database. --- blockchain/utxoentry.go | 16 ++++++++++++++-- blockchain/utxoentry_test.go | 12 ++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/blockchain/utxoentry.go b/blockchain/utxoentry.go index 43ecb394da..307089bcb8 100644 --- a/blockchain/utxoentry.go +++ b/blockchain/utxoentry.go @@ -4,14 +4,17 @@ package blockchain -import "github.com/decred/dcrd/blockchain/stake/v4" +import ( + "github.com/decred/dcrd/blockchain/stake/v4" +) // utxoState defines the in-memory state of a utxo entry. // // The bit representation is: // bit 0 - transaction output has been spent // bit 1 - transaction output has been modified since it was loaded -// bits 2-7 - unused +// bit 2 - transaction output is fresh +// bits 3-7 - unused type utxoState uint8 const ( @@ -21,6 +24,10 @@ const ( // utxoStateModified indicates that a txout has been modified since it was // loaded. utxoStateModified + + // utxoStateFresh indicates that a txout is fresh, which means that it exists + // in the utxo cache but does not exist in the underlying database. + utxoStateFresh ) // utxoFlags defines additional information for the containing transaction of a @@ -118,6 +125,11 @@ func (entry *UtxoEntry) isModified() bool { return entry.state&utxoStateModified == utxoStateModified } +// isFresh returns whether or not the output is fresh. +func (entry *UtxoEntry) isFresh() bool { + return entry.state&utxoStateFresh == utxoStateFresh +} + // IsCoinBase returns whether or not the output was contained in a coinbase // transaction. func (entry *UtxoEntry) IsCoinBase() bool { diff --git a/blockchain/utxoentry_test.go b/blockchain/utxoentry_test.go index 053149cb4f..6ffa949c1e 100644 --- a/blockchain/utxoentry_test.go +++ b/blockchain/utxoentry_test.go @@ -97,6 +97,7 @@ func TestUtxoEntry(t *testing.T) { name string spent bool modified bool + fresh bool coinbase bool expiry bool txType stake.TxType @@ -119,6 +120,7 @@ func TestUtxoEntry(t *testing.T) { scriptVersion: 0, }, { name: "ticket submission output", + fresh: true, expiry: true, txType: stake.TxTypeSStx, amount: 4294959555, @@ -174,6 +176,9 @@ func TestUtxoEntry(t *testing.T) { if test.modified { entry.state |= utxoStateModified } + if test.fresh { + entry.state |= utxoStateFresh + } // Validate the spent flag. isSpent := entry.IsSpent() @@ -189,6 +194,13 @@ func TestUtxoEntry(t *testing.T) { isModified, test.modified) } + // Validate the fresh flag. + isFresh := entry.isFresh() + if isFresh != test.fresh { + t.Fatalf("%q: unexpected fresh flag -- got %v, want %v", test.name, + isFresh, test.fresh) + } + // Validate the coinbase flag. isCoinBase := entry.IsCoinBase() if isCoinBase != test.coinbase { From c5d7003c6f4126c88a19b9c0961c36caf5e1faa4 Mon Sep 17 00:00:00 2001 From: Ryan Staudt Date: Sun, 24 Jan 2021 05:20:35 -0600 Subject: [PATCH 04/10] blockchain: Add size method to UtxoEntry. This adds a size method to UtxoEntry, which returns the number of bytes that the entry uses on a 64-bit platform. This will be used as part of tracking the total size of the utxo cache. --- blockchain/utxoentry.go | 16 ++++++++++++++++ blockchain/utxoentry_test.go | 12 ++++++++++++ 2 files changed, 28 insertions(+) diff --git a/blockchain/utxoentry.go b/blockchain/utxoentry.go index 307089bcb8..ac33452946 100644 --- a/blockchain/utxoentry.go +++ b/blockchain/utxoentry.go @@ -8,6 +8,13 @@ import ( "github.com/decred/dcrd/blockchain/stake/v4" ) +const ( + // baseEntrySize is the base size of a utxo entry on a 64-bit platform, + // excluding the contents of the script and ticket minimal outputs. It is + // equivalent to what unsafe.Sizeof(UtxoEntry{}) returns on a 64-bit platform. + baseEntrySize = 56 +) + // utxoState defines the in-memory state of a utxo entry. // // The bit representation is: @@ -119,6 +126,15 @@ type UtxoEntry struct { packedFlags utxoFlags } +// size returns the number of bytes that the entry uses on a 64-bit platform. +func (entry *UtxoEntry) size() uint64 { + size := baseEntrySize + len(entry.pkScript) + if entry.ticketMinOuts != nil { + size += len(entry.ticketMinOuts.data) + } + return uint64(size) +} + // isModified returns whether or not the output has been modified since it was // loaded. func (entry *UtxoEntry) isModified() bool { diff --git a/blockchain/utxoentry_test.go b/blockchain/utxoentry_test.go index 6ffa949c1e..94968b69cc 100644 --- a/blockchain/utxoentry_test.go +++ b/blockchain/utxoentry_test.go @@ -108,6 +108,7 @@ func TestUtxoEntry(t *testing.T) { scriptVersion uint16 ticketMinOuts *ticketMinimalOutputs deserializedTicketMinOuts []*stake.MinimalOutput + size uint64 }{{ name: "coinbase output", coinbase: true, @@ -118,6 +119,8 @@ func TestUtxoEntry(t *testing.T) { blockHeight: 54321, blockIndex: 0, scriptVersion: 0, + // baseEntrySize + len(pkScript). + size: baseEntrySize + 25, }, { name: "ticket submission output", fresh: true, @@ -151,6 +154,8 @@ func TestUtxoEntry(t *testing.T) { Value: 0, Version: 0, }}, + // baseEntrySize + len(pkScript) + len(ticketMinOuts.data). + size: baseEntrySize + 26 + 99, }} for _, test := range tests { @@ -180,6 +185,13 @@ func TestUtxoEntry(t *testing.T) { entry.state |= utxoStateFresh } + // Validate the size of the entry. + size := entry.size() + if size != test.size { + t.Fatalf("%q: unexpected size -- got %v, want %v", test.name, size, + test.size) + } + // Validate the spent flag. isSpent := entry.IsSpent() if isSpent != test.spent { From 4b42840e7d3c47714c80ecace251ecb2928e1d47 Mon Sep 17 00:00:00 2001 From: Ryan Staudt Date: Sun, 24 Jan 2021 05:41:31 -0600 Subject: [PATCH 05/10] config: Add utxocachemaxsize. This adds a utxocachemaxsize configuration option which represents the maximum size in MiB of the utxo cache. The default value is 150 MiB, the minimum value is 25 MiB, and the maximum value is 32768 MiB (32 GiB). --- config.go | 72 +++++++++++++++++++++--------------- doc.go | 4 +- sampleconfig/sampleconfig.go | 8 +++- 3 files changed, 52 insertions(+), 32 deletions(-) diff --git a/config.go b/config.go index cc026d88b7..477684c326 100644 --- a/config.go +++ b/config.go @@ -1,5 +1,5 @@ // Copyright (c) 2013-2016 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -38,13 +38,16 @@ import ( const ( // Defaults for general application behavior options. - defaultConfigFilename = "dcrd.conf" - defaultDataDirname = "data" - defaultLogDirname = "logs" - defaultLogFilename = "dcrd.log" - defaultDbType = "ffldb" - defaultLogLevel = "info" - defaultSigCacheMaxSize = 100000 + defaultConfigFilename = "dcrd.conf" + defaultDataDirname = "data" + defaultLogDirname = "logs" + defaultLogFilename = "dcrd.log" + defaultDbType = "ffldb" + defaultLogLevel = "info" + defaultSigCacheMaxSize = 100000 + defaultUtxoCacheMaxSize = 150 + minUtxoCacheMaxSize = 25 + maxUtxoCacheMaxSize = 32768 // 32 GiB // Defaults for RPC server options and policy. defaultTLSCurve = "P-256" @@ -118,21 +121,22 @@ func minUint32(a, b uint32) uint32 { // See loadConfig for details on the configuration load process. type config struct { // General application behavior. - ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` - HomeDir string `short:"A" long:"appdata" description:"Path to application home directory"` - ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` - DataDir string `short:"b" long:"datadir" description:"Directory to store data"` - LogDir string `long:"logdir" description:"Directory to log output"` - NoFileLogging bool `long:"nofilelogging" description:"Disable file logging"` - DbType string `long:"dbtype" description:"Database backend to use for the block chain"` - Profile string `long:"profile" description:"Enable HTTP profiling on given [addr:]port -- NOTE port must be between 1024 and 65536"` - CPUProfile string `long:"cpuprofile" description:"Write CPU profile to the specified file"` - MemProfile string `long:"memprofile" description:"Write mem profile to the specified file"` - TestNet bool `long:"testnet" description:"Use the test network"` - SimNet bool `long:"simnet" description:"Use the simulation test network"` - RegNet bool `long:"regnet" description:"Use the regression test network"` - DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify =,=,... to set the log level for individual subsystems -- Use show to list available subsystems"` - SigCacheMaxSize uint `long:"sigcachemaxsize" description:"The maximum number of entries in the signature verification cache"` + ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` + HomeDir string `short:"A" long:"appdata" description:"Path to application home directory"` + ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` + DataDir string `short:"b" long:"datadir" description:"Directory to store data"` + LogDir string `long:"logdir" description:"Directory to log output"` + NoFileLogging bool `long:"nofilelogging" description:"Disable file logging"` + DbType string `long:"dbtype" description:"Database backend to use for the block chain"` + Profile string `long:"profile" description:"Enable HTTP profiling on given [addr:]port -- NOTE port must be between 1024 and 65536"` + CPUProfile string `long:"cpuprofile" description:"Write CPU profile to the specified file"` + MemProfile string `long:"memprofile" description:"Write mem profile to the specified file"` + TestNet bool `long:"testnet" description:"Use the test network"` + SimNet bool `long:"simnet" description:"Use the simulation test network"` + RegNet bool `long:"regnet" description:"Use the regression test network"` + DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify =,=,... to set the log level for individual subsystems -- Use show to list available subsystems"` + SigCacheMaxSize uint `long:"sigcachemaxsize" description:"The maximum number of entries in the signature verification cache"` + UtxoCacheMaxSize uint `long:"utxocachemaxsize" description:"The maximum size in MiB of the utxo cache"` // RPC server options and policy. DisableRPC bool `long:"norpc" description:"Disable built-in RPC server -- NOTE: The RPC server is disabled by default if no rpcuser/rpcpass or rpclimituser/rpclimitpass is specified"` @@ -560,13 +564,14 @@ func loadConfig(appName string) (*config, []string, error) { // Default config. cfg := config{ // General application behavior. - HomeDir: defaultHomeDir, - ConfigFile: defaultConfigFile, - DataDir: defaultDataDir, - LogDir: defaultLogDir, - DbType: defaultDbType, - DebugLevel: defaultLogLevel, - SigCacheMaxSize: defaultSigCacheMaxSize, + HomeDir: defaultHomeDir, + ConfigFile: defaultConfigFile, + DataDir: defaultDataDir, + LogDir: defaultLogDir, + DbType: defaultDbType, + DebugLevel: defaultLogLevel, + SigCacheMaxSize: defaultSigCacheMaxSize, + UtxoCacheMaxSize: defaultUtxoCacheMaxSize, // RPC server options and policy. RPCCert: defaultRPCCertFile, @@ -846,6 +851,13 @@ func loadConfig(appName string) (*config, []string, error) { return nil, nil, err } + // Enforce the minimum and maximum utxo cache max size. + if cfg.UtxoCacheMaxSize < minUtxoCacheMaxSize { + cfg.UtxoCacheMaxSize = minUtxoCacheMaxSize + } else if cfg.UtxoCacheMaxSize > maxUtxoCacheMaxSize { + cfg.UtxoCacheMaxSize = maxUtxoCacheMaxSize + } + // Validate format of profile, can be an address:port, or just a port. if cfg.Profile != "" { // if profile is just a number, then add a default host of "127.0.0.1" such that Profile is a valid tcp address diff --git a/doc.go b/doc.go index 9642ab1c93..60ab5178eb 100644 --- a/doc.go +++ b/doc.go @@ -1,5 +1,5 @@ // Copyright (c) 2013-2016 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -45,6 +45,8 @@ Application Options: Use show to list available subsystems (info) --sigcachemaxsize= The maximum number of entries in the signature verification cache (default: 100000) + --utxocachemaxsize= The maximum size in MiB of the utxo cache + (default: 150, minimum: 25, maximum: 32768) --norpc Disable built-in RPC server -- NOTE: The RPC server is disabled by default if no rpcuser/rpcpass or rpclimituser/rpclimitpass is diff --git a/sampleconfig/sampleconfig.go b/sampleconfig/sampleconfig.go index 78e1bcb323..951135528b 100644 --- a/sampleconfig/sampleconfig.go +++ b/sampleconfig/sampleconfig.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020 The Decred developers +// Copyright (c) 2017-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -292,6 +292,12 @@ const fileContents = `[Application Options] ; Limit the signature cache to a max of 50000 entries. ; sigcachemaxsize=50000 +; ------------------------------------------------------------------------------ +; Unspent Transaction Output (UTXO) Cache +; ------------------------------------------------------------------------------ + +; Limit the utxo cache to a max of 100 MiB. +; utxocachemaxsize=150 ; ------------------------------------------------------------------------------ ; Coin Generation (Mining) Settings - The following options control the From 8de043cd73ba3674462a3a9f98512b3052f12cfd Mon Sep 17 00:00:00 2001 From: Ryan Staudt Date: Wed, 10 Feb 2021 06:06:05 -0600 Subject: [PATCH 06/10] blockchain: Deep copy view entry script from tx. This updates utxo viewpoints to deep copy the script when getting it from a msg tx. This is required since the tx out script is a subslice of the overall contiguous buffer that the msg tx houses for all scripts within the tx. It is deep copied here since this entry may be added to the utxo cache, and we don't want the utxo cache holding the entry to prevent all of the other tx scripts from getting garbage collected. --- blockchain/utxoviewpoint.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/blockchain/utxoviewpoint.go b/blockchain/utxoviewpoint.go index 629dda5f4a..f155014148 100644 --- a/blockchain/utxoviewpoint.go +++ b/blockchain/utxoviewpoint.go @@ -73,7 +73,6 @@ func (view *UtxoViewpoint) addTxOut(outpoint wire.OutPoint, txOut *wire.TxOut, } entry.amount = txOut.Value - entry.pkScript = txOut.PkScript entry.blockHeight = uint32(blockHeight) entry.blockIndex = blockIndex entry.scriptVersion = txOut.Version @@ -84,6 +83,17 @@ func (view *UtxoViewpoint) addTxOut(outpoint wire.OutPoint, txOut *wire.TxOut, // modified when being added to the view. entry.state &^= utxoStateSpent entry.state |= utxoStateModified + + // Deep copy the script. This is required since the tx out script is a + // subslice of the overall contiguous buffer that the msg tx houses for all + // scripts within the tx. It is deep copied here since this entry may be + // added to the utxo cache, and we don't want the utxo cache holding the entry + // to prevent all of the other tx scripts from getting garbage collected. + scriptLen := len(txOut.PkScript) + if scriptLen != 0 { + entry.pkScript = make([]byte, scriptLen) + copy(entry.pkScript, txOut.PkScript) + } } // AddTxOut adds the specified output of the passed transaction to the view if @@ -328,7 +338,6 @@ func (view *UtxoViewpoint) disconnectTransactions(block *dcrutil.Block, stxos [] if entry == nil { entry = &UtxoEntry{ amount: txOut.Value, - pkScript: txOut.PkScript, blockHeight: uint32(block.Height()), blockIndex: uint32(txIdx), scriptVersion: txOut.Version, @@ -336,6 +345,18 @@ func (view *UtxoViewpoint) disconnectTransactions(block *dcrutil.Block, stxos [] packedFlags: encodeUtxoFlags(isCoinBase, hasExpiry, txType), } + // Deep copy the script. This is required since the tx out script is a + // subslice of the overall contiguous buffer that the msg tx houses for + // all scripts within the tx. It is deep copied here since this entry + // may be added to the utxo cache, and we don't want the utxo cache + // holding the entry to prevent all of the other tx scripts from getting + // garbage collected. + scriptLen := len(txOut.PkScript) + if scriptLen != 0 { + entry.pkScript = make([]byte, scriptLen) + copy(entry.pkScript, txOut.PkScript) + } + if isTicketSubmissionOutput(txType, uint32(txOutIdx)) { entry.ticketMinOuts = &ticketMinimalOutputs{ data: make([]byte, serializeSizeForMinimalOutputs(tx)), From d0814b1aa5dea601a89f6820a5bd0945cce70dae Mon Sep 17 00:00:00 2001 From: Ryan Staudt Date: Sat, 23 Jan 2021 06:52:01 -0600 Subject: [PATCH 07/10] blockchain: Add utxoSetState to the database. This adds a utxoSetState type that is tracked in the database. The utxo set state contains information regarding the current state of the utxo set. In particular, it tracks the block height and block hash of the last completed flush. The utxo set state is tracked in the database since at any given time, the utxo cache may not be consistent with the utxo set in the database. This is due to the fact that the utxo cache only flushes changes to the database periodically. Therefore, during initialization, the utxo set state is used to identify the last flushed state of the utxo set and it can be caught up to the current best state of the main chain. This additionally adds full test coverage to the new serialization and deserialization functions. --- blockchain/chainio.go | 99 ++++++++++++++++++++++ blockchain/chainio_test.go | 163 ++++++++++++++++++++++++++++++++++++- 2 files changed, 260 insertions(+), 2 deletions(-) diff --git a/blockchain/chainio.go b/blockchain/chainio.go index 3dbe9ddf70..7c1423572e 100644 --- a/blockchain/chainio.go +++ b/blockchain/chainio.go @@ -80,6 +80,10 @@ var ( // transaction output set. utxoSetBucketName = []byte("utxosetv3") + // utxoSetStateKeyName is the name of the database key used to house the + // state of the unspent transaction output set. + utxoSetStateKeyName = []byte("utxosetstate") + // blockIndexBucketName is the name of the db bucket used to house the block // index which consists of metadata for all known blocks both in the main // chain and on side chains. @@ -1155,6 +1159,101 @@ func dbPutUtxoView(dbTx database.Tx, view *UtxoViewpoint) error { return nil } +// ----------------------------------------------------------------------------- +// The utxo set state contains information regarding the current state of the +// utxo set. In particular, it tracks the block height and block hash of the +// last completed flush. +// +// The utxo set state is tracked in the database since at any given time, the +// utxo cache may not be consistent with the utxo set in the database. This is +// due to the fact that the utxo cache only flushes changes to the database +// periodically. Therefore, during initialization, the utxo set state is used +// to identify the last flushed state of the utxo set and it can be caught up +// to the current best state of the main chain. +// +// Note: The utxo set state MUST always be updated in the same database +// transaction that the utxo set is updated in to guarantee that they stay in +// sync in the database. +// +// The serialized format is: +// +// +// +// Field Type Size +// block height VLQ variable +// block hash chainhash.Hash chainhash.HashSize +// +// ----------------------------------------------------------------------------- + +// utxoSetState represents the current state of the utxo set. In particular, +// it tracks the block height and block hash of the last completed flush. +type utxoSetState struct { + lastFlushHeight uint32 + lastFlushHash chainhash.Hash +} + +// serializeUtxoSetState serializes the provided utxo set state. The format is +// described in detail above. +func serializeUtxoSetState(state *utxoSetState) []byte { + // Calculate the size needed to serialize the utxo set state. + size := serializeSizeVLQ(uint64(state.lastFlushHeight)) + chainhash.HashSize + + // Serialize the utxo set state and return it. + serialized := make([]byte, size) + offset := putVLQ(serialized, uint64(state.lastFlushHeight)) + copy(serialized[offset:], state.lastFlushHash[:]) + return serialized +} + +// deserializeUtxoSetState deserializes the passed serialized byte slice into +// the utxo set state. The format is described in detail above. +func deserializeUtxoSetState(serialized []byte) (*utxoSetState, error) { + // Deserialize the block height. + blockHeight, bytesRead := deserializeVLQ(serialized) + offset := bytesRead + if offset >= len(serialized) { + return nil, errDeserialize("unexpected end of data after height") + } + + // Deserialize the hash. + if len(serialized[offset:]) != chainhash.HashSize { + return nil, errDeserialize("unexpected length for serialized hash") + } + var hash chainhash.Hash + copy(hash[:], serialized[offset:offset+chainhash.HashSize]) + + // Create the utxo set state and return it. + return &utxoSetState{ + lastFlushHeight: uint32(blockHeight), + lastFlushHash: hash, + }, nil +} + +// dbPutUtxoSetState uses an existing database transaction to update the utxo +// set state in the database. +func dbPutUtxoSetState(dbTx database.Tx, state *utxoSetState) error { + // Serialize and store the utxo set state. + return dbTx.Metadata().Put(utxoSetStateKeyName, serializeUtxoSetState(state)) +} + +// dbFetchUtxoSetState uses an existing database transaction to fetch the utxo +// set state from the database. If the utxo set state does not exist in the +// database, nil is returned. +func dbFetchUtxoSetState(dbTx database.Tx) (*utxoSetState, error) { + // Fetch the serialized utxo set state from the database. + serialized := dbTx.Metadata().Get(utxoSetStateKeyName) + + // Return nil if the utxo set state does not exist in the database. This + // should only be the case when starting from a fresh database or a database + // that has not been run with the utxo cache yet. + if serialized == nil { + return nil, nil + } + + // Deserialize the utxo set state and return it. + return deserializeUtxoSetState(serialized) +} + // ----------------------------------------------------------------------------- // The GCS filter journal consists of an entry for each block connected to the // main chain (or has ever been connected to it) which consists of a serialized diff --git a/blockchain/chainio_test.go b/blockchain/chainio_test.go index 8dffdde0df..37a4c3df88 100644 --- a/blockchain/chainio_test.go +++ b/blockchain/chainio_test.go @@ -10,6 +10,8 @@ import ( "encoding/hex" "errors" "math/big" + "os" + "path/filepath" "reflect" "testing" "time" @@ -1000,12 +1002,12 @@ func TestUtxoSerialization(t *testing.T) { // Ensure that the serialized bytes are decoded back to the expected utxo. gotUtxo, err := deserializeUtxoEntry(test.serialized, test.txOutIndex) if err != nil { - t.Errorf("serializeUtxoEntry #%d (%s): unexpected error: %v", i, + t.Errorf("deserializeUtxoEntry #%d (%s): unexpected error: %v", i, test.name, err) continue } if !reflect.DeepEqual(gotUtxo, test.entry) { - t.Errorf("serializeUtxoEntry #%d (%s):\nwant: %+v\n got: %+v\n", i, + t.Errorf("deserializeUtxoEntry #%d (%s):\nwant: %+v\n got: %+v\n", i, test.name, test.entry, gotUtxo) } } @@ -1123,6 +1125,163 @@ func TestUtxoEntryDeserializeErrors(t *testing.T) { } } +// TestUtxoSetStateSerialization ensures that serializing and deserializing +// the utxo set state works as expected. +func TestUtxoSetStateSerialization(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + state *utxoSetState + serialized []byte + }{{ + name: "last flush height and hash updated", + state: &utxoSetState{ + lastFlushHeight: 432100, + lastFlushHash: *mustParseHash("000000000000000023455b4328635d8e014dbeea" + + "99c6140aa715836cc7e55981"), + }, + serialized: hexToBytes("99ae648159e5c76c8315a70a14c699eabe4d018e5d6328435" + + "b45230000000000000000"), + }, { + name: "last flush height and hash are the genesis block", + state: &utxoSetState{ + lastFlushHeight: 0, + lastFlushHash: *mustParseHash("298e5cc3d985bfe7f81dc135f360abe089edd439" + + "6b86d2de66b0cef42b21d980"), + }, + serialized: hexToBytes("0080d9212bf4ceb066ded2866b39d4ed89e0ab60f335c11df" + + "8e7bf85d9c35c8e29"), + }} + + for _, test := range tests { + // Ensure the utxo set state serializes to the expected value. + gotBytes := serializeUtxoSetState(test.state) + if !bytes.Equal(gotBytes, test.serialized) { + t.Errorf("serializeUtxoSetState (%s): mismatched bytes - got %x, "+ + "want %x", test.name, gotBytes, test.serialized) + continue + } + + // Ensure that the serialized bytes are decoded back to the expected utxo + // set state. + gotUtxoSetState, err := deserializeUtxoSetState(test.serialized) + if err != nil { + t.Errorf("deserializeUtxoSetState (%s): unexpected error: %v", test.name, + err) + continue + } + if !reflect.DeepEqual(gotUtxoSetState, test.state) { + t.Errorf("deserializeUtxoSetState (%s):\nwant: %+v\n got: %+v\n", + test.name, test.state, gotUtxoSetState) + } + } +} + +// TestUtxoSetStateDeserializeErrors performs negative tests against +// deserializing the utxo set state to ensure error paths work as expected. +func TestUtxoSetStateDeserializeErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + serialized []byte + errType error + }{{ + // [EOF] + name: "nothing serialized (no last flush height)", + serialized: hexToBytes(""), + errType: errDeserialize(""), + }, { + // [] + name: "no data after last flush height", + serialized: hexToBytes("99ae64"), + errType: errDeserialize(""), + }, { + // [] + name: "truncated hash", + serialized: hexToBytes("99ae648159e5c76c8315a70a14c699"), + errType: errDeserialize(""), + }} + + for _, test := range tests { + // Ensure the expected error type is returned and the returned + // utxo set state is nil. + entry, err := deserializeUtxoSetState(test.serialized) + if !errors.As(err, &test.errType) { + t.Errorf("deserializeUtxoSetState (%s): expected error type does not "+ + "match - got %T, want %T", test.name, err, test.errType) + continue + } + if entry != nil { + t.Errorf("deserializeUtxoSetState (%s): returned utxo set state is not "+ + "nil", test.name) + continue + } + } +} + +// TestDbFetchUtxoSetState ensures that putting and fetching the utxo set state +// works as expected. +func TestDbFetchUtxoSetState(t *testing.T) { + t.Parallel() + + // Create a test database. + dbPath := filepath.Join(os.TempDir(), "test-dbfetchutxosetstate") + _ = os.RemoveAll(dbPath) + db, err := database.Create("ffldb", dbPath, wire.MainNet) + if err != nil { + t.Fatalf("error creating test database: %v", err) + } + defer os.RemoveAll(dbPath) + defer db.Close() + + tests := []struct { + name string + state *utxoSetState + }{{ + name: "fresh database (no utxo set state saved)", + state: nil, + }, { + name: "last flush saved in database", + state: &utxoSetState{ + lastFlushHeight: 432100, + lastFlushHash: *mustParseHash("000000000000000023455b4328635d8e014dbeea" + + "99c6140aa715836cc7e55981"), + }, + }} + + for _, test := range tests { + // Update the utxo set state in the database. + if test.state != nil { + err = db.Update(func(dbTx database.Tx) error { + return dbPutUtxoSetState(dbTx, test.state) + }) + if err != nil { + t.Fatalf("%q: error putting utxo set state: %v", test.name, err) + } + } + + // Fetch the utxo set state from the database. + err = db.View(func(dbTx database.Tx) error { + gotState, err := dbFetchUtxoSetState(dbTx) + if err != nil { + return err + } + + // Ensure that the fetched utxo set state matches the expected state. + if !reflect.DeepEqual(gotState, test.state) { + t.Errorf("%q: mismatched state:\nwant: %+v\n got: %+v\n", test.name, + test.state, gotState) + } + return nil + }) + if err != nil { + t.Fatalf("%q: error fetching utxo set state: %v", test.name, err) + } + } +} + // TestBestChainStateSerialization ensures serializing and deserializing the // best chain state works as expected. func TestBestChainStateSerialization(t *testing.T) { From d12d1688d03247d2feb48bf52d9c089ab26f7a40 Mon Sep 17 00:00:00 2001 From: Ryan Staudt Date: Thu, 21 Jan 2021 16:25:50 -0600 Subject: [PATCH 08/10] multi: Add UtxoCache. UtxoCache is an unspent transaction output cache that sits on top of the utxo set database and provides significant runtime performance benefits at the cost of some additional memory usage. It drastically reduces the amount of reading and writing to disk, especially during initial block download when a very large number of blocks are being processed in quick succession. The UtxoCache is a read-through cache. All utxo reads go through the cache. When there is a cache miss, the cache loads the missing data from the database, caches it, and returns it to the caller. The UtxoCache is a write-back cache. Writes to the cache are acknowledged by the cache immediately but are only periodically flushed to the database. This allows intermediate steps to effectively be skipped. For example, a utxo that is created and then spent in between flushes never needs to be written to the utxo set in the database. Due to the write-back nature of the cache, at any given time the database may not be in sync with the cache, and therefore all utxo reads and writes MUST go through the cache, and never read or write to the database directly. An overview of the changes is as follows: - Add UtxoCache and UtxoCacheConfig struct types and NewUtxoCache method - Update server to create the utxo cache with the configured max size and pass to the block chain instance that is created - Update all test block chains to create a utxo cache - Add FetchEntry to UtxoCache - FetchEntry returns the specified transaction output from the utxo set - If the output exists in the cache, it is returned immediately. Otherwise, it uses an existing database transaction to fetch the output from the database, caches it, and returns it to the caller. - Add AddEntry to UtxoCache - AddEntry adds the specified output to the cache - Add SpendEntry to UtxoCache - SpendEntry marks the specified output as spent - Remove entries that are marked as fresh and then subsequently spent. This is an optimization to skip writing to the database for outputs that are added and spent in between flushes to the database. - Update UtxoViewpoint to hold the UtxoCache - Update fetching entries from the database to fetch entries from the cache instead - Add Commit to UtxoCache - Commit updates all entries in the cache based on the state of each entry in the provided view - All entries in the provided view that are marked as modified and spent are removed from the view - Additionally, all entries that are added to the cache are removed from the provided view - Add MaybeFlush to UtxoCache - MaybeFlush conditionally flushes the cache to the database - If the maximum size of the cache has been reached, or if the periodic flush duration has been reached, then a flush is required - A flush can be forced by setting the force flush parameter - Flushing commits all modified entries to the database and conditionally evicts entries - Entries that are nil or spent are always evicted since they are unlikely to be accessed again. Additionally, if the cache has reached its maximum size, entries are evicted based on the height of the block that they are contained in. - Update connect block and disconnect block to commit to the cache and conditionally flush to the database - Rather than writing to the utxo set in the database every time that a block is connected or disconnected, commit the updated view to the cache and call MaybeFlush on the cache to conditionally flush it to the database - Add InitUtxoCache to UtxoCache - InitUtxoCache initializes the utxo cache by ensuring that the utxo set is caught up to the tip of the best chain - Since the cache is only flushed to the database periodically, the utxo set may not be caught up to the tip of the best chain - InitUtxoCache catches the utxo set up by replaying all blocks from the block after the block that was last flushed to the tip block through the cache - Add ShutdownUtxoCache to BlockChain - ShutdownUtxoCache flushes the utxo cache to the database on shutdown. Since the cache is flushed periodically during initial block download and flushed after every block is connected after initial block download is complete, this flush that occurs during shutdown should finish relatively quickly - Note that if an unclean shutdown occurs, the cache will still be initialized properly when restarted as during initialization it will replay blocks to catch up to the tip block if it was not fully flushed before shutting down. However, it is still preferred to flush when shutting down versus always recovering on startup since it is faster - Track the hit ratio of UtxoCache - Track the number of hits and misses when accessing the cache in order to calculate the overall hit ratio of the cache to gauge its performance --- blockchain/chain.go | 91 ++- blockchain/chainio.go | 75 +- blockchain/common_test.go | 6 +- blockchain/example_test.go | 6 +- blockchain/fullblocks_test.go | 6 +- blockchain/headercmt.go | 4 +- blockchain/sequencelock_test.go | 2 +- blockchain/stakeext.go | 6 +- blockchain/utxocache.go | 910 +++++++++++++++++++++++++ blockchain/utxoviewpoint.go | 113 +-- blockchain/validate.go | 10 +- blockchain/validate_test.go | 6 +- internal/mempool/mempool_test.go | 4 +- internal/mining/mining_harness_test.go | 8 +- internal/rpcserver/interface.go | 5 + server.go | 9 +- 16 files changed, 1078 insertions(+), 183 deletions(-) create mode 100644 blockchain/utxocache.go diff --git a/blockchain/chain.go b/blockchain/chain.go index 92f1f40a96..3c22a9aef3 100644 --- a/blockchain/chain.go +++ b/blockchain/chain.go @@ -143,6 +143,7 @@ type BlockChain struct { sigCache *txscript.SigCache indexManager indexers.IndexManager interrupt <-chan struct{} + utxoCache *UtxoCache // subsidyCache is the cache that provides quick lookup of subsidy // values. @@ -619,14 +620,6 @@ func (b *BlockChain) connectBlock(node *blockNode, block, parent *dcrutil.Block, return err } - // Update the utxo set using the state of the utxo view. This - // entails removing all of the utxos spent and adding the new - // ones created by the block. - err = dbPutUtxoView(dbTx, view) - if err != nil { - return err - } - // Update the transaction spend journal by adding a record for // the block that contains all txos spent by it. err = dbPutSpendJournalEntry(dbTx, block.Hash(), stxos) @@ -676,9 +669,26 @@ func (b *BlockChain) connectBlock(node *blockNode, block, parent *dcrutil.Block, return err } - // Prune fully spent entries and mark all entries in the view unmodified - // now that the modifications have been committed to the database. - view.commit() + // Commit all entries in the view to the utxo cache. All entries in the view + // that are marked as modified and spent are removed from the view. + // Additionally, all entries that are added to the cache are removed from the + // view. + err = b.utxoCache.Commit(view) + if err != nil { + return err + } + + // Conditionally flush the utxo cache to the database. Force a flush if the + // chain believes it is current since blocks are connected infrequently at + // that point. Only log the flush when the chain is not current as it is + // mostly useful to see the flush details when many blocks are being connected + // (and subsequently flushed) in quick succession. + isCurrent := b.isCurrent(node) + err = b.utxoCache.MaybeFlush(&node.hash, uint32(node.height), isCurrent, + !isCurrent) + if err != nil { + return err + } // This node is now the end of the best chain. b.bestChain.SetTip(node) @@ -814,21 +824,6 @@ func (b *BlockChain) disconnectBlock(node *blockNode, block, parent *dcrutil.Blo return err } - // Update the utxo set using the state of the utxo view. This - // entails restoring all of the utxos spent and removing the new - // ones created by the block. - err = dbPutUtxoView(dbTx, view) - if err != nil { - return err - } - - // Update the transaction spend journal by removing the record - // that contains all txos spent by the block. - err = dbRemoveSpendJournalEntry(dbTx, block.Hash()) - if err != nil { - return err - } - err = stake.WriteDisconnectedBestNode(dbTx, parentStakeNode, node.parent.hash, childStakeNode.UndoData()) if err != nil { @@ -856,9 +851,35 @@ func (b *BlockChain) disconnectBlock(node *blockNode, block, parent *dcrutil.Blo return err } - // Prune fully spent entries and mark all entries in the view unmodified - // now that the modifications have been committed to the database. - view.commit() + // Commit all entries in the view to the utxo cache. All entries in the view + // that are marked as modified and spent are removed from the view. + // Additionally, all entries that are added to the cache are removed from the + // view. + err = b.utxoCache.Commit(view) + if err != nil { + return err + } + + // Force a utxo cache flush when blocks are being disconnected. A cache flush + // is forced here since the spend journal entry for the disconnected block + // will be removed below. + err = b.utxoCache.MaybeFlush(&node.parent.hash, uint32(node.parent.height), + true, false) + if err != nil { + return err + } + + // Update the transaction spend journal by removing the record that contains + // all txos spent by the block. This is intentionally done AFTER the utxo + // cache has been force flushed since the spend journal information will no + // longer be available for the cache to use for recovery purposes after being + // removed. + err = b.db.Update(func(dbTx database.Tx) error { + return dbRemoveSpendJournalEntry(dbTx, block.Hash()) + }) + if err != nil { + return err + } // This node's parent is now the end of the best chain. b.bestChain.SetTip(node.parent) @@ -993,7 +1014,7 @@ func (b *BlockChain) reorganizeChainInternal(target *blockNode) error { // using that information to unspend all of the spent txos and remove the // utxos created by the blocks. In addition, if a block votes against its // parent, the regular transactions are reconnected. - view := NewUtxoViewpoint() + view := NewUtxoViewpoint(b.utxoCache) view.SetBestHash(&tip.hash) var nextBlockToDetach *dcrutil.Block for tip != nil && tip != fork { @@ -1049,7 +1070,7 @@ func (b *BlockChain) reorganizeChainInternal(target *blockNode) error { // Update the view to unspend all of the spent txos and remove the utxos // created by the block. Also, if the block votes against its parent, // reconnect all of the regular transactions. - err = view.disconnectBlock(b.db, block, parent, stxos, isTreasuryEnabled) + err = view.disconnectBlock(block, parent, stxos, isTreasuryEnabled) if err != nil { return err } @@ -2125,6 +2146,13 @@ type Config struct { // This field can be nil if the caller does not wish to make use of an // index manager. IndexManager indexers.IndexManager + + // UtxoCache defines a utxo cache that sits on top of the utxo set database. + // All utxo reads and writes go through the cache, and never read or write to + // the database directly. + // + // This field is required. + UtxoCache *UtxoCache } // New returns a BlockChain instance using the provided configuration details. @@ -2190,6 +2218,7 @@ func New(ctx context.Context, config *Config) (*BlockChain, error) { calcPriorStakeVersionCache: make(map[[chainhash.HashSize]byte]uint32), calcVoterVersionIntervalCache: make(map[[chainhash.HashSize]byte]uint32), calcStakeVersionCache: make(map[[chainhash.HashSize]byte]uint32), + utxoCache: config.UtxoCache, } b.pruner = newChainPruner(&b) diff --git a/blockchain/chainio.go b/blockchain/chainio.go index 7c1423572e..bd7fffaf24 100644 --- a/blockchain/chainio.go +++ b/blockchain/chainio.go @@ -1117,43 +1117,42 @@ func dbFetchUtxoStats(dbTx database.Tx) (*UtxoStats, error) { return &stats, nil } -// dbPutUtxoView uses an existing database transaction to update the utxo set in -// the database based on the provided utxo view contents and state. In -// particular, only the entries that have been marked as modified are written to -// the database. -func dbPutUtxoView(dbTx database.Tx, view *UtxoViewpoint) error { - utxoBucket := dbTx.Metadata().Bucket(utxoSetBucketName) - for outpoint, entry := range view.entries { - // No need to update the database if the entry was not modified. - if entry == nil || !entry.isModified() { - continue - } - - // Remove the utxo entry if it is spent. - if entry.IsSpent() { - key := outpointKey(outpoint) - err := utxoBucket.Delete(*key) - recycleOutpointKey(key) - if err != nil { - return err - } - - continue - } +// dbPutUtxoEntry uses an existing database transaction to update the utxo +// entry for the given outpoint based on the provided utxo entry state. In +// particular, the entry is only written to the database if it is marked as +// modified, and if the entry is marked as spent it is removed from the +// database. +func dbPutUtxoEntry(dbTx database.Tx, outpoint wire.OutPoint, entry *UtxoEntry) error { + // No need to update the database if the entry was not modified. + if entry == nil || !entry.isModified() { + return nil + } - // Serialize and store the utxo entry. - serialized, err := serializeUtxoEntry(entry) - if err != nil { - return err - } + // Remove the utxo entry if it is spent. + utxoBucket := dbTx.Metadata().Bucket(utxoSetBucketName) + if entry.IsSpent() { key := outpointKey(outpoint) - err = utxoBucket.Put(*key, serialized) - // NOTE: The key is intentionally not recycled here since the database - // interface contract prohibits modifications. It will be garbage collected - // normally when the database is done with it. + err := utxoBucket.Delete(*key) + recycleOutpointKey(key) if err != nil { return err } + + return nil + } + + // Serialize and store the utxo entry. + serialized, err := serializeUtxoEntry(entry) + if err != nil { + return err + } + key := outpointKey(outpoint) + err = utxoBucket.Put(*key, serialized) + // NOTE: The key is intentionally not recycled here since the database + // interface contract prohibits modifications. It will be garbage collected + // normally when the database is done with it. + if err != nil { + return err } return nil @@ -1831,6 +1830,7 @@ func (b *BlockChain) initChainState(ctx context.Context) error { } // Attempt to load the chain state from the database. + var tip *blockNode err = b.db.View(func(dbTx database.Tx) error { // Fetch the stored best chain state from the database. state, err := dbFetchBestState(dbTx) @@ -1849,7 +1849,7 @@ func (b *BlockChain) initChainState(ctx context.Context) error { } // Set the best chain to the stored best state. - tip := b.index.lookupNode(&state.hash) + tip = b.index.lookupNode(&state.hash) if tip == nil { return AssertError(fmt.Sprintf("initChainState: cannot find "+ "chain tip %s in block index", state.hash)) @@ -1924,7 +1924,14 @@ func (b *BlockChain) initChainState(ctx context.Context) error { } // Upgrade the database post block index load as needed. - return upgradeDBPostBlockIndexLoad(ctx, b) + err = upgradeDBPostBlockIndexLoad(ctx, b) + if err != nil { + return err + } + + // Initialize the utxo cache to ensure that the state of the utxo set is + // caught up to the tip of the best chain. + return b.InitUtxoCache(tip) } // dbFetchBlockByNode uses an existing database transaction to retrieve the raw diff --git a/blockchain/common_test.go b/blockchain/common_test.go index 7c323dcc97..60954e44be 100644 --- a/blockchain/common_test.go +++ b/blockchain/common_test.go @@ -1,5 +1,5 @@ // Copyright (c) 2013-2016 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -117,6 +117,10 @@ func chainSetup(dbName string, params *chaincfg.Params) (*BlockChain, func(), er ChainParams: ¶msCopy, TimeSource: NewMedianTime(), SigCache: sigCache, + UtxoCache: NewUtxoCache(&UtxoCacheConfig{ + DB: db, + MaxSize: 100 * 1024 * 1024, // 100 MiB + }), }) if err != nil { diff --git a/blockchain/example_test.go b/blockchain/example_test.go index 54a27c4b1f..be9961154e 100644 --- a/blockchain/example_test.go +++ b/blockchain/example_test.go @@ -1,5 +1,5 @@ // Copyright (c) 2014-2016 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -52,6 +52,10 @@ func ExampleBlockChain_ProcessBlock() { DB: db, ChainParams: mainNetParams, TimeSource: blockchain.NewMedianTime(), + UtxoCache: blockchain.NewUtxoCache(&blockchain.UtxoCacheConfig{ + DB: db, + MaxSize: 100 * 1024 * 1024, // 100 MiB + }), }) if err != nil { fmt.Printf("Failed to create chain instance: %v\n", err) diff --git a/blockchain/fullblocks_test.go b/blockchain/fullblocks_test.go index 6d7a784ad6..f59ae7c188 100644 --- a/blockchain/fullblocks_test.go +++ b/blockchain/fullblocks_test.go @@ -1,5 +1,5 @@ // Copyright (c) 2016 The btcsuite developers -// Copyright (c) 2016-2020 The Decred developers +// Copyright (c) 2016-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -111,6 +111,10 @@ func chainSetup(dbName string, params *chaincfg.Params) (*blockchain.BlockChain, ChainParams: ¶msCopy, TimeSource: blockchain.NewMedianTime(), SigCache: sigCache, + UtxoCache: blockchain.NewUtxoCache(&blockchain.UtxoCacheConfig{ + DB: db, + MaxSize: 100 * 1024 * 1024, // 100 MiB + }), }) if err != nil { diff --git a/blockchain/headercmt.go b/blockchain/headercmt.go index 123901ef95..964073b90a 100644 --- a/blockchain/headercmt.go +++ b/blockchain/headercmt.go @@ -74,7 +74,7 @@ func (b *BlockChain) FetchUtxoViewParentTemplate(block *wire.MsgBlock) (*UtxoVie // Since the block template is building on the parent of the current tip, // undo the transactions and spend information for the tip block to reach // the point of view of the block template. - view := NewUtxoViewpoint() + view := NewUtxoViewpoint(b.utxoCache) view.SetBestHash(&tip.hash) tipBlock, err := b.fetchMainChainBlockByNode(tip) if err != nil { @@ -103,7 +103,7 @@ func (b *BlockChain) FetchUtxoViewParentTemplate(block *wire.MsgBlock) (*UtxoVie // Update the view to unspend all of the spent txos and remove the utxos // created by the tip block. Also, if the block votes against its parent, // reconnect all of the regular transactions. - err = view.disconnectBlock(b.db, tipBlock, parent, stxos, isTreasuryEnabled) + err = view.disconnectBlock(tipBlock, parent, stxos, isTreasuryEnabled) if err != nil { return nil, err } diff --git a/blockchain/sequencelock_test.go b/blockchain/sequencelock_test.go index 8746fe2a0d..befc82b512 100644 --- a/blockchain/sequencelock_test.go +++ b/blockchain/sequencelock_test.go @@ -55,7 +55,7 @@ func TestCalcSequenceLock(t *testing.T) { PkScript: nil, }}, }) - view := NewUtxoViewpoint() + view := NewUtxoViewpoint(nil) view.AddTxOuts(targetTx, int64(numBlocks)-4, 0, noTreasury) view.SetBestHash(&node.hash) diff --git a/blockchain/stakeext.go b/blockchain/stakeext.go index 244ba69447..19b7166a86 100644 --- a/blockchain/stakeext.go +++ b/blockchain/stakeext.go @@ -1,5 +1,5 @@ // Copyright (c) 2013-2014 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -112,7 +112,7 @@ func (b *BlockChain) TicketsWithAddress(address dcrutil.Address, isTreasuryEnabl err := b.db.View(func(dbTx database.Tx) error { for _, hash := range tickets { outpoint := wire.OutPoint{Hash: hash, Index: 0, Tree: wire.TxTreeStake} - utxo, err := dbFetchUtxoEntry(dbTx, outpoint) + utxo, err := b.utxoCache.FetchEntry(dbTx, outpoint) if err != nil { return err } @@ -224,7 +224,7 @@ func (b *BlockChain) TicketPoolValue() (dcrutil.Amount, error) { err := b.db.View(func(dbTx database.Tx) error { for _, hash := range sn.LiveTickets() { outpoint := wire.OutPoint{Hash: hash, Index: 0, Tree: wire.TxTreeStake} - utxo, err := dbFetchUtxoEntry(dbTx, outpoint) + utxo, err := b.utxoCache.FetchEntry(dbTx, outpoint) if err != nil { return err } diff --git a/blockchain/utxocache.go b/blockchain/utxocache.go new file mode 100644 index 0000000000..3cb048ba0c --- /dev/null +++ b/blockchain/utxocache.go @@ -0,0 +1,910 @@ +// Copyright (c) 2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package blockchain + +import ( + "fmt" + "math" + "sync" + "time" + + "github.com/decred/dcrd/chaincfg/chainhash" + "github.com/decred/dcrd/database/v2" + "github.com/decred/dcrd/dcrutil/v4" + "github.com/decred/dcrd/wire" +) + +const ( + // outpointSize is the size of an outpoint on a 64-bit platform. It is + // equivalent to what unsafe.Sizeof(wire.OutPoint{}) returns on a 64-bit + // platform. + outpointSize = 56 + + // pointerSize is the size of a pointer on a 64-bit platform. + pointerSize = 8 + + // p2pkhScriptLen is the length of a standard pay-to-pubkey-hash script. It + // is used in the calculation to approximate the average size of a utxo entry + // when setting the initial capacity of the cache. + p2pkhScriptLen = 25 + + // mapOverhead is the number of bytes per entry to use when approximating the + // memory overhead of the entries map itself (i.e. the memory usage due to + // internals of the map, such as the underlying buckets that are allocated). + // This number was determined by inspecting the true size of the map with + // various numbers of entries and comparing it with the total size of all + // entries in the map. The average overhead came out to 57 bytes per entry. + mapOverhead = 57 + + // evictionPercentage is the targeted percentage of entries to evict from the + // cache when its maximum size has been reached. + // + // A lower percentage will result in a higher overall hit ratio for the cache, + // and thus better performance, but will require eviction again sooner. This + // value was selected to keep the hit ratio of the cache as high as possible + // while still evicting a significant portion of the cache when it reaches its + // maximum allowed size. + evictionPercentage = 0.15 + + // periodicFlushInterval is the amount of time to wait before a periodic flush + // is required. + // + // The cache is flushed periodically during initial block download to avoid + // requiring a flush that would take a significant amount of time on shutdown + // (or, in the case of an unclean shutdown, a significant amount of time to + // initialize the cache when restarted). + periodicFlushInterval = time.Minute * 2 +) + +// UtxoCache is an unspent transaction output cache that sits on top of the +// utxo set database and provides significant runtime performance benefits at +// the cost of some additional memory usage. It drastically reduces the amount +// of reading and writing to disk, especially during initial block download when +// a very large number of blocks are being processed in quick succession. +// +// The UtxoCache is a read-through cache. All utxo reads go through the cache. +// When there is a cache miss, the cache loads the missing data from the +// database, caches it, and returns it to the caller. +// +// The UtxoCache is a write-back cache. Writes to the cache are acknowledged +// by the cache immediately but are only periodically flushed to the database. +// This allows intermediate steps to effectively be skipped. For example, a +// utxo that is created and then spent in between flushes never needs to be +// written to the utxo set in the database. +// +// Due to the write-back nature of the cache, at any given time the database +// may not be in sync with the cache, and therefore all utxo reads and writes +// MUST go through the cache, and never read or write to the database directly. +type UtxoCache struct { + // db is the database that contains the utxo set. It is set when the instance + // is created and is not changed afterward. + db database.DB + + // maxSize is the maximum allowed size of the utxo cache, in bytes. It is set + // when the instance is created and is not changed afterward. + maxSize uint64 + + // cacheLock protects access to the fields in the struct below this point. A + // standard mutex is used rather than a read-write mutex since the cache will + // often write when reads result in a cache miss, so it is generally not worth + // the additional overhead of using a read-write mutex. + cacheLock sync.Mutex + + // entries holds the cached utxo entries. + entries map[wire.OutPoint]*UtxoEntry + + // lastFlushHash is the block hash of the last flush. It is used to compare + // the state of the cache to the utxo set state in the database so that the + // utxo set can properly be initialized in the case that the latest utxo data + // had not been flushed to the database yet. + lastFlushHash chainhash.Hash + + // lastFlushTime is the last time that the cache was flushed to the database. + // It is used to determine when to periodically flush the cache to the + // database during initial block download even if the cache isn't full to + // minimize the amount of progress lost if an unclean shutdown occurs. + lastFlushTime time.Time + + // lastEvictionHeight is the block height of the last eviction. When the + // cache reaches the maximum allowed size, entries are evicted based on the + // height of the block that they are contained in, and last eviction height is + // updated to the current height. + lastEvictionHeight uint32 + + // totalEntrySize is the total size of all utxo entries in the cache, in + // bytes. It is updated whenever an entry is added or removed from the cache. + totalEntrySize uint64 + + // The following fields track the total number of cache hits and misses and + // are used to measure the overall cache hit ratio. + hits uint64 + misses uint64 +} + +// UtxoCacheConfig is a descriptor which specifies the utxo cache instance +// configuration. +type UtxoCacheConfig struct { + // DB defines the database which houses the utxo set. + // + // This field is required. + DB database.DB + + // MaxSize defines the maximum allowed size of the utxo cache, in bytes. + // + // This field is required. + MaxSize uint64 +} + +// NewUtxoCache returns a UtxoCache instance using the provided configuration +// details. +func NewUtxoCache(config *UtxoCacheConfig) *UtxoCache { + // Approximate the maximum number of entries allowed in the cache in order + // to set the initial capacity of the entries map. + avgEntrySize := mapOverhead + outpointSize + pointerSize + baseEntrySize + + p2pkhScriptLen + maxEntries := math.Ceil(float64(config.MaxSize) / float64(avgEntrySize)) + + return &UtxoCache{ + db: config.DB, + maxSize: config.MaxSize, + entries: make(map[wire.OutPoint]*UtxoEntry, uint64(maxEntries)), + lastFlushTime: time.Now(), + } +} + +// totalSize returns the total size of the cache on a 64-bit platform, in bytes. +// Note that this only takes the entries map into account, which represents the +// vast majoirty of the memory that the cache uses, and does not include the +// memory usage of other fields in the utxo cache struct. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) totalSize() uint64 { + numEntries := uint64(len(c.entries)) + return mapOverhead*numEntries + outpointSize*numEntries + + pointerSize*numEntries + c.totalEntrySize +} + +// hitRatio returns the percentage of cache lookups that resulted in a cache +// hit. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) hitRatio() float64 { + totalLookups := c.hits + c.misses + if totalLookups == 0 { + return 100 + } + + return float64(c.hits) / float64(totalLookups) * 100 +} + +// addEntry adds the specified output to the cache. The entry being added MUST +// NOT be mutated by the caller after being passed to this function. +// +// Note that this function does not check if the entry is unspendable and +// therefore the caller should ensure that the entry is spendable before adding +// it to the cache. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) addEntry(outpoint wire.OutPoint, entry *UtxoEntry) error { + // Attempt to get an existing entry from the cache. + cachedEntry := c.entries[outpoint] + + // If an existing entry does not exist, the added entry should be marked as + // modified and fresh. + if cachedEntry == nil { + entry.state |= utxoStateModified | utxoStateFresh + } + + // Add the entry to the cache. In the case that an entry already exists, the + // existing entry is overwritten. All fields are overwritten because it's + // possible (although extremely unlikely) that the existing entry is being + // replaced by a different transaction with the same hash. This is allowed so + // long as the previous transaction is fully spent. + c.entries[outpoint] = entry + + // Update the total entry size of the cache. + if cachedEntry != nil { + c.totalEntrySize -= cachedEntry.size() + } + c.totalEntrySize += entry.size() + + return nil +} + +// AddEntry adds the specified output to the cache. The entry being added MUST +// NOT be mutated by the caller after being passed to this function. +// +// Note that this function does not check if the entry is unspendable and +// therefore the caller should ensure that the entry is spendable before adding +// it to the cache. +// +// This function is safe for concurrent access. +func (c *UtxoCache) AddEntry(outpoint wire.OutPoint, entry *UtxoEntry) error { + c.cacheLock.Lock() + err := c.addEntry(outpoint, entry) + c.cacheLock.Unlock() + return err +} + +// spendEntry marks the specified output as spent. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) spendEntry(outpoint wire.OutPoint) { + // Attempt to get an existing entry from the cache. + cachedEntry := c.entries[outpoint] + + // If the entry is nil or already spent, return immediately. + if cachedEntry == nil || cachedEntry.IsSpent() { + return + } + + // If the entry is fresh, and is now being spent, it can safely be removed. + // This is an optimization to skip writing to the database for outputs that + // are added and spent in between flushes to the database. + if cachedEntry.isFresh() { + // The entry in the map is marked as nil rather than deleting it so that + // subsequent lookups for the outpoint will still result in a cache hit and + // avoid querying the database. + c.entries[outpoint] = nil + c.totalEntrySize -= cachedEntry.size() + return + } + + // Mark the output as spent and modified. + cachedEntry.Spend() +} + +// SpendEntry marks the specified output as spent. +// +// This function is safe for concurrent access. +func (c *UtxoCache) SpendEntry(outpoint wire.OutPoint) { + c.cacheLock.Lock() + c.spendEntry(outpoint) + c.cacheLock.Unlock() +} + +// fetchEntry returns the specified transaction output from the utxo set. If +// the output exists in the cache, it is returned immediately. Otherwise, it +// uses an existing database transaction to fetch the output from the database, +// cache it, and return it to the caller. A cloned copy of the entry is +// returned so it can safely be mutated by the caller without invalidating the +// cache. +// +// When there is no entry for the provided output, nil will be returned for both +// the entry and the error. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) fetchEntry(dbTx database.Tx, outpoint wire.OutPoint) (*UtxoEntry, error) { + // If the cache already has the entry, return it immediately. A cloned copy + // of the entry is returned so it can safely be mutated by the caller without + // invalidating the cache. + if entry, found := c.entries[outpoint]; found { + c.hits++ + return entry.Clone(), nil + } + + // Increment cache misses. + c.misses++ + + // Fetch the entry from the database. + // + // NOTE: Missing entries are not considered an error here and instead + // will result in nil entries in the view. This is intentionally done + // so other code can use the presence of an entry in the view as a way + // to unnecessarily avoid attempting to reload it from the database. + entry, err := dbFetchUtxoEntry(dbTx, outpoint) + if err != nil { + return nil, err + } + + // Update the total entry size of the cache. + if entry != nil { + c.totalEntrySize += entry.size() + } + + // Add the entry to the cache and return it. A cloned copy of the entry is + // returned so it can safely be mutated by the caller without invalidating the + // cache. + c.entries[outpoint] = entry + return entry.Clone(), nil +} + +// FetchEntry returns the specified transaction output from the utxo set. If +// the output exists in the cache, it is returned immediately. Otherwise, it +// uses an existing database transaction to fetch the output from the database, +// cache it, and return it to the caller. A cloned copy of the entry is +// returned so it can safely be mutated by the caller without invalidating the +// cache. +// +// When there is no entry for the provided output, nil will be returned for both +// the entry and the error. +// +// This function is safe for concurrent access. +func (c *UtxoCache) FetchEntry(dbTx database.Tx, outpoint wire.OutPoint) (*UtxoEntry, error) { + c.cacheLock.Lock() + entry, err := c.fetchEntry(dbTx, outpoint) + c.cacheLock.Unlock() + return entry, err +} + +// FetchEntries adds the requested transaction outputs to the provided view. It +// first checks the cache for each output, and if an output does not exist in +// the cache, it will fetch it from the database. +// +// Upon completion of this function, the view will contain an entry for each +// requested outpoint. Spent outputs, or those which otherwise don't exist, +// will result in a nil entry in the view. +// +// This function is safe for concurrent access. +func (c *UtxoCache) FetchEntries(filteredSet viewFilteredSet, view *UtxoViewpoint) error { + c.cacheLock.Lock() + err := c.db.View(func(dbTx database.Tx) error { + for outpoint := range filteredSet { + entry, err := c.fetchEntry(dbTx, outpoint) + if err != nil { + return err + } + + // NOTE: Missing entries are not considered an error here and instead + // will result in nil entries in the view. This is intentionally done + // so other code can use the presence of an entry in the view as a way + // to unnecessarily avoid attempting to reload it from the database. + view.entries[outpoint] = entry + } + + return nil + }) + c.cacheLock.Unlock() + + return err +} + +// Commit updates all entries in the cache based on the state of each entry in +// the provided view. +// +// All entries in the provided view that are marked as modified and spent are +// removed from the view. Additionally, all entries that are added to the cache +// are removed from the provided view. +// +// This function is safe for concurrent access. +func (c *UtxoCache) Commit(view *UtxoViewpoint) error { + c.cacheLock.Lock() + for outpoint, entry := range view.entries { + // If the entry is nil, delete it from the view and continue. + if entry == nil { + delete(view.entries, outpoint) + continue + } + + // If the entry is not modified and not fresh, continue as there is nothing + // to do. + if !entry.isModified() && !entry.isFresh() { + continue + } + + // If the entry is modified and spent, mark it as spent in the cache and + // then delete it from the view. + if entry.isModified() && entry.IsSpent() { + // Mark the entry as spent in the cache. + c.spendEntry(outpoint) + + // Delete the entry from the view. + delete(view.entries, outpoint) + continue + } + + // If we passed all of the conditions above, the entry is modified or fresh, + // but not spent, and should be added to the cache. + err := c.addEntry(outpoint, entry) + if err != nil { + c.cacheLock.Unlock() + return err + } + + // All entries that are added to the cache should be removed from the + // provided view. This is an optimization to allow the cache to take + // ownership of the entry from the view and avoid an additional allocation. + // It is removed from the view to ensure that it is not mutated by the + // caller after being added to the cache. + // + // This does cause the view to refetch the entry if it is requested again + // after being removed. However, this only really occurs during reorgs, + // whereas committing the view to the cache happens with every connected + // block, so this optimizes for the much more common case. + delete(view.entries, outpoint) + } + c.cacheLock.Unlock() + + return nil +} + +// calcEvictionHeight returns the eviction height based on the best height of +// the main chain and the last eviction height. All entries that are contained +// in a block at a height less than the eviction height will be evicted from the +// cache when the cache reaches its maximum allowed size. +// +// Eviction is based on height since the height of the block that an entry is +// contained in is a proxy for how old the utxo is. On average, recent utxos +// are much more likely to be spent in upcoming blocks than older utxos, so the +// strategy used is to evict the oldest utxos in order to maximize the hit ratio +// of the cache. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) calcEvictionHeight(bestHeight uint32) uint32 { + if bestHeight < c.lastEvictionHeight { + return bestHeight + } + + lastEvictionDepth := bestHeight - c.lastEvictionHeight + numBlocksToEvict := math.Ceil(float64(lastEvictionDepth) * evictionPercentage) + return c.lastEvictionHeight + uint32(numBlocksToEvict) +} + +// shouldFlush returns whether or not a flush should be performed. +// +// If the maximum size of the cache has been reached, or if the periodic flush +// interval has been reached, then a flush is required. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) shouldFlush(bestHash *chainhash.Hash) bool { + // No need to flush if the cache has already been flushed through the best + // hash. + if c.lastFlushHash == *bestHash { + return false + } + + // Flush if the max size of the cache has been reached. + if c.totalSize() >= c.maxSize { + return true + } + + // Flush if the periodic flush interval has been reached. + return time.Since(c.lastFlushTime) >= periodicFlushInterval +} + +// flush commits all modified entries to the database and conditionally evicts +// entries. +// +// Entries that are nil or spent are always evicted since they are +// unlikely to be accessed again. Additionally, if the cache has reached its +// maximum size, entries are evicted based on the height of the block that they +// are contained in. +// +// This function MUST be called with the cache lock held. +func (c *UtxoCache) flush(bestHash *chainhash.Hash, bestHeight uint32, logFlush bool) error { + // If the maximum allowed size of the cache has been reached, determine the + // eviction height. + var evictionHeight uint32 + memUsage := c.totalSize() + if memUsage >= c.maxSize { + evictionHeight = c.calcEvictionHeight(bestHeight) + } + + // Log that a flush is starting and indicate the current memory usage, hit + // ratio, and eviction height. + var hitRatio float64 + var evictionLog string + var preFlushNumEntries int + if logFlush { + preFlushNumEntries = len(c.entries) + memUsageMiB := float64(memUsage) / 1024 / 1024 + memUsagePercent := float64(memUsage) / float64(c.maxSize) * 100 + hitRatio = c.hitRatio() + if evictionHeight != 0 { + evictionLog = fmt.Sprintf(", eviction height: %d", evictionHeight) + } + log.Debugf("UTXO cache flush starting (%d entries, %.2f MiB (%.2f%%), "+ + "%.2f%% hit ratio, height: %d%s)", preFlushNumEntries, memUsageMiB, + memUsagePercent, hitRatio, bestHeight, evictionLog) + } + + // Flush the entries in the cache to the database and update the utxo set + // state in the database. + // + // It is important that the utxo set state is always updated in the same + // database transaction as the utxo set itself so that it is always in sync. + err := c.db.Update(func(dbTx database.Tx) error { + for outpoint, entry := range c.entries { + // Write the entry to the database. + err := dbPutUtxoEntry(dbTx, outpoint, entry) + if err != nil { + return err + } + } + + // Update the utxo set state in the database. + return dbPutUtxoSetState(dbTx, &utxoSetState{ + lastFlushHeight: bestHeight, + lastFlushHash: *bestHash, + }) + }) + if err != nil { + return err + } + + // Update the entries in the cache after flushing to the database. This is + // done after the updates to the database have been successfully committed to + // ensure that an unexpected database error would not leave the cache in an + // inconsistent state. + for outpoint, entry := range c.entries { + // Conditionally evict entries from the cache. Entries that are nil or + // spent are always evicted since they are unlikely to be accessed again. + // Additionally, entries that are contained in a block with a height less + // than the eviction height are evicted. + if entry == nil || entry.IsSpent() || + entry.BlockHeight() < int64(evictionHeight) { + + // Remove the entry from the cache. + delete(c.entries, outpoint) + + // Update the total entry size of the cache. + if entry != nil { + c.totalEntrySize -= entry.size() + } + + continue + } + + // If the entry wasn't removed from the cache, clear the modified and + // fresh flags since it has been updated in the database. + entry.state &^= utxoStateModified + entry.state &^= utxoStateFresh + } + + // Update the last flush on the cache instance now that the flush has been + // completed. + c.lastFlushHash = *bestHash + c.lastFlushTime = time.Now() + + // Update the last eviction height on the cache instance if we evicted just + // now. + if evictionHeight != 0 { + c.lastEvictionHeight = evictionHeight + } + + // Log that the flush has been completed and indicate the updated memory usage + // as it will be reduced due to evicting entries above. + if logFlush { + remainingEntries := len(c.entries) + flushedEntries := preFlushNumEntries - remainingEntries + memUsage = c.totalSize() + memUsageMiB := float64(memUsage) / 1024 / 1024 + memUsagePercent := float64(memUsage) / float64(c.maxSize) * 100 + log.Debugf("UTXO cache flush completed (%d entries flushed, %d entries "+ + "remaining, %.2f MiB (%.2f%%))", flushedEntries, remainingEntries, + memUsageMiB, memUsagePercent) + } + + return nil +} + +// MaybeFlush conditionally flushes the cache to the database. +// +// If the maximum size of the cache has been reached, or if the periodic flush +// interval has been reached, then a flush is required. Additionally, a flush +// can be forced by setting the force flush parameter. +// +// This function is safe for concurrent access. +func (c *UtxoCache) MaybeFlush(bestHash *chainhash.Hash, bestHeight uint32, + forceFlush bool, logFlush bool) error { + + c.cacheLock.Lock() + if forceFlush || c.shouldFlush(bestHash) { + err := c.flush(bestHash, bestHeight, logFlush) + c.cacheLock.Unlock() + return err + } + + c.cacheLock.Unlock() + return nil +} + +// InitUtxoCache initializes the utxo cache by ensuring that the utxo set is +// caught up to the tip of the best chain. +// +// Since the cache is only flushed to the database periodically, the utxo set +// may not be caught up to the tip of the best chain. This function catches the +// utxo set up by replaying all blocks from the block after the block that was +// last flushed to the tip block through the cache. +// +// This function should only be called during initialization. +func (b *BlockChain) InitUtxoCache(tip *blockNode) error { + log.Infof("UTXO cache initializing (max size: %d MiB)...", + b.utxoCache.maxSize/1024/1024) + + // Fetch the utxo set state from the database. + var state *utxoSetState + err := b.db.View(func(dbTx database.Tx) error { + var err error + state, err = dbFetchUtxoSetState(dbTx) + return err + }) + if err != nil { + return err + } + + // If the state is nil, update the state to the tip. This should only be the + // case when starting from a fresh database or a database that has not been + // run with the utxo cache yet. + if state == nil { + state = &utxoSetState{ + lastFlushHeight: uint32(tip.height), + lastFlushHash: tip.hash, + } + err := b.db.Update(func(dbTx database.Tx) error { + return dbPutUtxoSetState(dbTx, state) + }) + if err != nil { + return err + } + } + + // Set the last flush hash and the last eviction height from the saved state + // since that is where we are starting from. + b.utxoCache.lastFlushHash = state.lastFlushHash + b.utxoCache.lastEvictionHeight = state.lastFlushHeight + + // If state is already caught up to the tip, return as there is nothing to do. + if state.lastFlushHash == tip.hash { + log.Info("UTXO cache initialization completed") + return nil + } + + // Find the fork point between the current tip and the last flushed block. + lastFlushedNode := b.index.LookupNode(&state.lastFlushHash) + if lastFlushedNode == nil { + // panic if the last flushed block node does not exist. This should never + // happen unless the database is corrupted. + panicf("last flushed block node hash %v (height %v) does not exist", + state.lastFlushHash, state.lastFlushHeight) + } + fork := b.bestChain.FindFork(lastFlushedNode) + + // Disconnect all of the blocks back to the point of the fork. This entails + // loading the blocks and their associated spent txos from the database and + // using that information to unspend all of the spent txos and remove the + // utxos created by the blocks. In addition, if a block votes against its + // parent, the regular transactions are reconnected. + // + // Note that blocks will only need to be disconnected during initialization + // if an unclean shutdown occurred between a block being disconnected and the + // cache being flushed. Since the cache is always flushed immediately after + // disconnecting a block, this will occur very infrequently. In the typical + // catchup case, the fork node will be the last flushed node itself and this + // loop will be skipped. + view := NewUtxoViewpoint(b.utxoCache) + view.SetBestHash(&tip.hash) + var nextBlockToDetach *dcrutil.Block + n := lastFlushedNode + for n != nil && n != fork { + select { + case <-b.interrupt: + return errInterruptRequested + default: + } + + // Grab the block to detach based on the node. Use the fact that the + // blocks are being detached in reverse order, so the parent of the + // current block being detached is the next one being detached. + block := nextBlockToDetach + if block == nil { + var err error + block, err = b.fetchBlockByNode(n) + if err != nil { + return err + } + } + if n.hash != *block.Hash() { + panicf("detach block node hash %v (height %v) does not match "+ + "previous parent block hash %v", &n.hash, n.height, + block.Hash()) + } + + // Grab the parent of the current block and also save a reference to it + // as the next block to detach so it doesn't need to be loaded again on + // the next iteration. + parent, err := b.fetchBlockByNode(n.parent) + if err != nil { + return err + } + nextBlockToDetach = parent + + // Determine if treasury agenda is active. + isTreasuryEnabled, err := b.isTreasuryAgendaActive(n.parent) + if err != nil { + return err + } + + // Load all of the spent txos for the block from the spend journal. + var stxos []spentTxOut + err = b.db.View(func(dbTx database.Tx) error { + stxos, err = dbFetchSpendJournalEntry(dbTx, block, isTreasuryEnabled) + return err + }) + if err != nil { + return err + } + + // Update the view to unspend all of the spent txos and remove the utxos + // created by the block. Also, if the block votes against its parent, + // reconnect all of the regular transactions. + err = view.disconnectBlock(block, parent, stxos, isTreasuryEnabled) + if err != nil { + return err + } + + // Commit all entries in the view to the utxo cache. All entries in the + // view that are marked as modified and spent are removed from the view. + // Additionally, all entries that are added to the cache are removed from + // the view. + err = b.utxoCache.Commit(view) + if err != nil { + return err + } + + // Conditionally flush the utxo cache to the database. Don't force flush + // since many blocks may be disconnected and connected in quick succession + // when initializing. + err = b.utxoCache.MaybeFlush(&n.hash, uint32(n.height), false, true) + if err != nil { + return err + } + + n = n.parent + } + + // Determine the blocks to attach after the fork point. Each block is added + // to the slice from back to front so they are attached in the appropriate + // order when iterating the slice below. + replayNodes := make([]*blockNode, tip.height-fork.height) + for n := tip; n != nil && n != fork; n = n.parent { + replayNodes[n.height-fork.height-1] = n + } + + // Replay all of the blocks through the cache. + var prevBlockAttached *dcrutil.Block + for i, n := range replayNodes { + select { + case <-b.interrupt: + return errInterruptRequested + default: + } + + // Grab the block to attach based on the node. The parent of the block is + // the previous one that was attached except for the first node being + // attached, which needs to be fetched. + block, err := b.fetchBlockByNode(n) + if err != nil { + return err + } + parent := prevBlockAttached + if i == 0 { + parent, err = b.fetchBlockByNode(n.parent) + if err != nil { + return err + } + } + if n.parent.hash != *parent.Hash() { + panicf("attach block node hash %v (height %v) parent hash %v does "+ + "not match previous parent block hash %v", &n.hash, n.height, + &n.parent.hash, parent.Hash()) + } + + // Store the loaded block as parent of the block in the next iteration. + prevBlockAttached = block + + // Determine if treasury agenda is active. + isTreasuryEnabled, err := b.isTreasuryAgendaActive(n.parent) + if err != nil { + return err + } + + // Update the view to mark all utxos referenced by the block as + // spent and add all transactions being created by this block to it. + // In the case the block votes against the parent, also disconnect + // all of the regular transactions in the parent block. + err = view.connectBlock(b.db, block, parent, nil, isTreasuryEnabled) + if err != nil { + return err + } + + // Commit all entries in the view to the utxo cache. All entries in the + // view that are marked as modified and spent are removed from the view. + // Additionally, all entries that are added to the cache are removed from + // the view. + err = b.utxoCache.Commit(view) + if err != nil { + return err + } + + // Conditionally flush the utxo cache to the database. Don't force flush + // since many blocks may be connected in quick succession when initializing. + err = b.utxoCache.MaybeFlush(&n.hash, uint32(n.height), false, true) + if err != nil { + return err + } + } + + log.Info("UTXO cache initialization completed") + return nil +} + +// ShutdownUtxoCache flushes the utxo cache to the database on shutdown. Since +// the cache is flushed periodically during initial block download and flushed +// after every block is connected after initial block download is complete, +// this flush that occurs during shutdown should finish relatively quickly. +// +// Note that if an unclean shutdown occurs, the cache will still be initialized +// properly when restarted as during initialization it will replay blocks to +// catch up to the tip block if it was not fully flushed before shutting down. +// However, it is still preferred to flush when shutting down versus always +// recovering on startup since it is faster. +// +// This function should only be called during shutdown. +func (b *BlockChain) ShutdownUtxoCache() { + b.chainLock.RLock() + defer b.chainLock.RUnlock() + + tip := b.bestChain.Tip() + + // Force a cache flush and log the flush details. + b.utxoCache.MaybeFlush(&tip.hash, uint32(tip.height), true, true) +} + +// FetchUtxoEntry loads and returns the requested unspent transaction output +// from the point of view of the the main chain tip. +// +// NOTE: Requesting an output for which there is no data will NOT return an +// error. Instead both the entry and the error will be nil. This is done to +// allow pruning of spent transaction outputs. In practice this means the +// caller must check if the returned entry is nil before invoking methods on it. +// +// This function is safe for concurrent access however the returned entry (if +// any) is NOT. +func (b *BlockChain) FetchUtxoEntry(outpoint wire.OutPoint) (*UtxoEntry, error) { + b.chainLock.RLock() + defer b.chainLock.RUnlock() + + var entry *UtxoEntry + err := b.db.View(func(dbTx database.Tx) error { + var err error + entry, err = b.utxoCache.FetchEntry(dbTx, outpoint) + return err + }) + if err != nil { + return nil, err + } + + return entry, nil +} + +// UtxoStats represents unspent output statistics on the current utxo set. +type UtxoStats struct { + Utxos int64 + Transactions int64 + Size int64 + Total int64 + SerializedHash chainhash.Hash +} + +// FetchUtxoStats returns statistics on the current utxo set. +// +// NOTE: During initial block download the utxo stats will lag behind the best +// block that is currently synced since the utxo cache is only flushed to the +// database periodically. After initial block download the utxo stats will +// always be in sync with the best block. +func (b *BlockChain) FetchUtxoStats() (*UtxoStats, error) { + var stats *UtxoStats + err := b.db.View(func(dbTx database.Tx) error { + var err error + stats, err = dbFetchUtxoStats(dbTx) + return err + }) + if err != nil { + return nil, err + } + + return stats, nil +} diff --git a/blockchain/utxoviewpoint.go b/blockchain/utxoviewpoint.go index f155014148..4c7c4783b9 100644 --- a/blockchain/utxoviewpoint.go +++ b/blockchain/utxoviewpoint.go @@ -25,6 +25,7 @@ import ( // The unspent outputs are needed by other transactions for things such as // script validation and double spend prevention. type UtxoViewpoint struct { + cache *UtxoCache entries map[wire.OutPoint]*UtxoEntry bestHash chainhash.Hash } @@ -462,7 +463,7 @@ func (view *UtxoViewpoint) disconnectDisapprovedBlock(db database.DB, block *dcr // Load all of the utxos referenced by the inputs for all transactions in // the block that don't already exist in the utxo view from the database. - err = view.fetchRegularInputUtxos(db, block, isTreasuryEnabled) + err = view.fetchRegularInputUtxos(block, isTreasuryEnabled) if err != nil { return err } @@ -505,7 +506,7 @@ func (view *UtxoViewpoint) connectBlock(db database.DB, block, parent *dcrutil.B // Load all of the utxos referenced by the inputs for all transactions in // the block that don't already exist in the utxo view from the database. - err := view.fetchInputUtxos(db, block, isTreasuryEnabled) + err := view.fetchInputUtxos(block, isTreasuryEnabled) if err != nil { return err } @@ -553,7 +554,7 @@ func (view *UtxoViewpoint) connectBlock(db database.DB, block, parent *dcrutil.B // Note that, unlike block connection, the spent transaction output (stxo) // information is required and failure to provide it will result in an assertion // panic. -func (view *UtxoViewpoint) disconnectBlock(db database.DB, block, parent *dcrutil.Block, stxos []spentTxOut, isTreasuryEnabled bool) error { +func (view *UtxoViewpoint) disconnectBlock(block, parent *dcrutil.Block, stxos []spentTxOut, isTreasuryEnabled bool) error { // Sanity check the correct number of stxos are provided. if len(stxos) != countSpentOutputs(block, isTreasuryEnabled) { panicf("provided %v stxos for block %v (height %v) which spends %v "+ @@ -563,7 +564,7 @@ func (view *UtxoViewpoint) disconnectBlock(db database.DB, block, parent *dcruti // Load all of the utxos referenced by the inputs for all transactions in // the block don't already exist in the utxo view from the database. - err := view.fetchInputUtxos(db, block, isTreasuryEnabled) + err := view.fetchInputUtxos(block, isTreasuryEnabled) if err != nil { return err } @@ -586,7 +587,7 @@ func (view *UtxoViewpoint) disconnectBlock(db database.DB, block, parent *dcruti // Load all of the utxos referenced by the inputs for all transactions // in the regular tree of the parent block that don't already exist in // the utxo view from the database. - err := view.fetchRegularInputUtxos(db, parent, isTreasuryEnabled) + err := view.fetchRegularInputUtxos(parent, isTreasuryEnabled) if err != nil { return err } @@ -611,19 +612,6 @@ func (view *UtxoViewpoint) Entries() map[wire.OutPoint]*UtxoEntry { return view.entries } -// commit prunes all entries marked modified that are now spent and marks all -// entries as unmodified. -func (view *UtxoViewpoint) commit() { - for outpoint, entry := range view.entries { - if entry == nil || (entry.isModified() && entry.IsSpent()) { - delete(view.entries, outpoint) - continue - } - - entry.state &^= utxoStateModified - } -} - // viewFilteredSet represents a set of utxos to fetch from the database that are // not already in a view. type viewFilteredSet map[wire.OutPoint]struct{} @@ -643,31 +631,13 @@ func (set viewFilteredSet) add(view *UtxoViewpoint, outpoint *wire.OutPoint) { // Upon completion of this function, the view will contain an entry for each // requested outpoint. Spent outputs, or those which otherwise don't exist, // will result in a nil entry in the view. -func (view *UtxoViewpoint) fetchUtxosMain(db database.DB, filteredSet viewFilteredSet) error { +func (view *UtxoViewpoint) fetchUtxosMain(filteredSet viewFilteredSet) error { // Nothing to do if there are no requested outputs. if len(filteredSet) == 0 { return nil } - // Load the requested set of unspent transaction outputs from the point - // of view of the end of the main chain. - // - // NOTE: Missing entries are not considered an error here and instead - // will result in nil entries in the view. This is intentionally done - // so other code can use the presence of an entry in the view as a way - // to unnecessarily avoid attempting to reload it from the database. - return db.View(func(dbTx database.Tx) error { - for outpoint := range filteredSet { - entry, err := dbFetchUtxoEntry(dbTx, outpoint) - if err != nil { - return err - } - - view.entries[outpoint] = entry - } - - return nil - }) + return view.cache.FetchEntries(filteredSet, view) } // addRegularInputUtxos adds any outputs of transactions in the regular tree of @@ -725,12 +695,12 @@ func (view *UtxoViewpoint) addRegularInputUtxos(block *dcrutil.Block, isTreasury // the view from the database as needed. In particular, referenced entries that // are earlier in the block are added to the view and entries that are already // in the view are not modified. -func (view *UtxoViewpoint) fetchRegularInputUtxos(db database.DB, block *dcrutil.Block, isTreasuryEnabled bool) error { +func (view *UtxoViewpoint) fetchRegularInputUtxos(block *dcrutil.Block, isTreasuryEnabled bool) error { // Add any outputs of transactions in the regular tree of the block that are // referenced by inputs of transactions that are located later in the tree // and fetch any inputs that are not already in the view from the database. filteredSet := view.addRegularInputUtxos(block, isTreasuryEnabled) - return view.fetchUtxosMain(db, filteredSet) + return view.fetchUtxosMain(filteredSet) } // fetchInputUtxos loads the unspent transaction outputs for the inputs @@ -739,7 +709,7 @@ func (view *UtxoViewpoint) fetchRegularInputUtxos(db database.DB, block *dcrutil // regular tree, referenced entries that are earlier in the regular tree of the // block are added to the view. In all cases, entries that are already in the // view are not modified. -func (view *UtxoViewpoint) fetchInputUtxos(db database.DB, block *dcrutil.Block, isTreasuryEnabled bool) error { +func (view *UtxoViewpoint) fetchInputUtxos(block *dcrutil.Block, isTreasuryEnabled bool) error { // Add any outputs of transactions in the regular tree of the block that are // referenced by inputs of transactions that are located later in the tree // and, while doing so, determine which inputs are not already in the view @@ -768,12 +738,13 @@ func (view *UtxoViewpoint) fetchInputUtxos(db database.DB, block *dcrutil.Block, } // Request the input utxos from the database. - return view.fetchUtxosMain(db, filteredSet) + return view.fetchUtxosMain(filteredSet) } // clone returns a deep copy of the view. func (view *UtxoViewpoint) clone() *UtxoViewpoint { clonedView := &UtxoViewpoint{ + cache: view.cache, entries: make(map[wire.OutPoint]*UtxoEntry), bestHash: view.bestHash, } @@ -786,8 +757,9 @@ func (view *UtxoViewpoint) clone() *UtxoViewpoint { } // NewUtxoViewpoint returns a new empty unspent transaction output view. -func NewUtxoViewpoint() *UtxoViewpoint { +func NewUtxoViewpoint(cache *UtxoCache) *UtxoViewpoint { return &UtxoViewpoint{ + cache: cache, entries: make(map[wire.OutPoint]*UtxoEntry), } } @@ -809,7 +781,7 @@ func (b *BlockChain) FetchUtxoView(tx *dcrutil.Tx, includeRegularTxns bool) (*Ut // because the code below requires the parent block and the genesis // block doesn't have one. tip := b.bestChain.Tip() - view := NewUtxoViewpoint() + view := NewUtxoViewpoint(b.utxoCache) view.SetBestHash(&tip.hash) if tip.height == 0 { return view, nil @@ -879,57 +851,6 @@ func (b *BlockChain) FetchUtxoView(tx *dcrutil.Tx, includeRegularTxns bool) (*Ut } } - err = view.fetchUtxosMain(b.db, filteredSet) + err = view.fetchUtxosMain(filteredSet) return view, err } - -// FetchUtxoEntry loads and returns the requested unspent transaction output -// from the point of view of the the main chain tip. -// -// NOTE: Requesting an output for which there is no data will NOT return an -// error. Instead both the entry and the error will be nil. This is done to -// allow pruning of spent transaction outputs. In practice this means the -// caller must check if the returned entry is nil before invoking methods on it. -// -// This function is safe for concurrent access however the returned entry (if -// any) is NOT. -func (b *BlockChain) FetchUtxoEntry(outpoint wire.OutPoint) (*UtxoEntry, error) { - b.chainLock.RLock() - defer b.chainLock.RUnlock() - - var entry *UtxoEntry - err := b.db.View(func(dbTx database.Tx) error { - var err error - entry, err = dbFetchUtxoEntry(dbTx, outpoint) - return err - }) - if err != nil { - return nil, err - } - - return entry, nil -} - -// UtxoStats represents unspent output statistics on the current utxo set. -type UtxoStats struct { - Utxos int64 - Transactions int64 - Size int64 - Total int64 - SerializedHash chainhash.Hash -} - -// FetchUtxoStats returns statistics on the current utxo set. -func (b *BlockChain) FetchUtxoStats() (*UtxoStats, error) { - var stats *UtxoStats - err := b.db.View(func(dbTx database.Tx) error { - var err error - stats, err = dbFetchUtxoStats(dbTx) - return err - }) - if err != nil { - return nil, err - } - - return stats, nil -} diff --git a/blockchain/validate.go b/blockchain/validate.go index ee41de7015..2f152518db 100644 --- a/blockchain/validate.go +++ b/blockchain/validate.go @@ -1854,7 +1854,7 @@ func (b *BlockChain) checkDupTxs(txSet []*dcrutil.Tx, view *UtxoViewpoint, tree filteredSet.add(view, &outpoint) } } - err := view.fetchUtxosMain(b.db, filteredSet) + err := view.fetchUtxosMain(filteredSet) if err != nil { return err } @@ -3557,7 +3557,7 @@ func (b *BlockChain) checkConnectBlock(node *blockNode, block, parent *dcrutil.B // // These utxo entries are needed for verification of things such as // transaction inputs, counting pay-to-script-hashes, and scripts. - err = view.fetchInputUtxos(b.db, block, isTreasuryEnabled) + err = view.fetchInputUtxos(block, isTreasuryEnabled) if err != nil { return err } @@ -3791,7 +3791,7 @@ func (b *BlockChain) CheckConnectBlockTemplate(block *dcrutil.Block) error { return ruleError(ErrMissingParent, err.Error()) } - view := NewUtxoViewpoint() + view := NewUtxoViewpoint(b.utxoCache) view.SetBestHash(&tip.hash) return b.checkConnectBlock(newNode, block, parent, view, nil, nil) @@ -3801,7 +3801,7 @@ func (b *BlockChain) CheckConnectBlockTemplate(block *dcrutil.Block) error { // current tip due to the previous checks, so undo the transactions and // spend information for the tip block to reach the point of view of the // block template. - view := NewUtxoViewpoint() + view := NewUtxoViewpoint(b.utxoCache) view.SetBestHash(&tip.hash) tipBlock, err := b.fetchMainChainBlockByNode(tip) if err != nil { @@ -3829,7 +3829,7 @@ func (b *BlockChain) CheckConnectBlockTemplate(block *dcrutil.Block) error { // Update the view to unspend all of the spent txos and remove the utxos // created by the tip block. Also, if the block votes against its parent, // reconnect all of the regular transactions. - err = view.disconnectBlock(b.db, tipBlock, parent, stxos, isTreasuryEnabled) + err = view.disconnectBlock(tipBlock, parent, stxos, isTreasuryEnabled) if err != nil { return err } diff --git a/blockchain/validate_test.go b/blockchain/validate_test.go index 6154b25209..e008542948 100644 --- a/blockchain/validate_test.go +++ b/blockchain/validate_test.go @@ -1,5 +1,5 @@ // Copyright (c) 2013-2016 The btcsuite developers -// Copyright (c) 2015-2020 The Decred developers +// Copyright (c) 2015-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -290,6 +290,10 @@ func TestCheckBlockHeaderContext(t *testing.T) { DB: db, ChainParams: params, TimeSource: NewMedianTime(), + UtxoCache: NewUtxoCache(&UtxoCacheConfig{ + DB: db, + MaxSize: 100 * 1024 * 1024, // 100 MiB + }), }) if err != nil { t.Fatalf("Failed to create chain instance: %v\n", err) diff --git a/internal/mempool/mempool_test.go b/internal/mempool/mempool_test.go index aed020e85d..124a298ee9 100644 --- a/internal/mempool/mempool_test.go +++ b/internal/mempool/mempool_test.go @@ -87,7 +87,7 @@ func (s *fakeChain) FetchUtxoView(tx *dcrutil.Tx, treeValid bool) (*blockchain.U // Add entries for the outputs of the tx to the new view. msgTx := tx.MsgTx() - viewpoint := blockchain.NewUtxoViewpoint() + viewpoint := blockchain.NewUtxoViewpoint(nil) outpoint := wire.OutPoint{Hash: *tx.Hash(), Tree: tx.Tree()} for txOutIdx := range msgTx.TxOut { outpoint.Index = uint32(txOutIdx) @@ -754,7 +754,7 @@ func newPoolHarness(chainParams *chaincfg.Params) (*poolHarness, []spendableOutp // Create a new fake chain and harness bound to it. subsidyCache := standalone.NewSubsidyCache(chainParams) chain := &fakeChain{ - utxos: blockchain.NewUtxoViewpoint(), + utxos: blockchain.NewUtxoViewpoint(nil), utxoTimes: make(map[wire.OutPoint]int64), blocks: make(map[chainhash.Hash]*dcrutil.Block), scriptFlags: BaseStandardVerifyFlags, diff --git a/internal/mining/mining_harness_test.go b/internal/mining/mining_harness_test.go index f95cc1f90a..c92697cad8 100644 --- a/internal/mining/mining_harness_test.go +++ b/internal/mining/mining_harness_test.go @@ -144,7 +144,7 @@ func (c *fakeChain) FetchUtxoView(tx *dcrutil.Tx, treeValid bool) (*blockchain.U // Add entries for the outputs of the tx to the new view. msgTx := tx.MsgTx() - viewpoint := blockchain.NewUtxoViewpoint() + viewpoint := blockchain.NewUtxoViewpoint(nil) prevOut := wire.OutPoint{Hash: *tx.Hash(), Tree: tx.Tree()} for txOutIdx := range msgTx.TxOut { prevOut.Index = uint32(txOutIdx) @@ -195,7 +195,7 @@ func (c *fakeChain) MaxTreasuryExpenditure(preTVIBlock *chainhash.Hash) (int64, // NewUtxoViewpoint returns a new empty unspent transaction output view. func (c *fakeChain) NewUtxoViewpoint() *blockchain.UtxoViewpoint { - return blockchain.NewUtxoViewpoint() + return blockchain.NewUtxoViewpoint(nil) } // TipGeneration returns a mocked entire generation of blocks stemming from the @@ -1296,8 +1296,8 @@ func newMiningHarness(chainParams *chaincfg.Params) (*miningHarness, []spendable blocks: make(map[chainhash.Hash]*dcrutil.Block), isHeaderCommitmentsAgendaActive: true, isTreasuryAgendaActive: true, - parentUtxos: blockchain.NewUtxoViewpoint(), - utxos: blockchain.NewUtxoViewpoint(), + parentUtxos: blockchain.NewUtxoViewpoint(nil), + utxos: blockchain.NewUtxoViewpoint(nil), } // Set the proof of work limit and next required difficulty very high by diff --git a/internal/rpcserver/interface.go b/internal/rpcserver/interface.go index 16d8e5ecc2..5b8cc0800d 100644 --- a/internal/rpcserver/interface.go +++ b/internal/rpcserver/interface.go @@ -281,6 +281,11 @@ type Chain interface { FetchUtxoEntry(outpoint wire.OutPoint) (UtxoEntry, error) // FetchUtxoStats returns statistics on the current utxo set. + // + // NOTE: During initial block download the utxo stats will lag behind the best + // block that is currently synced since the utxo cache is only flushed to the + // database periodically. After initial block download the utxo stats will + // always be in sync with the best block. FetchUtxoStats() (*blockchain.UtxoStats, error) // GetStakeVersions returns a cooked array of StakeVersions. We do this in diff --git a/server.go b/server.go index 14df8de989..bc6e60c4b3 100644 --- a/server.go +++ b/server.go @@ -3017,6 +3017,8 @@ func (s *server) Run(ctx context.Context) { s.feeEstimator.Close() + s.chain.ShutdownUtxoCache() + s.wg.Wait() srvrLog.Trace("Server stopped") } @@ -3367,6 +3369,10 @@ func newServer(ctx context.Context, listenAddrs []string, db database.DB, chainP } // Create a new block chain instance with the appropriate configuration. + utxoCache := blockchain.NewUtxoCache(&blockchain.UtxoCacheConfig{ + DB: s.db, + MaxSize: uint64(cfg.UtxoCacheMaxSize) * 1024 * 1024, + }) s.chain, err = blockchain.New(ctx, &blockchain.Config{ DB: s.db, @@ -3377,6 +3383,7 @@ func newServer(ctx context.Context, listenAddrs []string, db database.DB, chainP SigCache: s.sigCache, SubsidyCache: s.subsidyCache, IndexManager: indexManager, + UtxoCache: utxoCache, }) if err != nil { return nil, err @@ -3526,7 +3533,7 @@ func newServer(ctx context.Context, listenAddrs []string, db database.DB, chainP IsTreasuryAgendaActive: s.chain.IsTreasuryAgendaActive, MaxTreasuryExpenditure: s.chain.MaxTreasuryExpenditure, NewUtxoViewpoint: func() *blockchain.UtxoViewpoint { - return blockchain.NewUtxoViewpoint() + return blockchain.NewUtxoViewpoint(utxoCache) }, TipGeneration: s.chain.TipGeneration, ValidateTransactionScripts: func(tx *dcrutil.Tx, From 3afb4458a6f5646f27d1a50362c64428d8f30df0 Mon Sep 17 00:00:00 2001 From: Ryan Staudt Date: Thu, 4 Feb 2021 06:55:56 -0600 Subject: [PATCH 09/10] blockchain: Add UtxoCache test coverage. This adds full test coverage to the UtxoCache type and its methods. Additionally, since this uses the testing Cleanup function that was introduced in Go 1.14, this bumps the required go version for the blockchain package from 1.13 to 1.14. --- blockchain/go.mod | 2 +- blockchain/utxocache_test.go | 1061 ++++++++++++++++++++++++++++++++++ 2 files changed, 1062 insertions(+), 1 deletion(-) create mode 100644 blockchain/utxocache_test.go diff --git a/blockchain/go.mod b/blockchain/go.mod index cc3a211a48..fe0aca00e7 100644 --- a/blockchain/go.mod +++ b/blockchain/go.mod @@ -1,6 +1,6 @@ module github.com/decred/dcrd/blockchain/v4 -go 1.13 +go 1.14 require ( github.com/decred/dcrd/blockchain/stake/v4 v4.0.0-20210129192908-660d0518b4cf diff --git a/blockchain/utxocache_test.go b/blockchain/utxocache_test.go new file mode 100644 index 0000000000..4dfddc369f --- /dev/null +++ b/blockchain/utxocache_test.go @@ -0,0 +1,1061 @@ +// Copyright (c) 2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package blockchain + +import ( + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/decred/dcrd/blockchain/stake/v4" + "github.com/decred/dcrd/chaincfg/chainhash" + "github.com/decred/dcrd/database/v2" + "github.com/decred/dcrd/wire" +) + +// Define constants for indicating flags throughout the tests. +const ( + noCoinbase = false + withCoinbase = true + noExpiry = false + withExpiry = true +) + +// outpoint299 returns a test outpoint from block height 299 that can be used +// throughout the tests. +func outpoint299() wire.OutPoint { + return wire.OutPoint{ + Hash: *mustParseHash("e299d2cc5deb5b39d230ad2a6046ff9cc164064f431a2893eb6" + + "28b467d018452"), + Index: 0, + Tree: wire.TxTreeRegular, + } +} + +// entry299 returns a utxo entry from block height 299 that can be used +// throughout the tests. +func entry299() *UtxoEntry { + return &UtxoEntry{ + amount: 58795424, + pkScript: hexToBytes("76a914454017705ab80470d089c7f644e39cc9e0fd308e" + + "88ac"), + blockHeight: 299, + blockIndex: 1, + scriptVersion: 0, + packedFlags: encodeUtxoFlags( + noCoinbase, + noExpiry, + stake.TxTypeRegular, + ), + } +} + +// outpoint1100 returns a test outpoint from block height 1100 that can be used +// throughout the tests. +func outpoint1100() wire.OutPoint { + return wire.OutPoint{ + Hash: *mustParseHash("ce1d0f74440c391d15516015224755a8661e56e796ac25490f3" + + "0ad1081c5d638"), + Index: 1, + Tree: wire.TxTreeRegular, + } +} + +// entry1100 returns a utxo entry from block height 1100 that can be used +// throughout the tests. +func entry1100() *UtxoEntry { + return &UtxoEntry{ + amount: 52454022, + pkScript: hexToBytes("76a9146b65f16ebca9b848158701d5a2eb5124547a2144" + + "88ac"), + blockHeight: 1100, + blockIndex: 1, + scriptVersion: 0, + packedFlags: encodeUtxoFlags( + noCoinbase, + noExpiry, + stake.TxTypeRegular, + ), + } +} + +// outpoint1200 returns a test outpoint from block height 1200 that can be used +// throughout the tests. +func outpoint1200() wire.OutPoint { + return wire.OutPoint{ + Hash: *mustParseHash("72914cae2d4bc75f7777373b7c085c4b92d59f3e059fc7fd39d" + + "ef71c9fe188b5"), + Index: 2, + Tree: wire.TxTreeRegular, + } +} + +// entry1200 returns a utxo entry from block height 1200 that can be used +// throughout the tests. +func entry1200() *UtxoEntry { + return &UtxoEntry{ + amount: 1871749598, + pkScript: hexToBytes("76a9142ec5027abadede723c47b6acdbace3be10b7e937" + + "88ac"), + blockHeight: 1200, + blockIndex: 0, + scriptVersion: 0, + packedFlags: encodeUtxoFlags( + withCoinbase, + noExpiry, + stake.TxTypeRegular, + ), + } +} + +// outpoint85314 returns a test outpoint from block height 85314 that can be +// used throughout the tests. +func outpoint85314() wire.OutPoint { + return wire.OutPoint{ + Hash: *mustParseHash("d3bce77da2747baa85fb7ca4f6f8e123f31cd15ac691b2f8254" + + "3780158587d3a"), + Index: 0, + Tree: wire.TxTreeStake, + } +} + +// entry85314 returns a utxo entry from block height 85314 that can be used +// throughout the tests. +func entry85314() *UtxoEntry { + return &UtxoEntry{ + amount: 4294959555, + pkScript: hexToBytes("ba76a914a13afb81d54c9f8bb0c5e082d56fd563ab9b359" + + "688ac"), + blockHeight: 85314, + blockIndex: 6, + scriptVersion: 0, + packedFlags: encodeUtxoFlags( + noCoinbase, + withExpiry, + stake.TxTypeSStx, + ), + ticketMinOuts: &ticketMinimalOutputs{ + data: hexToBytes("03808efefade57001aba76a914a13afb81d54c9f8bb0c5e08" + + "2d56fd563ab9b359688ac0000206a1e9ac39159847e259c9162405b5f6c8135d" + + "2c7eaf1a375040001000000005800001abd76a91400000000000000000000000" + + "0000000000000000088ac"), + }, + } +} + +// createTestUtxoDatabase creates a test database with the utxo set bucket. +func createTestUtxoDatabase(t *testing.T) database.DB { + t.Helper() + + // Create a test database. + dbPath := filepath.Join(os.TempDir(), t.Name()) + _ = os.RemoveAll(dbPath) + db, err := database.Create("ffldb", dbPath, wire.MainNet) + if err != nil { + t.Fatalf("error creating test database: %v", err) + } + t.Cleanup(func() { + os.RemoveAll(dbPath) + }) + t.Cleanup(func() { + db.Close() + }) + + // Create the utxo set bucket. + err = db.Update(func(dbTx database.Tx) error { + _, err := dbTx.Metadata().CreateBucketIfNotExists(utxoSetBucketName) + return err + }) + if err != nil { + t.Fatalf("error creating utxo bucket: %v", err) + } + + return db +} + +// createTestUtxoCache creates a test utxo cache with the specified entries. +func createTestUtxoCache(t *testing.T, entries map[wire.OutPoint]*UtxoEntry) *UtxoCache { + t.Helper() + + utxoCache := NewUtxoCache(&UtxoCacheConfig{}) + for outpoint, entry := range entries { + // Add the entry to the cache. The entry is cloned before being added so + // that any modifications that the cache makes to the entry are not + // reflected in the provided test entry. + err := utxoCache.AddEntry(outpoint, entry.Clone()) + if err != nil { + t.Fatalf("unexpected error when adding entry: %v", err) + } + + // Set the state of the cached entries based on the provided entries. This + // is allowed for tests to easily simulate entries in the cache that are not + // fresh without having to fetch them from the database. + cachedEntry := utxoCache.entries[outpoint] + if cachedEntry != nil { + cachedEntry.state = entry.state + } + } + return utxoCache +} + +// TestTotalSize validates that the correct number of bytes is returned for the +// size of the utxo cache. +func TestTotalSize(t *testing.T) { + t.Parallel() + + // Create test entries to be used throughout the tests. + outpointRegular := outpoint1200() + entryRegular := entry1200() + outpointTicket := outpoint85314() + entryTicket := entry85314() + + tests := []struct { + name string + entries map[wire.OutPoint]*UtxoEntry + want uint64 + }{{ + name: "without any entries", + entries: map[wire.OutPoint]*UtxoEntry{}, + want: 0, + }, { + name: "with entries", + entries: map[wire.OutPoint]*UtxoEntry{ + outpointRegular: entryRegular, + outpointTicket: entryTicket, + }, + // mapOverhead*numEntries + outpointSize*numEntries + + // pointerSize*numEntries + (first entry: base entry size + len(pkScript)) + + // (second entry: base entry size + len(pkScript) + len(ticketMinOuts.data)) + want: mapOverhead*2 + outpointSize*2 + pointerSize*2 + + (baseEntrySize + 25) + (baseEntrySize + 26 + 99), + }} + + for _, test := range tests { + // Create a utxo cache with the entries specified by the test. + utxoCache := createTestUtxoCache(t, test.entries) + + // Validate that total size returns the expected value. + got := utxoCache.totalSize() + if got != test.want { + t.Errorf("%q: unexpected result -- got %d, want %d", test.name, got, + test.want) + } + } +} + +// TestHitRatio validates that the correct hit ratio is returned based on the +// number of cache hits and misses. +func TestHitRatio(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + hits uint64 + misses uint64 + want float64 + }{{ + name: "no hits or misses", + want: 100, + }, { + name: "all hits, no misses", + hits: 50, + want: 100, + }, { + name: "98.5% hit ratio", + hits: 197, + misses: 3, + want: 98.5, + }} + + for _, test := range tests { + // Create a utxo cache with hits and misses as specified by the test. + utxoCache := NewUtxoCache(&UtxoCacheConfig{}) + utxoCache.hits = test.hits + utxoCache.misses = test.misses + + // Validate that hit ratio returns the expected value. + got := utxoCache.hitRatio() + if got != test.want { + t.Errorf("%q: unexpected result -- got %f, want %f", test.name, got, + test.want) + } + } +} + +// TestAddEntry validates that entries are added to the cache properly under a +// variety of conditions. +func TestAddEntry(t *testing.T) { + t.Parallel() + + // Create test entries to be used throughout the tests. + outpoint := outpoint299() + entry := entry299() + entryModified := entry.Clone() + entryModified.amount++ + entryModified.state |= utxoStateModified + entryFresh := entry.Clone() + entryFresh.state |= utxoStateModified | utxoStateFresh + + tests := []struct { + name string + existingEntries map[wire.OutPoint]*UtxoEntry + outpoint wire.OutPoint + entry *UtxoEntry + wantEntry *UtxoEntry + }{{ + name: "add an entry that does not already exist in the cache", + outpoint: outpoint, + entry: entry, + wantEntry: entryFresh, + }, { + name: "add an entry that overwrites an existing entry", + existingEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint: entry, + }, + outpoint: outpoint, + entry: entryModified, + wantEntry: entryModified, + }} + + for _, test := range tests { + // Create a utxo cache with the existing entries specified by the test. + utxoCache := createTestUtxoCache(t, test.existingEntries) + wantTotalEntrySize := utxoCache.totalEntrySize + + // Attempt to get an existing entry from the cache. If it exists, subtract + // its size from the expected total entry size since it will be overwritten. + existingEntry := utxoCache.entries[test.outpoint] + if existingEntry != nil { + wantTotalEntrySize -= test.entry.size() + } + + // Add the entry specified by the test. + err := utxoCache.AddEntry(test.outpoint, test.entry) + if err != nil { + t.Fatalf("%q: unexpected error when adding entry: %v", test.name, err) + } + wantTotalEntrySize += test.entry.size() + + // Attempt to get the added entry from the cache. + cachedEntry := utxoCache.entries[test.outpoint] + + // Validate that the added entry exists in the cache. + if cachedEntry == nil { + t.Fatalf("%q: expected entry for outpoint %v to exist in the cache", + test.name, test.outpoint) + } + + // Validate that the entry is marked as modified. + if !cachedEntry.isModified() { + t.Fatalf("%q: unexpected modified flag -- got false, want true", + test.name) + } + + // Validate that the cached entry matches the expected entry. + if !reflect.DeepEqual(cachedEntry, test.wantEntry) { + t.Fatalf("%q: mismatched cached entry:\nwant: %+v\n got: %+v\n", + test.name, test.wantEntry, cachedEntry) + } + + // Validate that the total entry size was updated as expected. + if utxoCache.totalEntrySize != wantTotalEntrySize { + t.Fatalf("%q: unexpected total entry size -- got %v, want %v", test.name, + utxoCache.totalEntrySize, wantTotalEntrySize) + } + } +} + +// TestSpendEntry validates that entries in the cache are properly updated when +// being spent under a variety of conditions. +func TestSpendEntry(t *testing.T) { + t.Parallel() + + // Create test entries to be used throughout the tests. + outpoint := outpoint299() + entry := entry299() + entryFresh := entry.Clone() + entryFresh.state |= utxoStateModified | utxoStateFresh + entrySpent := entry.Clone() + entrySpent.Spend() + + tests := []struct { + name string + existingEntries map[wire.OutPoint]*UtxoEntry + outpoint wire.OutPoint + entry *UtxoEntry + }{{ + name: "spend an entry that does not exist in the cache", + outpoint: outpoint, + entry: entry, + }, { + name: "spend an entry that exists in the cache but is already spent", + existingEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint: entrySpent, + }, + outpoint: outpoint, + entry: entrySpent, + }, { + name: "spend an entry that exists in the cache and is fresh", + existingEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint: entryFresh, + }, + outpoint: outpoint, + entry: entryFresh, + }, { + name: "spend an entry that exists in the cache and is not fresh", + existingEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint: entry, + }, + outpoint: outpoint, + entry: entry, + }} + + for _, test := range tests { + // Create a utxo cache with the existing entries specified by the test. + utxoCache := createTestUtxoCache(t, test.existingEntries) + wantTotalEntrySize := utxoCache.totalEntrySize + + // Attempt to get an existing entry from the cache. + entry := utxoCache.entries[test.outpoint] + var entryAlreadySpent bool + if entry != nil { + entryAlreadySpent = entry.IsSpent() + } + + // Spend the entry specified by the test. + utxoCache.SpendEntry(test.outpoint) + + // If the existing entry was nil or spent, continue as there is nothing + // else to validate. + if entry == nil || entryAlreadySpent { + continue + } + + // If the entry is fresh, validate that it was removed from the cache when + // spent. + if entry.isFresh() { + wantTotalEntrySize -= test.entry.size() + if utxoCache.entries[test.outpoint] != nil { + t.Fatalf("%q: entry for outpoint %v was not removed from the cache", + test.name, test.outpoint) + } + } + + // Validate that the total entry size was updated as expected. + if utxoCache.totalEntrySize != wantTotalEntrySize { + t.Fatalf("%q: unexpected total entry size -- got %v, want %v", test.name, + utxoCache.totalEntrySize, wantTotalEntrySize) + } + + // If entry is not fresh, validate that it still exists in the cache and is + // now marked as spent. + if !entry.isFresh() { + cachedEntry := utxoCache.entries[test.outpoint] + if cachedEntry == nil || !cachedEntry.IsSpent() { + t.Fatalf("%q: expected entry for outpoint %v to exist in the cache "+ + "and be marked spent", test.name, test.outpoint) + } + } + } +} + +// TestFetchEntry validates that fetch entry returns the correct entry under a +// variety of conditions. +func TestFetchEntry(t *testing.T) { + t.Parallel() + + // Create a test database. + db := createTestUtxoDatabase(t) + + // Create test entries to be used throughout the tests. + outpoint := outpoint299() + entry := entry299() + entryModified := entry.Clone() + entryModified.state |= utxoStateModified + + tests := []struct { + name string + cachedEntries map[wire.OutPoint]*UtxoEntry + dbEntries map[wire.OutPoint]*UtxoEntry + outpoint wire.OutPoint + cacheHit bool + wantEntry *UtxoEntry + }{{ + name: "entry is not in the cache or the database", + outpoint: outpoint, + }, { + name: "entry is in the cache", + cachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint: entry, + }, + outpoint: outpoint, + cacheHit: true, + wantEntry: entry, + }, { + name: "entry is not in the cache but is in the database", + dbEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint: entryModified, + }, + outpoint: outpoint, + wantEntry: entry, + }} + + for _, test := range tests { + // Create a utxo cache with the cached entries specified by the test. + utxoCache := createTestUtxoCache(t, test.cachedEntries) + wantTotalEntrySize := utxoCache.totalEntrySize + + // Add entries specified by the test to the test database. + err := db.Update(func(dbTx database.Tx) error { + for outpoint, entry := range test.dbEntries { + err := dbPutUtxoEntry(dbTx, outpoint, entry) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatalf("%q: unexpected error adding entries to test db: %v", test.name, + err) + } + + // Attempt to fetch the entry for the outpoint specified by the test. + var entry *UtxoEntry + err = db.View(func(dbTx database.Tx) error { + var err error + entry, err = utxoCache.FetchEntry(dbTx, test.outpoint) + return err + }) + if err != nil { + t.Fatalf("%q: unexpected error fetching entry: %v", test.name, err) + } + + // Ensure that the fetched entry matches the expected entry. + if !reflect.DeepEqual(entry, test.wantEntry) { + t.Fatalf("%q: mismatched entry:\nwant: %+v\n got: %+v\n", test.name, + test.wantEntry, entry) + } + + // Ensure that the entry is now cached. + cachedEntry := utxoCache.entries[test.outpoint] + if !reflect.DeepEqual(cachedEntry, test.wantEntry) { + t.Fatalf("%q: mismatched cached entry:\nwant: %+v\n got: %+v\n", + test.name, test.wantEntry, cachedEntry) + } + + // Validate the cache hits and misses counts. + if test.cacheHit && utxoCache.hits != 1 { + t.Fatalf("%q: unexpected cache hits -- got %v, want 1", test.name, + utxoCache.hits) + } + if !test.cacheHit && utxoCache.misses != 1 { + t.Fatalf("%q: unexpected cache misses -- got %v, want 1", test.name, + utxoCache.misses) + } + + // Validate that the total entry size was updated as expected. + if !test.cacheHit && cachedEntry != nil { + wantTotalEntrySize += cachedEntry.size() + } + if utxoCache.totalEntrySize != wantTotalEntrySize { + t.Fatalf("%q: unexpected total entry size -- got %v, want %v", test.name, + utxoCache.totalEntrySize, wantTotalEntrySize) + } + } +} + +// TestFetchEntries validates that the provided view is populated with the +// requested entries as expecetd. +func TestFetchEntries(t *testing.T) { + t.Parallel() + + // Create a test database. + db := createTestUtxoDatabase(t) + + // Create test entries to be used throughout the tests. + outpoint299 := outpoint299() + outpoint1100 := outpoint1100() + entry1100 := entry1100() + outpoint1200 := outpoint1200() + entry1200 := entry1200() + entry1200Modified := entry1200.Clone() + entry1200Modified.state |= utxoStateModified + + tests := []struct { + name string + cachedEntries map[wire.OutPoint]*UtxoEntry + dbEntries map[wire.OutPoint]*UtxoEntry + filteredSet viewFilteredSet + wantEntries map[wire.OutPoint]*UtxoEntry + }{{ + name: "entries are fetched from the cache and the database", + cachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1100: entry1100, + }, + dbEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1200: entry1200Modified, + }, + filteredSet: viewFilteredSet{ + outpoint299: struct{}{}, + outpoint1100: struct{}{}, + outpoint1200: struct{}{}, + }, + wantEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint299: nil, + outpoint1100: entry1100, + outpoint1200: entry1200, + }, + }} + + for _, test := range tests { + // Create a utxo cache with the cached entries specified by the test. + utxoCache := createTestUtxoCache(t, test.cachedEntries) + utxoCache.db = db + + // Add entries specified by the test to the test database. + err := db.Update(func(dbTx database.Tx) error { + for outpoint, entry := range test.dbEntries { + err := dbPutUtxoEntry(dbTx, outpoint, entry) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatalf("%q: unexpected error adding entries to test db: %v", test.name, + err) + } + + // Fetch the entries requested by the test and add them to a view. + view := NewUtxoViewpoint(utxoCache) + err = utxoCache.FetchEntries(test.filteredSet, view) + if err != nil { + t.Fatalf("%q: unexpected error fetching entries for view: %v", test.name, + err) + } + + // Ensure that the fetched entries match the expected entries. + if !reflect.DeepEqual(view.entries, test.wantEntries) { + t.Fatalf("%q: mismatched entries:\nwant: %+v\n got: %+v\n", test.name, + test.wantEntries, view.entries) + } + } +} + +// TestCommit validates that all entries in both the cache and the provided view +// are updated appropriately when committing the provided view to the cache. +func TestCommit(t *testing.T) { + t.Parallel() + + // Create test entries to be used throughout the tests. + outpoint299 := outpoint299() + outpoint1100 := outpoint1100() + entry1100Unmodified := entry1100() + outpoint1200 := outpoint1200() + entry1200 := entry1200() + entry1200Spent := entry1200.Clone() + entry1200Spent.Spend() + outpoint85314 := outpoint85314() + entry85314Modified := entry85314() + entry85314Modified.state |= utxoStateModified + + tests := []struct { + name string + viewEntries map[wire.OutPoint]*UtxoEntry + cachedEntries map[wire.OutPoint]*UtxoEntry + wantViewEntries map[wire.OutPoint]*UtxoEntry + wantCachedEntries map[wire.OutPoint]*UtxoEntry + }{{ + name: "view contains nil, unmodified, spent, and modified entries", + viewEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint299: nil, + outpoint1100: entry1100Unmodified, + outpoint1200: entry1200Spent, + outpoint85314: entry85314Modified, + }, + cachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1200: entry1200, + }, + // outpoint299 is removed from the view since the entry is nil. + // entry1100Unmodified remains in the view since it is unmodified. + // entry1200Spent is removed from the view since the entry is spent. + // entry85314Modified is removed from the view since it is modified and + // added to the cache. + wantViewEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1100: entry1100Unmodified, + }, + // entry1200Spent remains in the cache but is now spent. + // entry85314Modified is added to the cache. + wantCachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1200: entry1200Spent, + outpoint85314: entry85314Modified, + }, + }} + + for _, test := range tests { + // Create a utxo cache with the cached entries specified by the test. + utxoCache := createTestUtxoCache(t, test.cachedEntries) + + // Create a utxo cache with the view entries specified by the test. + view := &UtxoViewpoint{ + cache: utxoCache, + entries: test.viewEntries, + } + + // Commit the view to the cache. + err := utxoCache.Commit(view) + if err != nil { + t.Fatalf("%q: unexpected error committing view to the cache: %v", + test.name, err) + } + + // Validate the cached entries after committing. + if !reflect.DeepEqual(utxoCache.entries, test.wantCachedEntries) { + t.Fatalf("%q: mismatched cached entries:\nwant: %+v\n got: %+v\n", + test.name, test.wantCachedEntries, utxoCache.entries) + } + + // Validate the view entries after committing. + if !reflect.DeepEqual(view.entries, test.wantViewEntries) { + t.Fatalf("%q: mismatched view entries:\nwant: %+v\n got: %+v\n", + test.name, test.wantViewEntries, view.entries) + } + } +} + +// TestCalcEvictionHeight validates that the correct eviction height is returned +// based on the provided best height and the last eviction height. +func TestCalcEvictionHeight(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + lastEvictionHeight uint32 + bestHeight uint32 + want uint32 + }{{ + name: "no last eviction", + bestHeight: 100, + want: 15, + }, { + name: "best height less than last eviction height", + lastEvictionHeight: 101, + bestHeight: 100, + want: 100, + }, { + name: "best height greater than last eviction height", + lastEvictionHeight: 99, + bestHeight: 200, + want: 115, + }} + + for _, test := range tests { + // Create a utxo cache with the last eviction height as specified by the + // test. + utxoCache := NewUtxoCache(&UtxoCacheConfig{}) + utxoCache.lastEvictionHeight = test.lastEvictionHeight + + // Validate that calc eviction height returns the expected value. + got := utxoCache.calcEvictionHeight(test.bestHeight) + if got != test.want { + t.Errorf("%q: unexpected result -- got %d, want %d", test.name, got, + test.want) + } + } +} + +// TestShouldFlush validates that it is correctly determined whether or not a +// flush should be performed given various conditions. +func TestShouldFlush(t *testing.T) { + t.Parallel() + + // Create test hashes to be used throughout the tests. + block1000Hash := mustParseHash("0000000000004740ad140c86753f9295e09f9cc81b1" + + "bb75d7f5552aeeedb7012") + block2000Hash := mustParseHash("0000000000000c8a886e3f7c32b1bb08422066dcfd0" + + "08de596471f11a5aff475") + + tests := []struct { + name string + totalEntrySize uint64 + maxSize uint64 + lastFlushTime time.Time + lastFlushHash *chainhash.Hash + bestHash *chainhash.Hash + want bool + }{{ + name: "already flushed through the best hash", + totalEntrySize: 100, + maxSize: 1000, + lastFlushTime: time.Now(), + lastFlushHash: block1000Hash, + bestHash: block1000Hash, + want: false, + }, { + name: "less than max size and periodic duration not reached", + totalEntrySize: 100, + maxSize: 1000, + lastFlushTime: time.Now(), + lastFlushHash: block1000Hash, + bestHash: block2000Hash, + want: false, + }, { + name: "equal to max size", + totalEntrySize: 1000, + maxSize: 1000, + lastFlushTime: time.Now(), + lastFlushHash: block1000Hash, + bestHash: block2000Hash, + want: true, + }, { + name: "greater than max size", + totalEntrySize: 1001, + maxSize: 1000, + lastFlushTime: time.Now(), + lastFlushHash: block1000Hash, + bestHash: block2000Hash, + want: true, + }, { + name: "less than max size but periodic duration reached", + totalEntrySize: 100, + maxSize: 1000, + lastFlushTime: time.Now().Add(periodicFlushInterval * -1), + lastFlushHash: block1000Hash, + bestHash: block2000Hash, + want: true, + }} + + for _, test := range tests { + // Create a utxo cache and set the field values as specified by the test. + utxoCache := NewUtxoCache(&UtxoCacheConfig{ + MaxSize: test.maxSize, + }) + utxoCache.totalEntrySize = test.totalEntrySize + utxoCache.lastFlushTime = test.lastFlushTime + utxoCache.lastFlushHash = *test.lastFlushHash + + // Validate that should flush returns the expected value. + got := utxoCache.shouldFlush(test.bestHash) + if got != test.want { + t.Errorf("%q: unexpected result -- got %v, want %v", test.name, got, + test.want) + } + } +} + +// TestMaybeFlush validates that the cache is properly flushed to the database +// under a variety of conditions. +func TestMaybeFlush(t *testing.T) { + t.Parallel() + + // Create a test database. + db := createTestUtxoDatabase(t) + + // Create test hashes to be used throughout the tests. + block1000Hash := mustParseHash("0000000000004740ad140c86753f9295e09f9cc81b1" + + "bb75d7f5552aeeedb7012") + block2000Hash := mustParseHash("0000000000000c8a886e3f7c32b1bb08422066dcfd0" + + "08de596471f11a5aff475") + + // entry299Fresh is from block height 299 and is modified and fresh. + outpoint299 := outpoint299() + entry299Fresh := entry299() + entry299Fresh.state |= utxoStateModified | utxoStateFresh + + // entry299Unmodified is from block height 299 and is unmodified. + entry299Unmodified := entry299() + + // entry1100Spent is from block height 1100 and is modified and spent. + outpoint1100 := outpoint1100() + entry1100Spent := entry1100() + entry1100Spent.Spend() + + // entry1100Modified is from block height 1100 and is modified and unspent. + entry1100Modified := entry1100() + entry1100Modified.state |= utxoStateModified + + // entry1100Unmodified is from block height 1100 and is unspent and + // unmodified. + entry1100Unmodified := entry1100() + + // entry1200Fresh is from block height 1200 and is modified and fresh. + outpoint1200 := outpoint1200() + entry1200Fresh := entry1200() + entry1200Fresh.state |= utxoStateModified | utxoStateFresh + + // entry1200Unmodified is from block height 1200 and is unmodified. + entry1200Unmodified := entry1200() + + tests := []struct { + name string + maxSize uint64 + lastEvictionHeight uint32 + lastFlushHash *chainhash.Hash + bestHash *chainhash.Hash + bestHeight uint32 + forceFlush bool + cachedEntries map[wire.OutPoint]*UtxoEntry + dbEntries map[wire.OutPoint]*UtxoEntry + wantCachedEntries map[wire.OutPoint]*UtxoEntry + wantDbEntries map[wire.OutPoint]*UtxoEntry + wantLastEvictionHeight uint32 + wantLastFlushHash *chainhash.Hash + wantUpdatedLastFlushTime bool + }{{ + name: "flush not required", + maxSize: 1000, + lastEvictionHeight: 0, + lastFlushHash: block1000Hash, + bestHash: block2000Hash, + bestHeight: 2000, + cachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint299: entry299Fresh, + outpoint1100: entry1100Spent, + outpoint1200: entry1200Fresh, + }, + dbEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1100: entry1100Modified, + }, + // The cache should remain unchanged since a flush is not required. + wantCachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint299: entry299Fresh, + outpoint1100: entry1100Spent, + outpoint1200: entry1200Fresh, + }, + // The db should remain unchanged since a flush is not required. + wantDbEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1100: entry1100Unmodified, + }, + wantLastEvictionHeight: 0, + wantLastFlushHash: block1000Hash, + wantUpdatedLastFlushTime: false, + }, { + name: "all entries flushed, some entries evicted", + maxSize: 0, + lastEvictionHeight: 0, + lastFlushHash: block1000Hash, + bestHash: block2000Hash, + bestHeight: 2000, + cachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint299: entry299Fresh, + outpoint1100: entry1100Spent, + outpoint1200: entry1200Fresh, + }, + dbEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1100: entry1100Modified, + }, + // entry299Fresh should be evicted from the cache due to its height. + // entry1100Spent should be evicted since it is spent. + // entry1200Fresh should remain in the cache but should now be unmodified. + wantCachedEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint1200: entry1200Unmodified, + }, + // entry299Unmodified should be added to the db during the flush. + // entry1100Unmodified should be removed from the db since it now spent. + // entry1200Unmodified should be added to the db during the flush. + wantDbEntries: map[wire.OutPoint]*UtxoEntry{ + outpoint299: entry299Unmodified, + outpoint1200: entry1200Unmodified, + }, + wantLastEvictionHeight: 300, + wantLastFlushHash: block2000Hash, + wantUpdatedLastFlushTime: true, + }} + + for _, test := range tests { + // Create a utxo cache with the cached entries specified by the test. + utxoCache := createTestUtxoCache(t, test.cachedEntries) + utxoCache.db = db + utxoCache.maxSize = test.maxSize + utxoCache.lastEvictionHeight = test.lastEvictionHeight + utxoCache.lastFlushHash = *test.lastFlushHash + origLastFlushTime := utxoCache.lastFlushTime + + // Add entries specified by the test to the test database. + err := db.Update(func(dbTx database.Tx) error { + for outpoint, entry := range test.dbEntries { + err := dbPutUtxoEntry(dbTx, outpoint, entry) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatalf("%q: unexpected error adding entries to test db: %v", test.name, + err) + } + + // Conditionally flush the cache based on the test parameters. + err = utxoCache.MaybeFlush(test.bestHash, test.bestHeight, test.forceFlush, + false) + if err != nil { + t.Fatalf("%q: unexpected error flushing cache: %v", test.name, err) + } + + // Validate that the cached entries match the expected entries after + // eviction. + if !reflect.DeepEqual(utxoCache.entries, test.wantCachedEntries) { + t.Fatalf("%q: mismatched cached entries:\nwant: %+v\n got: %+v\n", + test.name, test.wantCachedEntries, utxoCache.entries) + } + + // Validate that the db entries match the expected entries after flushing + // the cache. + dbEntries := make(map[wire.OutPoint]*UtxoEntry) + err = db.View(func(dbTx database.Tx) error { + for outpoint := range test.cachedEntries { + entry, err := dbFetchUtxoEntry(dbTx, outpoint) + if err != nil { + return err + } + + if entry != nil { + dbEntries[outpoint] = entry + } + } + return nil + }) + if err != nil { + t.Fatalf("%q: unexpected error fetching entries from test db: %v", + test.name, err) + } + if !reflect.DeepEqual(dbEntries, test.wantDbEntries) { + t.Fatalf("%q: mismatched db entries:\nwant: %+v\n got: %+v\n", test.name, + test.wantDbEntries, dbEntries) + } + + // Validate that the last flush hash and time have been updated as expexted. + if utxoCache.lastFlushHash != *test.wantLastFlushHash { + t.Fatalf("%q: unexpected last flush hash -- got %x, want %x", test.name, + utxoCache.lastFlushHash, *test.wantLastFlushHash) + } + updatedLastFlushTime := utxoCache.lastFlushTime != origLastFlushTime + if updatedLastFlushTime != test.wantUpdatedLastFlushTime { + t.Fatalf("%q: unexpected updated last flush time -- got %v, want %v", + test.name, updatedLastFlushTime, test.wantUpdatedLastFlushTime) + } + + // Validate the updated last eviction height. + if utxoCache.lastEvictionHeight != test.wantLastEvictionHeight { + t.Fatalf("%q: unexpected last eviction height -- got %d, want %d", + test.name, utxoCache.lastEvictionHeight, test.wantLastEvictionHeight) + } + + // Validate the updated total entry size of the cache. + wantTotalEntrySize := uint64(0) + for _, entry := range test.wantCachedEntries { + wantTotalEntrySize += entry.size() + } + if utxoCache.totalEntrySize != wantTotalEntrySize { + t.Fatalf("%q: unexpected total entry size -- got %v, want %v", test.name, + utxoCache.totalEntrySize, wantTotalEntrySize) + } + } +} From e4d6623b2d801f745c96844a3b8940515f2da8bb Mon Sep 17 00:00:00 2001 From: Ryan Staudt Date: Thu, 11 Feb 2021 15:14:39 -0600 Subject: [PATCH 10/10] docs: Update min recommended specs in README.md. This updates the minimum recommended memory (RAM) from 1GB to 2GB in the main README.md. The minimum recommended memory is being increased due to the introduction of the utxo cache. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9634fbed39..2cf8d688fd 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ and [Decrediton (GUI)](https://github.com/decred/decrediton). ## Minimum Recommended Specifications (dcrd only) * 12 GB disk space (as of April 2020, increases over time) -* 1GB memory (RAM) +* 2GB memory (RAM) * ~150MB/day download, ~1.5GB/day upload * Plus one-time initial download of the entire block chain * Windows 10 (server preferred), macOS, Linux