Skip to content

Commit

Permalink
indexers: get rid of proofInterval related code from
Browse files Browse the repository at this point in the history 
flatutreexoproofindex
  • Loading branch information
@kcalvinalvin
kcalvinalvin committed Oct 22, 2024
1 parent 282401c commit 900d953
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 202 deletions.
206 changes: 19 additions & 187 deletions blockchain/indexers/flatutreexoproofindex.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"fmt"
"os"
"path/filepath"
"reflect"
"sync"
"time"

Expand Down Expand Up @@ -77,11 +76,10 @@ var _ NeedsInputser = (*FlatUtreexoProofIndex)(nil)
// FlatUtreexoProofIndex implements a utreexo accumulator proof index for all the blocks.
// In a flat file.
type FlatUtreexoProofIndex struct {
proofGenInterVal int32
proofState FlatFileState
undoState FlatFileState
proofStatsState FlatFileState
rootsState FlatFileState
proofState FlatFileState
undoState FlatFileState
proofStatsState FlatFileState
rootsState FlatFileState

// All the configurable metadata.
config *UtreexoConfig
Expand Down Expand Up @@ -352,11 +350,11 @@ func (idx *FlatUtreexoProofIndex) ConnectBlock(dbTx database.Tx, block *btcutil.
for _, ld := range ud.LeafDatas {
delHashes = append(delHashes, ld.LeafHash())
}
// For pruned nodes and for multi-block proofs, we need to save the
// undo block in order to undo a block on reorgs. If we have all the
// proofs block by block, that data can be used for reorgs but these
// two modes will not have the proofs available.
if idx.config.Pruned || idx.proofGenInterVal != 1 {
// For pruned nodes, we need to save the undo block in order to undo
// a block on reorgs. If we have all the proofs block by block, that
// data can be used for reorgs but a pruned node will not have the
// proofs available.
if idx.config.Pruned {
err = idx.storeUndoBlock(block.Height(),
uint64(len(adds)), ud.AccProof.Targets, delHashes)
if err != nil {
Expand Down Expand Up @@ -400,26 +398,9 @@ func (idx *FlatUtreexoProofIndex) ConnectBlock(dbTx database.Tx, block *btcutil.
return err
}

// If the interval is 1, then just save the utreexo proof and we're done.
if idx.proofGenInterVal == 1 {
err = idx.storeProof(block.Height(), false, ud)
if err != nil {
return err
}
} else {
// Every proof generation interval, we'll make a multi-block proof.
if (block.Height() % idx.proofGenInterVal) == 0 {
err = idx.MakeMultiBlockProof(block.Height(), block.Height()-idx.proofGenInterVal,
block, ud, stxos)
if err != nil {
return err
}
} else {
err = idx.storeProof(block.Height(), true, ud)
if err != nil {
return err
}
}
err = idx.storeProof(block.Height(), false, ud)
if err != nil {
return err
}

return nil
Expand Down Expand Up @@ -459,65 +440,6 @@ func (idx *FlatUtreexoProofIndex) fetchBlocks(start, end int32) (
return blocks, allStxos, nil
}

// deletionsToProve returns all the deletions that need to be proven from the given
// blocks and stxos.
func (idx *FlatUtreexoProofIndex) deletionsToProve(blocks []*btcutil.Block,
stxos [][]blockchain.SpentTxOut) ([]wire.LeafData, [][]uint32, error) {

// Check that the length is equal to prevent index errors in the below loop.
if len(blocks) != len(stxos) {
err := fmt.Errorf("Got %d blocks but %d stxos", len(blocks), len(stxos))
return nil, nil, err
}

// createdMap will keep track of all the utxos created in the blocks that were
// passed in.
createdMap := make(map[wire.OutPoint]uint32)

remembers := make([][]uint32, len(blocks))

var delsToProve []wire.LeafData
for i, block := range blocks {
_, _, inskip, outskip := blockchain.DedupeBlock(block)

var txOutBlockIdx uint32
for _, tx := range block.Transactions() {
for outIdx := range tx.MsgTx().TxOut {
// Skip txos on the skip list
if len(outskip) > 0 && outskip[0] == txOutBlockIdx {
outskip = outskip[1:]
txOutBlockIdx++
continue
}

op := wire.OutPoint{Hash: *tx.Hash(), Index: uint32(outIdx)}
createdMap[op] = txOutBlockIdx
txOutBlockIdx++
}
}

excludeAfter := block.Height() - (block.Height() % idx.proofGenInterVal)

dels, excludes, err := blockchain.BlockToDelLeaves(stxos[i], idx.chain,
block, inskip, excludeAfter)
if err != nil {
return nil, nil, err
}

for _, excluded := range excludes {
val, ok := createdMap[excluded.Outpoint]
if ok {
idx := excluded.Height - excludeAfter
remembers[idx] = append(remembers[idx], val)
}
}

delsToProve = append(delsToProve, dels...)
}

return delsToProve, remembers, nil
}

// attachBlock attaches the passed in block to the utreexo accumulator state.
func (idx *FlatUtreexoProofIndex) attachBlock(blk *btcutil.Block, stxos []blockchain.SpentTxOut) error {
_, outCount, inskip, outskip := blockchain.DedupeBlock(blk)
Expand Down Expand Up @@ -693,91 +615,16 @@ func (idx *FlatUtreexoProofIndex) undoUtreexoState(currentHeight, desiredHeight
return nil
}

// MakeMultiBlockProof reverses the utreexo accumulator state to the multi-block proof
// generation height and makes a proof of all the stxos in the upcoming interval. The
// utreexo state is caught back up to the current height after the mulit-block proof is
// generated.
func (idx *FlatUtreexoProofIndex) MakeMultiBlockProof(currentHeight, proveHeight int32,
block *btcutil.Block, currentUD *wire.UData, stxos []blockchain.SpentTxOut) error {

idx.mtx.Lock()
defer idx.mtx.Unlock()

startRoots := idx.utreexoState.state.GetRoots()

// Go back to the desired block to generate the multi-block proof.
err := idx.undoUtreexoState(currentHeight, proveHeight)
if err != nil {
return err
}

blocks, allStxos, err := idx.fetchBlocks(proveHeight, currentHeight)
if err != nil {
return err
}

if int32(len(blocks)) != idx.proofGenInterVal {
err := fmt.Errorf("Only fetched %d blocks but the proofGenInterVal is %d",
len(blocks), idx.proofGenInterVal)
panic(err)
}

delsToProve, _, err := idx.deletionsToProve(blocks, allStxos)
if err != nil {
return err
}

ud, err := wire.GenerateUData(delsToProve, idx.utreexoState.state)
if err != nil {
panic(err)
}

delHashes := make([]utreexo.Hash, 0, len(delsToProve))
for _, del := range delsToProve {
delHashes = append(delHashes, del.LeafHash())
}

// Store the proof that we have created.
err = idx.storeMultiBlockProof(currentHeight, currentUD, ud, delHashes)
if err != nil {
return err
}

// Re-sync all the reorged blocks.
err = idx.reattachToUtreexoState(blocks, allStxos)
if err != nil {
return err
}

// Attach the current block.
err = idx.attachBlock(block, stxos)
if err != nil {
return err
}

// Sanity check.
endRoots := idx.utreexoState.state.GetRoots()
if !reflect.DeepEqual(endRoots, startRoots) {
err := fmt.Errorf("MakeMultiBlockProof: start roots and endroots differ. " +
"Likely that the database is corrupted.")
panic(err)
}

idx.pStats.UpdateMultiUDStats(len(delsToProve), ud)

return nil
}

// getUndoData returns the data needed for undo. For pruned nodes and multi-block proof enabled nodes,
// we fetch the data from the undo block. For archive nodes, we generate the data from the proof.
// getUndoData returns the data needed for undo. For pruned nodes, we fetch the data from the undo block.
// For archive nodes, we generate the data from the proof.
func (idx *FlatUtreexoProofIndex) getUndoData(block *btcutil.Block) (uint64, []uint64, []utreexo.Hash, error) {
var (
numAdds uint64
targets []uint64
delHashes []utreexo.Hash
)

if !idx.config.Pruned || idx.proofGenInterVal != 1 {
if !idx.config.Pruned {
ud, err := idx.FetchUtreexoProof(block.Height(), false)
if err != nil {
return 0, nil, nil, err
Expand Down Expand Up @@ -839,9 +686,8 @@ func (idx *FlatUtreexoProofIndex) DisconnectBlock(dbTx database.Tx, block *btcut

// Check if we're at a height where proof was generated. Only check if we're not
// pruned as we don't keep the historical proofs as a pruned node.
if (block.Height()%idx.proofGenInterVal) == 0 && !idx.config.Pruned {
height := block.Height() / idx.proofGenInterVal
err = idx.proofState.DisconnectBlock(height)
if !idx.config.Pruned {
err = idx.proofState.DisconnectBlock(block.Height())
if err != nil {
return err
}
Expand Down Expand Up @@ -990,11 +836,6 @@ func (idx *FlatUtreexoProofIndex) FetchMultiUtreexoProof(height int32) (
return nil, nil, nil, fmt.Errorf("Cannot fetch historical proof as the node is pruned")
}

if height%idx.proofGenInterVal != 0 {
return nil, nil, nil, fmt.Errorf("Attempting to fetch multi-block proof at the wrong height "+
"height:%d, proofGenInterVal:%d", height, idx.proofGenInterVal)
}

// Fetch the serialized data.
proofBytes, err := idx.proofState.FetchData(height)
if err != nil {
Expand Down Expand Up @@ -1300,19 +1141,10 @@ func loadFlatFileState(dataDir, name string) (*FlatFileState, error) {
// turn is used by the blockchain package. This allows the index to be
// seamlessly maintained along with the chain.
func NewFlatUtreexoProofIndex(pruned bool, chainParams *chaincfg.Params,
proofGenInterVal *int32, maxMemoryUsage int64, dataDir string, flush func() error) (*FlatUtreexoProofIndex, error) {

// If the proofGenInterVal argument is nil, use the default value.
var intervalToUse int32
if proofGenInterVal != nil {
intervalToUse = *proofGenInterVal
} else {
intervalToUse = defaultProofGenInterval
}
maxMemoryUsage int64, dataDir string, flush func() error) (*FlatUtreexoProofIndex, error) {

idx := &FlatUtreexoProofIndex{
proofGenInterVal: intervalToUse,
mtx: new(sync.RWMutex),
mtx: new(sync.RWMutex),
config: &UtreexoConfig{
MaxMemoryUsage: maxMemoryUsage,
Params: chainParams,
Expand Down
16 changes: 7 additions & 9 deletions blockchain/indexers/indexers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,12 +70,10 @@ func createDB(dbName string) (database.DB, string, error) {
return db, dbPath, nil
}

func initIndexes(interval int32, dbPath string, db database.DB, params *chaincfg.Params) (
func initIndexes(dbPath string, db database.DB, params *chaincfg.Params) (
*Manager, []Indexer, error) {

proofGenInterval := new(int32)
*proofGenInterval = interval
flatUtreexoProofIndex, err := NewFlatUtreexoProofIndex(false, params, proofGenInterval, 50*1024*1024, dbPath, db.Flush)
flatUtreexoProofIndex, err := NewFlatUtreexoProofIndex(false, params, 50*1024*1024, dbPath, db.Flush)
if err != nil {
return nil, nil, err
}
Expand All @@ -93,7 +91,7 @@ func initIndexes(interval int32, dbPath string, db database.DB, params *chaincfg
return indexManager, indexes, nil
}

func indexersTestChain(testName string, proofGenInterval int32) (*blockchain.BlockChain, []Indexer, *chaincfg.Params, *Manager, func()) {
func indexersTestChain(testName string) (*blockchain.BlockChain, []Indexer, *chaincfg.Params, *Manager, func()) {
params := chaincfg.RegressionNetParams
params.CoinbaseMaturity = 1

Expand All @@ -109,7 +107,7 @@ func indexersTestChain(testName string, proofGenInterval int32) (*blockchain.Blo
}

// Create the indexes to be used in the chain.
indexManager, indexes, err := initIndexes(proofGenInterval, dbPath, db, &params)
indexManager, indexes, err := initIndexes(dbPath, db, &params)
if err != nil {
tearDown()
os.RemoveAll(testDbRoot)
Expand Down Expand Up @@ -483,7 +481,7 @@ func TestProveUtxos(t *testing.T) {
source := rand.NewSource(timenow)
rand := rand.New(source)

chain, indexes, params, _, tearDown := indexersTestChain("TestProveUtxos", 1)
chain, indexes, params, _, tearDown := indexersTestChain("TestProveUtxos")
defer tearDown()

var allSpends []*blockchain.SpendableOut
Expand Down Expand Up @@ -621,7 +619,7 @@ func TestUtreexoProofIndex(t *testing.T) {
source := rand.NewSource(timenow)
rand := rand.New(source)

chain, indexes, params, _, tearDown := indexersTestChain("TestUtreexoProofIndex", 1)
chain, indexes, params, _, tearDown := indexersTestChain("TestUtreexoProofIndex")
defer tearDown()

tip := btcutil.NewBlock(params.GenesisBlock)
Expand Down Expand Up @@ -742,7 +740,7 @@ func TestBridgeNodePruneUndoDataGen(t *testing.T) {
// Always remove the root on return.
defer os.RemoveAll(testDbRoot)

chain, indexes, params, indexManager, tearDown := indexersTestChain("TestBridgeNodePruneUndoDataGen", 1)
chain, indexes, params, indexManager, tearDown := indexersTestChain("TestBridgeNodePruneUndoDataGen")
defer tearDown()

var allSpends []*blockchain.SpendableOut
Expand Down
8 changes: 2 additions & 6 deletions server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3249,14 +3249,10 @@ func newServer(listenAddrs, agentBlacklist, agentWhitelist []string,
if cfg.FlatUtreexoProofIndex {
indxLog.Info("Flat Utreexo Proof index is enabled")

// Create interval to pass to the flat utreexo proof index.
interval := new(int32)
*interval = 1

var err error
s.flatUtreexoProofIndex, err = indexers.NewFlatUtreexoProofIndex(
cfg.Prune != 0, chainParams, interval,
cfg.UtreexoProofIndexMaxMemory*1024*1024, cfg.DataDir, db.Flush)
cfg.Prune != 0, chainParams, cfg.UtreexoProofIndexMaxMemory*1024*1024,
cfg.DataDir, db.Flush)
if err != nil {
return nil, err
}
Expand Down

0 comments on commit 900d953

Please sign in to comment.