Merge pull request #85 from kcalvinalvin/2023-11-06-initial-mempool-s…

…upport-for-partial-proofs

main, mempool, blockchain, wire, wallet:  initial mempool support for partial proofs

blockchain/utreexoviewpoint.go

@@ -826,7 +826,7 @@ func (b *BlockChain) IsUtreexoViewActive() bool {
 //
 // This function does not modify the underlying UtreexoViewpoint.
 // This function is safe for concurrent access.
-func (b *BlockChain) VerifyUData(ud *wire.UData, txIns []*wire.TxIn) error {
+func (b *BlockChain) VerifyUData(ud *wire.UData, txIns []*wire.TxIn, remember bool) error {
 	// Nothing to prove.
 	if len(txIns) == 0 {
 		return nil
@@ -909,7 +909,7 @@ func (b *BlockChain) VerifyUData(ud *wire.UData, txIns []*wire.TxIn) error {
 
 	// VerifyBatchProof checks that the utreexo proofs are valid without
 	// mutating the accumulator.
-	err := b.utreexoView.accumulator.Verify(delHashes, ud.AccProof, false)
+	err := b.utreexoView.accumulator.Verify(delHashes, ud.AccProof, remember)
 	if err != nil {
 		str := "Verify fail. All txIns-leaf datas:\n"
 		for i, txIn := range txIns {
@@ -921,6 +921,10 @@ func (b *BlockChain) VerifyUData(ud *wire.UData, txIns []*wire.TxIn) error {
 		return fmt.Errorf(str)
 	}
 
+	if remember {
+		log.Debugf("cached hashes: %v", delHashes)
+	}
+
 	return nil
 }
 
@@ -939,6 +943,42 @@ func (b *BlockChain) GenerateUData(dels []wire.LeafData) (*wire.UData, error) {
 	return ud, nil
 }
 
+// PruneFromAccumulator uncaches the given hashes from the accumulator.  No action is taken
+// if the hashes are not already cached.
+func (b *BlockChain) PruneFromAccumulator(leaves []wire.LeafData) error {
+	if b.utreexoView == nil {
+		return fmt.Errorf("This blockchain instance doesn't have an " +
+			"accumulator. Cannot prune leaves")
+	}
+
+	b.chainLock.Lock()
+	defer b.chainLock.Unlock()
+
+	hashes := make([]utreexo.Hash, 0, len(leaves))
+	for i := range leaves {
+		// Unconfirmed leaves aren't present in the accumulator.
+		if leaves[i].IsUnconfirmed() {
+			continue
+		}
+
+		if leaves[i].IsCompact() {
+			return fmt.Errorf("Cannot generate hash as " +
+				"the leafdata is compact")
+		}
+
+		hashes = append(hashes, leaves[i].LeafHash())
+	}
+
+	log.Debugf("uncaching hashes: %v", hashes)
+
+	err := b.utreexoView.accumulator.Prune(hashes)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // ChainTipProof represents all the information that is needed to prove that a
 // utxo exists in the chain tip with utreexo accumulator proof.
 type ChainTipProof struct {

mempool/mempool.go

@@ -97,7 +97,10 @@ type Config struct {
 	// VerifyUData defines the function to use to verify the utreexo
 	// data.  This is only used when the node is run with the UtreexoView
 	// activated.
-	VerifyUData func(ud *wire.UData, txIns []*wire.TxIn) error
+	VerifyUData func(ud *wire.UData, txIns []*wire.TxIn, remember bool) error
+
+	// PruneFromAccumulator uncaches the given hashes from the accumulator.
+	PruneFromAccumulator func(hashes []wire.LeafData) error
 
 	// SigCache defines a signature cache to use.
 	SigCache *txscript.SigCache
@@ -189,6 +192,7 @@ type TxPool struct {
 	mtx           sync.RWMutex
 	cfg           Config
 	pool          map[chainhash.Hash]*TxDesc
+	poolLeaves    map[chainhash.Hash][]wire.LeafData
 	orphans       map[chainhash.Hash]*orphanTx
 	orphansByPrev map[wire.OutPoint]map[chainhash.Hash]*btcutil.Tx
 	outpoints     map[wire.OutPoint]*btcutil.Tx
@@ -496,6 +500,25 @@ func (mp *TxPool) removeTransaction(tx *btcutil.Tx, removeRedeemers bool) {
 			mp.cfg.AddrIndex.RemoveUnconfirmedTx(txHash)
 		}
 
+		// If the utreexo view is active, then remove the cached hashes from the
+		// accumulator.
+		if mp.cfg.IsUtreexoViewActive != nil && mp.cfg.IsUtreexoViewActive() {
+			leaves, found := mp.poolLeaves[*txHash]
+			if !found {
+				log.Infof("missing the leaf hashes for tx %s from while "+
+					"removing it from the pool",
+					tx.MsgTx().TxHash().String())
+			} else {
+				delete(mp.poolLeaves, *txHash)
+
+				err := mp.cfg.PruneFromAccumulator(leaves)
+				if err != nil {
+					log.Infof("err while pruning proof for inputs of tx %s: ",
+						err, tx.MsgTx().TxHash().String())
+				}
+			}
+		}
+
 		// Mark the referenced outpoints as unspent by the pool.
 		for _, txIn := range txDesc.Tx.MsgTx().TxIn {
 			delete(mp.outpoints, txIn.PreviousOutPoint)
@@ -543,7 +566,21 @@ func (mp *TxPool) RemoveDoubleSpends(tx *btcutil.Tx) {
 // helper for maybeAcceptTransaction.
 //
 // This function MUST be called with the mempool lock held (for writes).
-func (mp *TxPool) addTransaction(utxoView *blockchain.UtxoViewpoint, tx *btcutil.Tx, height int32, fee int64) *TxDesc {
+func (mp *TxPool) addTransaction(utxoView *blockchain.UtxoViewpoint, tx *btcutil.Tx, height int32, fee int64) (*TxDesc, error) {
+	if mp.cfg.IsUtreexoViewActive != nil && mp.cfg.IsUtreexoViewActive() {
+		// Ingest the proof. Shouldn't error out with the proof being invalid
+		// here since we've already verified it above.
+		err := mp.cfg.VerifyUData(tx.MsgTx().UData, tx.MsgTx().TxIn, true)
+		if err != nil {
+			return nil, fmt.Errorf("error while ingesting proof. %v", err)
+		}
+
+		mp.poolLeaves[*tx.Hash()] = tx.MsgTx().UData.LeafDatas
+	}
+
+	// Nil out uneeded udata for the mempool.
+	tx.MsgTx().UData = nil
+
 	// Add the transaction to the pool and mark the referenced outpoints
 	// as spent by the pool.
 	txD := &TxDesc{
@@ -574,7 +611,7 @@ func (mp *TxPool) addTransaction(utxoView *blockchain.UtxoViewpoint, tx *btcutil
 		mp.cfg.FeeEstimator.ObserveTransaction(txD)
 	}
 
-	return txD
+	return txD, nil
 }
 
 // checkPoolDoubleSpend checks whether or not the passed transaction is
@@ -880,6 +917,21 @@ func (mp *TxPool) FetchTransaction(txHash *chainhash.Hash) (*btcutil.Tx, error)
 	return nil, fmt.Errorf("transaction is not in the pool")
 }
 
+// FetchLeafDatas returns the leafdatas for the given tx.  Returns an error if
+// the leaves for the given tx is not in the pool.
+func (mp *TxPool) FetchLeafDatas(txHash *chainhash.Hash) ([]wire.LeafData, error) {
+	// Protect concurrent access.
+	mp.mtx.RLock()
+	leaves, exists := mp.poolLeaves[*txHash]
+	mp.mtx.RUnlock()
+
+	if exists {
+		return leaves, nil
+	}
+
+	return nil, fmt.Errorf("leafdata for the transaction is not in the pool")
+}
+
 // validateReplacement determines whether a transaction is deemed as a valid
 // replacement of all of its conflicts according to the RBF policy. If it is
 // valid, no error is returned. Otherwise, an error is returned indicating what
@@ -1083,9 +1135,11 @@ func (mp *TxPool) maybeAcceptTransaction(tx *btcutil.Tx, isNew, rateLimit, rejec
 
 		// First verify the proof to ensure that the proof the peer has
 		// sent was over valid.
-		err := mp.cfg.VerifyUData(ud, tx.MsgTx().TxIn)
+		err = mp.cfg.VerifyUData(ud, tx.MsgTx().TxIn, false)
 		if err != nil {
-			return nil, nil, err
+			str := fmt.Sprintf("transaction %v failed the utreexo data verification.",
+				txHash)
+			return nil, nil, txRuleError(wire.RejectInvalid, str)
 		}
 		log.Debugf("VerifyUData passed for tx %s", txHash.String())
 
@@ -1303,7 +1357,10 @@ func (mp *TxPool) maybeAcceptTransaction(tx *btcutil.Tx, isNew, rateLimit, rejec
 		// this call as they'll be removed eventually.
 		mp.removeTransaction(conflict, false)
 	}
-	txD := mp.addTransaction(utxoView, tx, bestHeight, txFee)
+	txD, err := mp.addTransaction(utxoView, tx, bestHeight, txFee)
+	if err != nil {
+		return nil, txD, err
+	}
 
 	log.Debugf("Accepted transaction %v (pool size: %v)", txHash,
 		len(mp.pool))
@@ -1624,6 +1681,7 @@ func New(cfg *Config) *TxPool {
 	return &TxPool{
 		cfg:            *cfg,
 		pool:           make(map[chainhash.Hash]*TxDesc),
+		poolLeaves:     make(map[chainhash.Hash][]wire.LeafData),
 		orphans:        make(map[chainhash.Hash]*orphanTx),
 		orphansByPrev:  make(map[wire.OutPoint]map[chainhash.Hash]*btcutil.Tx),
 		nextExpireScan: time.Now().Add(orphanExpireScanInterval),

server.go

@@ -1529,8 +1529,35 @@ func (s *server) pushTxMsg(sp *serverPeer, hash *chainhash.Hash, doneChan chan<-
 			return err
 		}
 
-		// We already checked that at least one is active.  Pick one and
-		// generate the UData.
+		// For compact state nodes.
+		if cfg.Utreexo {
+			// Fetch the necessary leafdatas to create the utreexo data.
+			leafDatas, err := s.txMemPool.FetchLeafDatas(tx.Hash())
+			if err != nil {
+				chanLog.Errorf(err.Error())
+				if doneChan != nil {
+					doneChan <- struct{}{}
+				}
+				return err
+			}
+
+			btcdLog.Debugf("fetched %v for tx %s", leafDatas, tx.Hash())
+
+			// This creates the accumulator proof and also puts the leaf datas
+			// in the utreexo data.
+			ud, err := s.chain.GenerateUData(leafDatas)
+			if err != nil {
+				chanLog.Errorf(err.Error())
+				if doneChan != nil {
+					doneChan <- struct{}{}
+				}
+				return err
+			}
+
+			tx.MsgTx().UData = ud
+		}
+
+		// For bridge nodes.
 		if s.utreexoProofIndex != nil {
 			leafDatas, err := blockchain.TxToDelLeaves(tx, s.chain)
 			if err != nil {
@@ -2575,8 +2602,10 @@ func (s *server) UpdateProofBytesWritten(msgTx *wire.MsgTx) error {
 		s.addAccBytesSent(uint64(accSize))
 
 	} else if s.chain.IsUtreexoViewActive() {
-		s.addProofBytesSent(uint64(msgTx.UData.SerializeSizeCompact(true)))
-		s.addAccBytesSent(uint64(msgTx.UData.SerializeAccSizeCompact()))
+		if msgTx.UData != nil {
+			s.addProofBytesSent(uint64(msgTx.UData.SerializeSizeCompact(true)))
+			s.addAccBytesSent(uint64(msgTx.UData.SerializeAccSizeCompact()))
+		}
 	}
 
 	return nil
@@ -3160,13 +3189,14 @@ func newServer(listenAddrs, agentBlacklist, agentWhitelist []string,
 		CalcSequenceLock: func(tx *btcutil.Tx, view *blockchain.UtxoViewpoint) (*blockchain.SequenceLock, error) {
 			return s.chain.CalcSequenceLock(tx, view, true)
 		},
-		IsDeploymentActive:  s.chain.IsDeploymentActive,
-		IsUtreexoViewActive: s.chain.IsUtreexoViewActive,
-		VerifyUData:         s.chain.VerifyUData,
-		SigCache:            s.sigCache,
-		HashCache:           s.hashCache,
-		AddrIndex:           s.addrIndex,
-		FeeEstimator:        s.feeEstimator,
+		IsDeploymentActive:   s.chain.IsDeploymentActive,
+		IsUtreexoViewActive:  s.chain.IsUtreexoViewActive,
+		VerifyUData:          s.chain.VerifyUData,
+		PruneFromAccumulator: s.chain.PruneFromAccumulator,
+		SigCache:             s.sigCache,
+		HashCache:            s.hashCache,
+		AddrIndex:            s.addrIndex,
+		FeeEstimator:         s.feeEstimator,
 	}
 	s.txMemPool = mempool.New(&txC)
 

wallet/watchonly.go

@@ -864,7 +864,7 @@ func (wm *WatchOnlyWalletManager) ProveTx(tx *btcutil.Tx) (*wire.UData, error) {
 	}
 
 	// Verify that the generated proof passes verification.
-	err = wm.config.Chain.VerifyUData(&ud, tx.MsgTx().TxIn)
+	err = wm.config.Chain.VerifyUData(&ud, tx.MsgTx().TxIn, false)
 	if err != nil {
 		return nil, fmt.Errorf("Couldn't prove tx %s. Generated proof "+
 			"fails verification. Error: %v", tx.Hash(), err)

wire/leaf.go

@@ -149,6 +149,13 @@ func (l *LeafData) SetUnconfirmed() {
 	l.Height = -1
 }
 
+// IsCompact returns if the leaf data is in the compact state.
+func (l *LeafData) IsCompact() bool {
+	return l.BlockHash == empty &&
+		l.OutPoint.Hash == empty &&
+		l.OutPoint.Index == 0
+}
+
 // -----------------------------------------------------------------------------
 // LeafData serialization includes all the data needed for generating the hash
 // commitment of the LeafData.

wire/leaf_test.go

@@ -570,3 +570,63 @@ func TestLeafDataJsonMarshal(t *testing.T) {
 		}
 	}
 }
+
+func TestIsCompact(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		ld   LeafData
+	}{
+		{
+			name: "Testnet3 tx 061bb0bf... from block 1600000",
+			ld: LeafData{
+				BlockHash: *newHashFromStr("00000000000172ff8a4e14441512072bacaf8d38b995a3fcd2f8435efc61717d"),
+				OutPoint: OutPoint{
+					Hash:  *newHashFromStr("061bb0bf3a1b9df13773da06bf92920394887a9c2b8b8772ac06be4e077df5eb"),
+					Index: 10,
+				},
+				Amount:     200000,
+				PkScript:   hexToBytes("a914e8d74935cfa223f9750a32b18d609cba17a5c3fe87"),
+				Height:     1599255,
+				IsCoinBase: false,
+			},
+		},
+		{
+			name: "Mainnet coinbase tx fa201b65... from block 573123",
+			ld: LeafData{
+				BlockHash: *newHashFromStr("000000000000000000278eb9386b4e70b850a4ec21907af3a27f50330b7325aa"),
+				OutPoint: OutPoint{
+					Hash:  *newHashFromStr("fa201b650eef761f5701afbb610e4a211b86985da4745aec3ac0f4b7a8e2c8d2"),
+					Index: 0,
+				},
+				Amount:     1315080370,
+				PkScript:   hexToBytes("76a9142cc2b87a28c8a097f48fcc1d468ced6e7d39958d88ac"),
+				Height:     573123,
+				IsCoinBase: true,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		if test.ld.IsCompact() {
+			t.Fatalf("leafdata %v is not compact but IsCompact returned %v", test.ld, test.ld.IsCompact())
+		}
+
+		var w bytes.Buffer
+		err := test.ld.SerializeCompact(&w, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		compact := NewLeafData()
+		err = compact.DeserializeCompact(&w, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !compact.IsCompact() {
+			t.Fatalf("leafdata %v is compact but IsCompact returned %v", test.ld, compact.IsCompact())
+		}
+	}
+}

wire/udata.go

@@ -446,6 +446,14 @@ func GenerateUData(txIns []LeafData, pollard utreexo.Utreexo) (
 			unconfirmedCount++
 			continue
 		}
+
+		// We can't calculate the correct hash if the leaf data is in
+		// the compact state.
+		if ld.IsCompact() {
+			return nil, fmt.Errorf("leafdata is compact. Unable " +
+				"to generate a leafhash")
+		}
+
 		delHashes = append(delHashes, ld.LeafHash())
 	}