Skip to content

Commit

Permalink
[utils/bloom] Dynamic Bloom Filter (#452)
Browse files Browse the repository at this point in the history 
* update const name

* lower min target items

* fix lint

* add churn multiplier

* latest ago

* nits

* fix e2e

* update GetFilter

* use correct avalanche version

* update [email protected]

* update scripts version
  • Loading branch information
@patrick-ogrady
patrick-ogrady authored Jan 10, 2024
1 parent 8d23b32 commit d5a8550
Show file tree
Hide file tree
Showing 7 changed files with 25 additions and 25 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.20

require (
github.com/VictoriaMetrics/fastcache v1.10.0
github.com/ava-labs/avalanchego v1.10.18-rc.15
github.com/ava-labs/avalanchego v1.10.18-rc.17
github.com/cespare/cp v0.1.0
github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811
github.com/davecgh/go-spew v1.1.1
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/ava-labs/avalanchego v1.10.18-rc.15 h1:8HVOLBbaZnvE9YNL89DzUk+na9zluAz+GGbAwTXZZQs=
github.com/ava-labs/avalanchego v1.10.18-rc.15/go.mod h1:FsejaXWTz6rgjpk1dEbXUJvhX/Ip6ADmD4Q8WZHRNuY=
github.com/ava-labs/avalanchego v1.10.18-rc.17 h1:ZZxjDy54NDBwMW3b6LtBoDTe5jaJJipCQcq4iUXRskE=
github.com/ava-labs/avalanchego v1.10.18-rc.17/go.mod h1:7+Ao/l/f7PCMeVyYAn7xF0Xy8iX/MMLEr95pyoeURJM=
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
Expand Down
7 changes: 4 additions & 3 deletions plugin/evm/gossip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func (tx *GossipAtomicTx) GossipID() ids.ID {
}

func NewGossipEthTxPool(mempool *txpool.TxPool) (*GossipEthTxPool, error) {
bloom, err := gossip.NewBloomFilter(txGossipBloomMaxItems, txGossipBloomFalsePositiveRate)
bloom, err := gossip.NewBloomFilter(txGossipBloomMinTargetElements, txGossipBloomTargetFalsePositiveRate, txGossipBloomResetFalsePositiveRate)
if err != nil {
return nil, fmt.Errorf("failed to initialize bloom filter: %w", err)
}
Expand Down Expand Up @@ -139,10 +139,11 @@ func (g *GossipEthTxPool) Subscribe(ctx context.Context) {
return
case pendingTxs := <-g.pendingTxs:
g.lock.Lock()
optimalElements := (g.mempool.PendingSize() + len(pendingTxs.Txs)) * txGossipBloomChurnMultiplier
for _, pendingTx := range pendingTxs.Txs {
tx := &GossipEthTx{Tx: pendingTx}
g.bloom.Add(tx)
reset, err := gossip.ResetBloomFilterIfNeeded(g.bloom, txGossipMaxFalsePositiveRate)
reset, err := gossip.ResetBloomFilterIfNeeded(g.bloom, optimalElements)
if err != nil {
log.Error("failed to reset bloom filter", "err", err)
continue
Expand Down Expand Up @@ -174,7 +175,7 @@ func (g *GossipEthTxPool) Iterate(f func(tx *GossipEthTx) bool) {
})
}

func (g *GossipEthTxPool) GetFilter() ([]byte, []byte, error) {
func (g *GossipEthTxPool) GetFilter() ([]byte, []byte) {
g.lock.RLock()
defer g.lock.RUnlock()

Expand Down
6 changes: 3 additions & 3 deletions plugin/evm/mempool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ type Mempool struct {

// NewMempool returns a Mempool with [maxSize]
func NewMempool(ctx *snow.Context, maxSize int, verify func(tx *Tx) error) (*Mempool, error) {
bloom, err := gossip.NewBloomFilter(txGossipBloomMaxItems, txGossipBloomFalsePositiveRate)
bloom, err := gossip.NewBloomFilter(txGossipBloomMinTargetElements, txGossipBloomTargetFalsePositiveRate, txGossipBloomResetFalsePositiveRate)
if err != nil {
return nil, fmt.Errorf("failed to initialize bloom filter: %w", err)
}
Expand Down Expand Up @@ -340,7 +340,7 @@ func (m *Mempool) addTx(tx *Tx, force bool) error {
}

m.bloom.Add(&GossipAtomicTx{Tx: tx})
reset, err := gossip.ResetBloomFilterIfNeeded(m.bloom, txGossipMaxFalsePositiveRate)
reset, err := gossip.ResetBloomFilterIfNeeded(m.bloom, m.length()*txGossipBloomChurnMultiplier)
if err != nil {
return err
}
Expand Down Expand Up @@ -375,7 +375,7 @@ func (m *Mempool) Iterate(f func(tx *GossipAtomicTx) bool) {
}
}

func (m *Mempool) GetFilter() ([]byte, []byte, error) {
func (m *Mempool) GetFilter() ([]byte, []byte) {
m.lock.RLock()
defer m.lock.RUnlock()

Expand Down
10 changes: 4 additions & 6 deletions plugin/evm/tx_gossip_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,9 @@ func TestEthTxGossip(t *testing.T) {
}

// Ask the VM for any new transactions. We should get nothing at first.
emptyBloomFilter, err := gossip.NewBloomFilter(txGossipBloomMaxItems, txGossipBloomFalsePositiveRate)
require.NoError(err)
emptyBloomFilterBytes, _, err := emptyBloomFilter.Marshal()
emptyBloomFilter, err := gossip.NewBloomFilter(txGossipBloomMinTargetElements, txGossipBloomTargetFalsePositiveRate, txGossipBloomResetFalsePositiveRate)
require.NoError(err)
emptyBloomFilterBytes, _ := emptyBloomFilter.Marshal()
request := &sdk.PullGossipRequest{
Filter: emptyBloomFilterBytes,
Salt: agoUtils.RandomBytes(32),
Expand Down Expand Up @@ -225,10 +224,9 @@ func TestAtomicTxGossip(t *testing.T) {
}

// Ask the VM for any new transactions. We should get nothing at first.
emptyBloomFilter, err := gossip.NewBloomFilter(txGossipBloomMaxItems, txGossipBloomFalsePositiveRate)
require.NoError(err)
emptyBloomFilterBytes, _, err := emptyBloomFilter.Marshal()
emptyBloomFilter, err := gossip.NewBloomFilter(txGossipBloomMinTargetElements, txGossipBloomTargetFalsePositiveRate, txGossipBloomResetFalsePositiveRate)
require.NoError(err)
emptyBloomFilterBytes, _ := emptyBloomFilter.Marshal()
request := &sdk.PullGossipRequest{
Filter: emptyBloomFilterBytes,
Salt: agoUtils.RandomBytes(32),
Expand Down
19 changes: 10 additions & 9 deletions plugin/evm/vm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,15 +141,16 @@ const (
atomicTxGossipProtocol = 0x1

// gossip constants
txGossipBloomMaxItems = 8 * 1024
txGossipBloomFalsePositiveRate = 0.01
txGossipMaxFalsePositiveRate = 0.05
txGossipTargetMessageSize = 20 * units.KiB
maxValidatorSetStaleness = time.Minute
txGossipThrottlingPeriod = 10 * time.Second
txGossipThrottlingLimit = 2
gossipFrequency = 10 * time.Second
txGossipPollSize = 10
txGossipBloomMinTargetElements = 8 * 1024
txGossipBloomTargetFalsePositiveRate = 0.01
txGossipBloomResetFalsePositiveRate = 0.05
txGossipBloomChurnMultiplier = 3
txGossipTargetMessageSize = 20 * units.KiB
maxValidatorSetStaleness = time.Minute
txGossipThrottlingPeriod = 10 * time.Second
txGossipThrottlingLimit = 2
gossipFrequency = 10 * time.Second
txGossipPollSize = 10
)

// Define the API endpoints for the VM
Expand Down
2 changes: 1 addition & 1 deletion scripts/versions.sh
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env bash

# Don't export them as they're used in the context of other calls
avalanche_version=${AVALANCHE_VERSION:-'v1.10.18-rc.15'}
avalanche_version=${AVALANCHE_VERSION:-'v1.10.18-rc.17'}

0 comments on commit d5a8550

Please sign in to comment.