Skip to content

Commit

Permalink
fix: lotus-shed: make finality calculator calculate properly
Browse files Browse the repository at this point in the history
  • Loading branch information
@rvagg
rvagg committed Jun 17, 2024
1 parent f776afd commit 17a2894
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 72 deletions.
107 changes: 40 additions & 67 deletions cmd/lotus-shed/finality.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ import (
"os"
"strconv"

"github.com/dreading/gospecfunc/bessel"
"github.com/filecoin-project/lotus/build"
skellampmf "github.com/rvagg/go-skellam-pmf"
"github.com/urfave/cli/v2"
"golang.org/x/exp/constraints"
"gonum.org/v1/gonum/stat/distuv"

"github.com/filecoin-project/lotus/build"
)

var finalityCmd = &cli.Command{
Expand All @@ -25,6 +25,10 @@ var finalityCmd = &cli.Command{
&cli.StringFlag{
Name: "input",
},
&cli.IntFlag{
Name: "target",
Usage: "target epoch for which finality is calculated",
},
},
ArgsUsage: "[inputFile]",
Action: func(cctx *cli.Context) error {
Expand All @@ -33,7 +37,7 @@ var finalityCmd = &cli.Command{
if err != nil {
return err
}
defer file.Close()
defer func() { _ = file.Close() }()

var chain []int
scanner := bufio.NewScanner(file)
Expand All @@ -49,14 +53,15 @@ var finalityCmd = &cli.Command{
return err
}

blocksPerEpoch := 5.0 // Expected number of blocks per epoch
byzantineFraction := 0.3 // Upper bound on the fraction of malicious nodes in the network
currentEpoch := len(chain) - 1 // Current epoch (end of history)
targetEpoch := currentEpoch - 30 // Target epoch for which finality is calculated
blocksPerEpoch := 5.0 // Expected number of blocks per epoch
byzantineFraction := 0.3 // Upper bound on the fraction of malicious nodes in the network
currentEpoch := len(chain) - 1 // Current epoch (end of history)
// targetEpoch := currentEpoch - 30 // Target epoch for which finality is calculated
targetEpoch := cctx.Int("target")

finality := FinalityCalcValidator(chain, blocksPerEpoch, byzantineFraction, currentEpoch, targetEpoch)

fmt.Fprintf(cctx.App.Writer, "Finality probability: %f\n", finality)
_, _ = fmt.Fprintf(cctx.App.Writer, "Finality=%v @ %d for chain len=%d\n", finality, targetEpoch, currentEpoch)

return nil
},
Expand All @@ -69,10 +74,10 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio
// Threshold at which the probability of an event is considered negligible
const negligibleThreshold = 1e-25

maxKL := 400 // Max k for which to calculate Pr(L=k)
maxKB := int((currentEpoch - targetEpoch) * int(blocksPerEpoch)) // Max k for which to calculate Pr(B=k)
maxKM := 400 // Max k for which to calculate Pr(M=k)
maxIM := 100 // Maximum number of epochs for the calculation (after which the pr become negligible)
maxKL := 400 // Max k for which to calculate Pr(L=k)
maxKB := (currentEpoch - targetEpoch) * int(blocksPerEpoch) // Max k for which to calculate Pr(B=k)
maxKM := 400 // Max k for which to calculate Pr(M=k)
maxIM := 100 // Maximum number of epochs for the calculation (after which the pr become negligible)

rateMaliciousBlocks := blocksPerEpoch * byzantineFraction // upper bound
rateHonestBlocks := blocksPerEpoch - rateMaliciousBlocks // lower bound
Expand All @@ -88,15 +93,14 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio
sumExpectedAdversarialBlocksI += rateMaliciousBlocks
sumChainBlocksI += chain[i-1]
// Poisson(k=k, lambda=sum(f*e))
prLi := distuv.Poisson{Lambda: sumExpectedAdversarialBlocksI}.Prob(float64(k + sumChainBlocksI))
prLi := poissonProb(sumExpectedAdversarialBlocksI, float64(k+sumChainBlocksI))
prL[k] = math.Max(prL[k], prLi)

// Break if prL[k] becomes negligible
if k > 1 && prL[k] < negligibleThreshold && prL[k] < prL[k-1] {
maxKL = k
prL = prL[:k+1]
break
}
}
if k > 1 && prL[k] < negligibleThreshold && prL[k] < prL[k-1] {
maxKL = k
prL = prL[:k+1]
break
}
}

Expand All @@ -108,7 +112,7 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio

// Calculate Pr(B=k) for each value of k
for k := 0; k <= maxKB; k++ {
prB[k] = distuv.Poisson{Lambda: float64(currentEpoch-targetEpoch) * rateMaliciousBlocks}.Prob(float64(k))
prB[k] = poissonProb(float64(currentEpoch-targetEpoch)*rateMaliciousBlocks, float64(k))

// Break if prB[k] becomes negligible
if k > 1 && prB[k] < negligibleThreshold && prB[k] < prB[k-1] {
Expand All @@ -119,11 +123,11 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio
}

// Compute M
prHgt0 := 1 - distuv.Poisson{Lambda: rateHonestBlocks}.Prob(0)
prHgt0 := 1 - poissonProb(rateHonestBlocks, 0)

expZ := 0.0
for k := 0; k < int(4*blocksPerEpoch); k++ {
pmf := distuv.Poisson{Lambda: rateMaliciousBlocks}.Prob(float64(k))
pmf := poissonProb(rateMaliciousBlocks, float64(k))
expZ += ((rateHonestBlocks + float64(k)) / math.Pow(2, float64(k))) * pmf
}

Expand All @@ -132,7 +136,7 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio
prM := make([]float64, maxKM+1)
for k := 0; k <= maxKM; k++ {
for i := maxIM; i > 0; i-- {
probMI := SkellamPMF(k, float64(i)*rateMaliciousBlocks, float64(i)*ratePublicChain)
probMI := skellampmf.SkellamPMF(k, float64(i)*rateMaliciousBlocks, float64(i)*ratePublicChain)

// Break if probMI becomes negligible
if probMI < negligibleThreshold && probMI < prM[k] {
Expand Down Expand Up @@ -186,6 +190,18 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio
return math.Min(prError, 1.0)
}

func poissonProb(lambda float64, x float64) float64 {
return math.Exp(poissonLogProb(lambda, x))
}

func poissonLogProb(lambda float64, x float64) float64 {
if x < 0 || math.Floor(x) != x {
return math.Inf(-1)
}
lg, _ := math.Lgamma(math.Floor(x) + 1)
return x*math.Log(lambda) - lambda - lg
}

func sum[T constraints.Integer | constraints.Float](s []T) T {
var total T
for _, v := range s {
Expand All @@ -210,46 +226,3 @@ func min(a, b int) int {
}
return b
}

// SkellamPMF calculates the probability mass function (PMF) of a Skellam distribution.
//
// The Skellam distribution is the probability distribution of the difference
// of two independent Poisson random variables.
//
// Arguments:
// * k - The difference of two Poisson random variables.
// * mu1 - The expected value of the first Poisson distribution.
// * mu2 - The expected value of the second Poisson distribution.
//
// Returns:
// * A float64 representing the PMF of the Skellam distribution at k.
func SkellamPMF(k int, mu1 float64, mu2 float64) float64 {
// Based on https://github.com/jsoares/rusty-skellam/blob/main/src/lib.rs

// Return NaN if parameters outside range
if math.IsNaN(mu1) || mu1 <= 0 || math.IsNaN(mu2) || mu2 <= 0 {
return math.NaN()
}

// Parameterise and compute the Modified Bessel function of the first kind
nu := float64(k)
z := complex(2.0*math.Sqrt(mu1*mu2), 0)
besselResult := bessel.I(nu, z)

// Compute the pmf
return math.Exp(-(mu1 + mu2)) * math.Pow(mu1/mu2, nu/2.0) * real(besselResult)
}

/*
func main() {
seed := rand.NewSource(1)
random := rand.New(seed)
chain := make([]int, 1000)
for i := range chain {
chain[i] = random.Intn(5) + 1
}
errorProbability := FinalityCalcValidator(chain, 5.0, 0.3, 1000, 900)
fmt.Printf("Error probability: %f\n", errorProbability)
}
*/
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ require (
github.com/puzpuzpuz/xsync/v2 v2.4.0
github.com/raulk/clock v1.1.0
github.com/raulk/go-watchdog v1.3.0
github.com/rvagg/go-skellam-pmf v0.0.1
github.com/samber/lo v1.39.0
github.com/stretchr/testify v1.9.0
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
Expand All @@ -149,6 +150,7 @@ require (
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.23.0
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
golang.org/x/net v0.25.0
golang.org/x/sync v0.7.0
golang.org/x/sys v0.20.0
Expand Down Expand Up @@ -184,7 +186,6 @@ require (
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/drand/kyber-bls12381 v0.3.1 // indirect
github.com/dreading/gospecfunc v0.0.0-20191105042551-e794f60da5c3 // indirect
github.com/elastic/go-windows v1.0.0 // indirect
github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect
github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect
Expand Down Expand Up @@ -321,7 +322,6 @@ require (
go.uber.org/dig v1.17.1 // indirect
go.uber.org/mock v0.4.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/text v0.15.0 // indirect
gonum.org/v1/gonum v0.15.0 // indirect
Expand Down
5 changes: 2 additions & 3 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,8 +216,6 @@ github.com/drand/kyber v1.3.0 h1:TVd7+xoRgKQ4Ck1viNLPFy6IWhuZM36Bq6zDXD8Asls=
github.com/drand/kyber v1.3.0/go.mod h1:f+mNHjiGT++CuueBrpeMhFNdKZAsy0tu03bKq9D5LPA=
github.com/drand/kyber-bls12381 v0.3.1 h1:KWb8l/zYTP5yrvKTgvhOrk2eNPscbMiUOIeWBnmUxGo=
github.com/drand/kyber-bls12381 v0.3.1/go.mod h1:H4y9bLPu7KZA/1efDg+jtJ7emKx+ro3PU7/jWUVt140=
github.com/dreading/gospecfunc v0.0.0-20191105042551-e794f60da5c3 h1:oOp1la+wHlyd3ODqW2CbFj8w6Lod4gPMzHFbD0rbp88=
github.com/dreading/gospecfunc v0.0.0-20191105042551-e794f60da5c3/go.mod h1:lkytgpljbGOM3VZj4Fm7FkGy/oUInQFklbkHBVAvJEg=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
Expand Down Expand Up @@ -1180,6 +1178,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/rvagg/go-skellam-pmf v0.0.1 h1:li3G0jioT+8bRUseNP+h0WRVv/CI+9s2q1qwNvEE994=
github.com/rvagg/go-skellam-pmf v0.0.1/go.mod h1:/xSBO272x+iW1BjHnPL6p2O7kCpVTh7QK9hZcXE/Kqk=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
Expand Down Expand Up @@ -1745,7 +1745,6 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191022213345-0bbdf54effa2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
Expand Down

0 comments on commit 17a2894

Please sign in to comment.