v0.5.3

Signed-off-by: Ignacio Hagopian <[email protected]>

README.md

@@ -1,9 +1,6 @@
 # Lotus Devnet
 Runs a Lotus Devnet using a mocked _sectorbuilder_ which can be accesed as a real Lotus node through the API.
-The devnet code is always tried to be update with Lotus `master`.
-
-Note that now there're other `lotus-devnet` branches for other main networks/releases: `nerpa`, and `butterfly`.
-They basically have the same Lotus commit tag for the running devnet.
+The devnet code is always tried to be update with Lotus latest `master` tag.
 
 ## Run
 You can run the devnet with:

build/bootstrap/bootstrappers.pi

@@ -1,2 +1,2 @@
-/dns4/t01000.miner.interopnet.kittyhawk.wtf/tcp/1347/p2p/12D3KooWGdh7rVCouXZkDbyfpEG8XbSDRE2DcsXLES9Lk4mfayNR
-/ip4/52.36.61.156/tcp/1347/p2p/12D3KooWN4GdKRXvqUgbgdSRby1cSWf7bxQUwjDr7rFPzGWceSPW
+/dns4/bootstrap-0.butterfly.fildev.network/tcp/1347/p2p/12D3KooWA8GMmWjTZGVeNDxbnyeEN8WGrQxASgw74Y7g2Vo2yAKD
+/dns4/bootstrap-1.butterfly.fildev.network/tcp/1347/p2p/12D3KooWQMyCcJgUsYojDgyjWf5FS8ucG2bHLd8QyZ6vW8F58wQ6

go.mod

@@ -4,21 +4,25 @@ go 1.14
 
 require (
 	github.com/GeertJohan/go.rice v1.0.0
-	github.com/filecoin-project/go-address v0.0.2-0.20200504173055-8b6f2fb2b3ef
-	github.com/filecoin-project/go-fil-markets v0.3.0
-	github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
+	github.com/filecoin-project/go-address v0.0.3
+	github.com/filecoin-project/go-fil-markets v0.5.7
+	github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
-	github.com/filecoin-project/lotus v0.4.0
-	github.com/filecoin-project/sector-storage v0.0.0-20200618073200-d9de9b7cb4b4
-	github.com/filecoin-project/specs-actors v0.6.2-0.20200617175406-de392ca14121
+	github.com/filecoin-project/lotus v0.5.3
+	github.com/filecoin-project/specs-actors v0.9.3
 	github.com/ipfs/go-datastore v0.4.4
-	github.com/ipfs/go-log/v2 v2.1.2-0.20200609205458-f8d20c392cb7
-	github.com/libp2p/go-libp2p v0.9.4
-	github.com/libp2p/go-libp2p-core v0.5.7
+	github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4
+	github.com/libp2p/go-libp2p v0.10.3
+	github.com/libp2p/go-libp2p-core v0.6.1
 	github.com/spf13/pflag v1.0.3
 	github.com/spf13/viper v1.3.2
-	github.com/stretchr/testify v1.5.1
-	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
+	github.com/stretchr/testify v1.6.1
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
+
 )
 
 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi
+
+replace github.com/supranational/blst => github.com/supranational/blst v0.1.2-alpha.1
+
+replace github.com/filecoin-project/lotus => github.com/jsign/lotus v0.4.1-0.20200825125340-e3ba578c3a07

ldevnet/ldevnet.go

@@ -5,16 +5,18 @@ import (
 	"context"
 	"crypto/rand"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"sync"
 	"time"
 
+	miner2 "github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/storage/mockstorage"
 
 	"github.com/filecoin-project/go-storedcounter"
 	"github.com/filecoin-project/lotus/api/apistruct"
+	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
 	"github.com/ipfs/go-datastore"
 	"github.com/libp2p/go-libp2p-core/crypto"
 	"github.com/libp2p/go-libp2p-core/peer"
@@ -24,7 +26,7 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
-	saminer "github.com/filecoin-project/specs-actors/actors/builtin/miner"
+	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/builtin/verifreg"
 
@@ -34,19 +36,19 @@ import (
 	"github.com/filecoin-project/lotus/api/test"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/gen"
 	genesis2 "github.com/filecoin-project/lotus/chain/gen/genesis"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/wallet"
-	genesis "github.com/filecoin-project/lotus/genesis"
-	"github.com/filecoin-project/lotus/miner"
+	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
+	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/lotus/extern/sector-storage/mock"
+	"github.com/filecoin-project/lotus/genesis"
 	"github.com/filecoin-project/lotus/node"
 	"github.com/filecoin-project/lotus/node/modules"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	modtest "github.com/filecoin-project/lotus/node/modules/testing"
+	testing2 "github.com/filecoin-project/lotus/node/modules/testing"
 	"github.com/filecoin-project/lotus/node/repo"
-	sectorstorage "github.com/filecoin-project/sector-storage"
-	"github.com/filecoin-project/sector-storage/ffiwrapper"
-	"github.com/filecoin-project/sector-storage/mock"
 )
 
 const (
@@ -56,7 +58,7 @@ const (
 
 func init() {
 	power.ConsensusMinerMinPower = big.NewInt(2048)
-	saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{
+	miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{
 		abi.RegisteredSealProof_StackedDrg2KiBV1: {},
 	}
 	verifreg.MinVerifiedDealSize = big.NewInt(256)
@@ -87,7 +89,7 @@ var PresealGenesis = -1
 
 func New(numMiners int, blockDur time.Duration, bigSector bool, ipfsAddr string) (*LocalDevnet, error) {
 	if bigSector {
-		saminer.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{
+		miner.SupportedProofTypes = map[abi.RegisteredSealProof]struct{}{
 			abi.RegisteredSealProof_StackedDrg512MiBV1: {},
 		}
 	}
@@ -115,23 +117,55 @@ func New(numMiners int, blockDur time.Duration, bigSector bool, ipfsAddr string)
 		}
 		time.Sleep(time.Second)
 	}
+	var mineNext = miner2.MineReq{
+		InjectNulls: 0,
+		Done:        func(bool, error) {},
+	}
 	go func() {
 		i := 0
 		mine := true
 		defer func() { done <- struct{}{} }()
 		for mine {
 			time.Sleep(blockDur)
 			if ctx.Err() != nil {
+				panic("context cancelled")
 				mine = false
 				continue
 			}
-			if err := sn[i].MineOne(context.Background(), func(bool) {}); err != nil {
+			if err := sn[i].MineOne(context.Background(), mineNext); err != nil {
 				panic(err)
 			}
+
 			i = (i + 1) % len(miners)
 		}
 	}()
 
+	go func() {
+		for {
+			for i := 0; i < numMiners; i++ {
+				snums, err := sn[i].SectorsList(ctx)
+				if err != nil {
+					panic(err)
+				}
+				for _, snum := range snums {
+					si, err := sn[i].SectorsStatus(ctx, snum, false)
+					if err != nil {
+						panic(err)
+					}
+
+					fmt.Printf("######## STATE: %s\n", si.State)
+					if si.State == api.SectorState(sealing.WaitDeals) {
+						if err := sn[i].SectorStartSealing(ctx, snum); err != nil {
+							panic(err)
+
+						}
+					}
+				}
+			}
+			time.Sleep(time.Second)
+		}
+	}()
+
 	for i := range miners {
 		for j := range miners {
 			if j == i {
@@ -238,19 +272,13 @@ func mockSbBuilder(nFull int, storage []test.StorageMiner, bigSector bool, ipfsA
 	var genms []genesis.Miner
 	var genaccs []genesis.Actor
 	var maddrs []address.Address
-	var presealDirs []string
 	var keys []*wallet.Key
 	var pidKeys []crypto.PrivKey
 	for i := 0; i < len(storage); i++ {
 		maddr, err := address.NewIDAddress(genesis2.MinerStart + uint64(i))
 		if err != nil {
 			return nil, nil, err
 		}
-		tdir, err := ioutil.TempDir("", "preseal-memgen")
-		if err != nil {
-			return nil, nil, err
-		}
-
 		preseals := storage[i].Preseal
 		if preseals == test.PresealGenesis {
 			preseals = nGenesisPreseals
@@ -273,28 +301,29 @@ func mockSbBuilder(nFull int, storage []test.StorageMiner, bigSector bool, ipfsA
 
 		genaccs = append(genaccs, genesis.Actor{
 			Type:    genesis.TAccount,
-			Balance: big.Mul(big.NewInt(500000), types.NewInt(build.FilecoinPrecision)),
+			Balance: big.Mul(big.NewInt(400000000), types.NewInt(build.FilecoinPrecision)),
 			Meta:    (&genesis.AccountMeta{Owner: wk.Address}).ActorMeta(),
 		})
 
 		keys = append(keys, wk)
 		pidKeys = append(pidKeys, minersPk[i])
-		presealDirs = append(presealDirs, tdir)
 		maddrs = append(maddrs, maddr)
 		genms = append(genms, *genm)
 	}
 	templ := &genesis.Template{
-		Accounts:  genaccs,
-		Miners:    genms,
-		Timestamp: uint64(time.Now().Add(-time.Hour * 100000).Unix()),
+		Accounts:         genaccs,
+		Miners:           genms,
+		Timestamp:        uint64(time.Now().Add(-time.Hour * 100000).Unix()),
+		VerifregRootKey:  gen.DefaultVerifregRootkeyActor,
+		RemainderAccount: gen.DefaultRemainderAccountActor,
 	}
 
 	// END PRESEAL SECTION
 
 	for i := 0; i < nFull; i++ {
 		var genesis node.Option
 		if i == 0 {
-			genesis = node.Override(new(modules.Genesis), modtest.MakeGenesisMem(&genbuf, *templ))
+			genesis = node.Override(new(modules.Genesis), testing2.MakeGenesisMem(&genbuf, *templ))
 		} else {
 			genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes()))
 		}
@@ -311,14 +340,17 @@ func mockSbBuilder(nFull int, storage []test.StorageMiner, bigSector bool, ipfsA
 
 			genesis,
 			node.ApplyIf(func(s *node.Settings) bool { return len(ipfsAddr) > 0 },
-				node.Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore(ipfsAddr, true))),
+				node.Override(new(dtypes.ClientBlockstore), modules.IpfsClientBlockstore(ipfsAddr)),
+				node.Override(new(dtypes.ClientRetrievalStoreManager), modules.ClientBlockstoreRetrievalStoreManager),
+			),
 		)
 		if err != nil {
 			return nil, nil, err
 		}
 	}
 
 	for i, def := range storage {
+		minerID := abi.ActorID(genesis2.MinerStart + uint64(i))
 		if def.Full != 0 {
 			return nil, nil, fmt.Errorf("storage nodes only supported on the first full node")
 		}
@@ -331,22 +363,42 @@ func mockSbBuilder(nFull int, storage []test.StorageMiner, bigSector bool, ipfsA
 			return nil, nil, err
 		}
 
+		sectors := make([]abi.SectorID, len(genms[i].Sectors))
+		for i, sector := range genms[i].Sectors {
+			sectors[i] = abi.SectorID{
+				Miner:  minerID,
+				Number: sector.SectorID,
+			}
+		}
+
 		storers[i] = testStorageNode(ctx, genms[i].Worker, maddrs[i], minersPk[i], f, mn, node.Options(
 			node.Override(new(sectorstorage.SectorManager), func() (sectorstorage.SectorManager, error) {
-				return mock.NewMockSectorMgr(build.DefaultSectorSize()), nil
+				return mock.NewMockSectorMgr(build.DefaultSectorSize(), sectors), nil
 			}),
 			node.Override(new(ffiwrapper.Verifier), mock.MockVerifier),
 			node.Unset(new(*sectorstorage.Manager)),
 		))
-		//if err := storers[i].StorageMiner.MarketSetPrice(ctx, types.NewInt(1000)); err != nil {
-		//	return nil, nil, err
-		//}
 	}
 
 	if err := mn.LinkAll(); err != nil {
 		return nil, nil, err
 	}
 
+	if len(storers) > 0 {
+		// Mine 2 blocks to setup some CE stuff in some actors
+		var wait sync.Mutex
+		wait.Lock()
+
+		storers[0].MineOne(ctx, miner2.MineReq{Done: func(bool, error) {
+			wait.Unlock()
+		}})
+		wait.Lock()
+		storers[0].MineOne(ctx, miner2.MineReq{Done: func(bool, error) {
+			wait.Unlock()
+		}})
+		wait.Lock()
+	}
+
 	return fulls, storers, nil
 }
 
@@ -380,11 +432,17 @@ func testStorageNode(ctx context.Context, waddr address.Address, act address.Add
 	if err != nil {
 		panic(err)
 	}
-	nic := storedcounter.New(ds, datastore.NewKey("/storage/nextid"))
+	nic := storedcounter.New(ds, datastore.NewKey(modules.StorageCounterDSPrefix))
 	for i := 0; i < nGenesisPreseals; i++ {
-		nic.Next()
+		_, err := nic.Next()
+		if err != nil {
+			panic(err)
+		}
+	}
+	_, err = nic.Next()
+	if err != nil {
+		panic(err)
 	}
-	nic.Next()
 
 	err = lr.Close()
 	if err != nil {
@@ -394,28 +452,26 @@ func testStorageNode(ctx context.Context, waddr address.Address, act address.Add
 	if err != nil {
 		panic(err)
 	}
-	enc, err := actors.SerializeParams(&saminer.ChangePeerIDParams{NewID: abi.PeerID(peerid)})
+	enc, err := actors.SerializeParams(&miner.ChangePeerIDParams{NewID: abi.PeerID(peerid)})
 	if err != nil {
 		panic(err)
 	}
 	msg := &types.Message{
-		To:       act,
-		From:     waddr,
-		Method:   builtin.MethodsMiner.ChangePeerID,
-		Params:   enc,
-		Value:    types.NewInt(0),
-		GasPrice: types.NewInt(0),
-		GasLimit: 1000000,
+		To:     act,
+		From:   waddr,
+		Method: builtin.MethodsMiner.ChangePeerID,
+		Params: enc,
+		Value:  types.NewInt(0),
 	}
 
-	_, err = tnd.MpoolPushMessage(ctx, msg)
+	_, err = tnd.MpoolPushMessage(ctx, msg, nil)
 	if err != nil {
 		panic(err)
 	}
 	// start node
 	var minerapi api.StorageMiner
 
-	mineBlock := make(chan func(bool))
+	mineBlock := make(chan miner2.MineReq)
 	// TODO: use stop
 	_, err = node.New(ctx,
 		node.StorageMiner(&minerapi),
@@ -426,7 +482,7 @@ func testStorageNode(ctx context.Context, waddr address.Address, act address.Add
 		node.MockHost(mn),
 
 		node.Override(new(api.FullNode), tnd),
-		node.Override(new(*miner.Miner), miner.NewTestMiner(mineBlock, act)),
+		node.Override(new(*miner2.Miner), miner2.NewTestMiner(mineBlock, act)),
 
 		opts,
 	)
@@ -440,9 +496,9 @@ func testStorageNode(ctx context.Context, waddr address.Address, act address.Add
 
 	err = minerapi.NetConnect(ctx, remoteAddrs)
 	require.NoError(t, err)*/
-	mineOne := func(ctx context.Context, cb func(bool)) error {
+	mineOne := func(ctx context.Context, req miner2.MineReq) error {
 		select {
-		case mineBlock <- cb:
+		case mineBlock <- req:
 			return nil
 		case <-ctx.Done():
 			return ctx.Err()