diff --git a/.gitignore b/.gitignore index 8e5b4964..9242cbc0 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ __debug_bin # binaries /hotstuff /plot - +**/__debug_bin* # other *.in @@ -35,3 +35,5 @@ measurements.json *.pdf twins.json + +/debug_logs diff --git a/.vscode/launch.json b/.vscode/launch.json index 3674460f..7ebcc9e8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,7 +12,7 @@ "program": "${workspaceRoot}/cmd/hotstuff/main.go", "cwd": "${workspaceRoot}", "args": [ - "run" + "run", ] }, { @@ -55,4 +55,4 @@ "host": "127.0.0.1" } ] -} \ No newline at end of file +} diff --git a/README.md b/README.md index 88da3366..59506406 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +# NOTICE + +This implementation was done as part of Alan Rostem's master's thesis. + # hotstuff [![Go Reference](https://pkg.go.dev/badge/github.com/relab/consensus.svg)](https://pkg.go.dev/github.com/relab/hotstuff) diff --git a/backend/backend_test.go b/backend/backend_test.go index 95b6a4c9..0b73b278 100644 --- a/backend/backend_test.go +++ b/backend/backend_test.go @@ -46,8 +46,33 @@ func TestConnect(t *testing.T) { runBoth(t, run) } +// Mainly test initialization of scoped modules and how they depend on each other. +func TestConnectScoped(t *testing.T) { + run := func(t *testing.T, setup setupFunc) { + const n = 4 + ctrl := gomock.NewController(t) + td := setup(t, ctrl, n) + builder := modules.NewBuilder(1, td.keys[0]) + testutil.TestModulesScoped(t, ctrl, 1, td.keys[0], &builder, 4) + teardown := createServers(t, td, ctrl) + defer teardown() + td.builders.Build() + + cfg := NewConfig(td.creds, gorums.WithDialTimeout(time.Second)) + + builder.Add(cfg) + builder.Build() + + err := cfg.Connect(td.replicas) + if err != nil { + t.Error(err) + } + } + runBoth(t, run) +} + // testBase is a generic test for a unicast/multicast call -func testBase(t *testing.T, typ any, send func(modules.Configuration), handle eventloop.EventHandler) { +func testBase(t *testing.T, typ any, send func(modules.Configuration), handle eventloop.EventHandler, opts ...eventloop.HandlerOption) { run := func(t *testing.T, setup setupFunc) { const n = 4 ctrl := gomock.NewController(t) @@ -69,11 +94,11 @@ func testBase(t *testing.T, typ any, send func(modules.Configuration), handle ev ctx, cancel := context.WithCancel(context.Background()) for _, hs := range hl[1:] { var ( - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop synchronizer modules.Synchronizer ) hs.Get(&eventLoop, &synchronizer) - eventLoop.RegisterHandler(typ, handle) + eventLoop.RegisterHandler(typ, handle, opts...) synchronizer.Start(ctx) go eventLoop.Run(ctx) } @@ -89,10 +114,16 @@ func TestPropose(t *testing.T) { ID: 1, Block: hotstuff.NewBlock( hotstuff.GetGenesis().Hash(), - hotstuff.NewQuorumCert(nil, 0, hotstuff.GetGenesis().Hash()), - "foo", 1, 1, + hotstuff.NewQuorumCert( + nil, + 0, + hotstuff.NullPipe, // TODO: Verify if this code conflicts with pipelining + hotstuff.GetGenesis().Hash()), + "foo", 1, 1, 0, ), + Pipe: 0, } + testBase(t, want, func(cfg modules.Configuration) { wg.Add(3) cfg.Propose(want) @@ -109,13 +140,47 @@ func TestPropose(t *testing.T) { }) } +func TestProposeScoped(t *testing.T) { + var wg sync.WaitGroup + pipe := hotstuff.Pipe(123) + want := hotstuff.ProposeMsg{ + ID: 1, + Block: hotstuff.NewBlock( + hotstuff.GetGenesis().Hash(), + hotstuff.NewQuorumCert( + nil, + 0, + hotstuff.NullPipe, // TODO: Verify if this code conflicts with pipelining + hotstuff.GetGenesis().Hash()), + "foo", 1, 1, pipe, + ), + Pipe: pipe, + } + + testBase(t, want, func(cfg modules.Configuration) { + wg.Add(3) + cfg.Propose(want) + wg.Wait() + }, func(event any) { + got := event.(hotstuff.ProposeMsg) + if got.ID != want.ID { + t.Errorf("wrong id in proposal: got: %d, want: %d", got.ID, want.ID) + } + if got.Block.Hash() != want.Block.Hash() { + + t.Errorf("block hashes do not match. want %d got %d", got.Block.Pipe(), want.Block.Pipe()) + } + wg.Done() + }, eventloop.RespondToScope(pipe)) +} + func TestTimeout(t *testing.T) { var wg sync.WaitGroup want := hotstuff.TimeoutMsg{ ID: 1, View: 1, ViewSignature: nil, - SyncInfo: hotstuff.NewSyncInfo(), + SyncInfo: hotstuff.NewSyncInfo(hotstuff.NullPipe), } testBase(t, want, func(cfg modules.Configuration) { wg.Add(3) @@ -133,6 +198,33 @@ func TestTimeout(t *testing.T) { }) } +func TestTimeoutScoped(t *testing.T) { + var wg sync.WaitGroup + + pipe := hotstuff.Pipe(1) + want := hotstuff.TimeoutMsg{ + ID: 1, + View: 1, + ViewSignature: nil, + SyncInfo: hotstuff.NewSyncInfo(pipe), + Pipe: pipe, + } + testBase(t, want, func(cfg modules.Configuration) { + wg.Add(3) + cfg.Timeout(want) + wg.Wait() + }, func(event any) { + got := event.(hotstuff.TimeoutMsg) + if got.ID != want.ID { + t.Errorf("wrong id in proposal: got: %d, want: %d", got.ID, want.ID) + } + if got.View != want.View { + t.Errorf("wrong view in proposal: got: %d, want: %d", got.View, want.View) + } + wg.Done() + }, eventloop.RespondToScope(pipe)) +} + type testData struct { n int creds credentials.TransportCredentials diff --git a/backend/config.go b/backend/config.go index e02fbb60..e963655c 100644 --- a/backend/config.go +++ b/backend/config.go @@ -24,7 +24,7 @@ import ( // Replica provides methods used by hotstuff to send messages to replicas. type Replica struct { - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop node *hotstuffpb.Node id hotstuff.ID pubKey hotstuff.PublicKey @@ -46,7 +46,7 @@ func (r *Replica) Vote(cert hotstuff.PartialCert) { if r.node == nil { return } - ctx, cancel := synchronizer.TimeoutContext(r.eventLoop.Context(), r.eventLoop) + ctx, cancel := synchronizer.ScopedTimeoutContext(r.eventLoop.Context(), r.eventLoop, cert.Pipe()) defer cancel() pCert := hotstuffpb.PartialCertToProto(cert) r.node.Vote(ctx, pCert) @@ -57,7 +57,7 @@ func (r *Replica) NewView(msg hotstuff.SyncInfo) { if r.node == nil { return } - ctx, cancel := synchronizer.TimeoutContext(r.eventLoop.Context(), r.eventLoop) + ctx, cancel := synchronizer.ScopedTimeoutContext(r.eventLoop.Context(), r.eventLoop, msg.Pipe()) defer cancel() r.node.NewView(ctx, hotstuffpb.SyncInfoToProto(msg)) } @@ -78,7 +78,7 @@ type Config struct { } type subConfig struct { - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop logger logging.Logger opts *modules.Options @@ -87,7 +87,7 @@ type subConfig struct { } // InitModule initializes the configuration. -func (cfg *Config) InitModule(mods *modules.Core) { +func (cfg *Config) InitModule(mods *modules.Core, _ modules.ScopeInfo) { mods.Get( &cfg.eventLoop, &cfg.logger, @@ -287,7 +287,7 @@ func (cfg *subConfig) Propose(proposal hotstuff.ProposeMsg) { if cfg.cfg == nil { return } - ctx, cancel := synchronizer.TimeoutContext(cfg.eventLoop.Context(), cfg.eventLoop) + ctx, cancel := synchronizer.ScopedTimeoutContext(cfg.eventLoop.Context(), cfg.eventLoop, proposal.Pipe) defer cancel() cfg.cfg.Propose( ctx, @@ -302,7 +302,7 @@ func (cfg *subConfig) Timeout(msg hotstuff.TimeoutMsg) { } // will wait until the second timeout before canceling - ctx, cancel := synchronizer.TimeoutContext(cfg.eventLoop.Context(), cfg.eventLoop) + ctx, cancel := synchronizer.ScopedTimeoutContext(cfg.eventLoop.Context(), cfg.eventLoop, msg.Pipe) defer cancel() cfg.cfg.Timeout( diff --git a/backend/server.go b/backend/server.go index 8cbeb4ba..dab05b97 100644 --- a/backend/server.go +++ b/backend/server.go @@ -22,20 +22,21 @@ import ( ) // Server is the Server-side of the gorums backend. -// It is responsible for calling handler methods on the consensus instance. +// It is responsible for calling handler methods on the pipe. type Server struct { blockChain modules.BlockChain configuration modules.Configuration - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop logger logging.Logger location string locationInfo map[hotstuff.ID]string latencyMatrix map[string]time.Duration + hackyLatency time.Duration gorumsSrv *gorums.Server } // InitModule initializes the Server. -func (srv *Server) InitModule(mods *modules.Core) { +func (srv *Server) InitModule(mods *modules.Core, _ modules.ScopeInfo) { mods.Get( &srv.eventLoop, &srv.configuration, @@ -74,6 +75,26 @@ func (srv *Server) induceLatency(sender hotstuff.ID) { <-timer1.C } +func (srv *Server) SetHackyLatency(amount time.Duration) { + srv.hackyLatency = amount +} + +// Alan: Hacky version of induceLatency which just blocks execution for an arbitrary time. +func (srv *Server) induceLatencyHacky(sender hotstuff.ID, callback func()) { + if srv.hackyLatency == 0 { + callback() + return + } + go func() { + senderLatency := srv.hackyLatency + senderLocation := srv.locationInfo[sender] + srv.logger.Debugf("latency from server %s to server %s is %s\n", srv.location, senderLocation, senderLatency) + timer1 := time.NewTimer(senderLatency) + <-timer1.C + callback() + }() +} + // GetGorumsServer returns the underlying gorums Server. func (srv *Server) GetGorumsServer() *gorums.Server { return srv.gorumsSrv @@ -162,8 +183,14 @@ func (impl *serviceImpl) Propose(ctx gorums.ServerCtx, proposal *hotstuffpb.Prop proposal.Block.Proposer = uint32(id) proposeMsg := hotstuffpb.ProposalFromProto(proposal) proposeMsg.ID = id - impl.srv.induceLatency(id) - impl.srv.eventLoop.AddEvent(proposeMsg) + impl.srv.induceLatencyHacky(id, func() { + if proposeMsg.Pipe.IsNull() { + impl.srv.eventLoop.AddScopedEvent(proposeMsg.Pipe, proposeMsg) + return + } + + impl.srv.eventLoop.AddEvent(proposeMsg) + }) } // Vote handles an incoming vote message. @@ -173,10 +200,20 @@ func (impl *serviceImpl) Vote(ctx gorums.ServerCtx, cert *hotstuffpb.PartialCert impl.srv.logger.Infof("Failed to get client ID: %v", err) return } - impl.srv.induceLatency(id) - impl.srv.eventLoop.AddEvent(hotstuff.VoteMsg{ - ID: id, - PartialCert: hotstuffpb.PartialCertFromProto(cert), + impl.srv.induceLatencyHacky(id, func() { + pipe := hotstuff.Pipe(cert.Pipe) + if pipe.IsNull() { + impl.srv.eventLoop.AddScopedEvent(pipe, hotstuff.VoteMsg{ + ID: id, + PartialCert: hotstuffpb.PartialCertFromProto(cert), + }) + return + } + + impl.srv.eventLoop.AddEvent(hotstuff.VoteMsg{ + ID: id, + PartialCert: hotstuffpb.PartialCertFromProto(cert), + }) }) } @@ -187,10 +224,20 @@ func (impl *serviceImpl) NewView(ctx gorums.ServerCtx, msg *hotstuffpb.SyncInfo) impl.srv.logger.Infof("Failed to get client ID: %v", err) return } - impl.srv.induceLatency(id) - impl.srv.eventLoop.AddEvent(hotstuff.NewViewMsg{ - ID: id, - SyncInfo: hotstuffpb.SyncInfoFromProto(msg), + impl.srv.induceLatencyHacky(id, func() { + pipe := hotstuff.Pipe(msg.Pipe) + if pipe.IsNull() { + impl.srv.eventLoop.AddScopedEvent(pipe, hotstuff.NewViewMsg{ + ID: id, + SyncInfo: hotstuffpb.SyncInfoFromProto(msg), + }) + return + } + + impl.srv.eventLoop.AddEvent(hotstuff.NewViewMsg{ + ID: id, + SyncInfo: hotstuffpb.SyncInfoFromProto(msg), + }) }) } @@ -217,8 +264,15 @@ func (impl *serviceImpl) Timeout(ctx gorums.ServerCtx, msg *hotstuffpb.TimeoutMs if err != nil { impl.srv.logger.Infof("Could not get ID of replica: %v", err) } - impl.srv.induceLatency(timeoutMsg.ID) - impl.srv.eventLoop.AddEvent(timeoutMsg) + + impl.srv.induceLatencyHacky(timeoutMsg.ID, func() { + if timeoutMsg.Pipe.IsNull() { + impl.srv.eventLoop.AddScopedEvent(timeoutMsg.Pipe, timeoutMsg) + return + } + + impl.srv.eventLoop.AddEvent(timeoutMsg) + }) } type replicaConnected struct { diff --git a/block.go b/block.go index 062c9beb..55d04854 100644 --- a/block.go +++ b/block.go @@ -15,16 +15,18 @@ type Block struct { cmd Command cert QuorumCert view View + pipe Pipe } // NewBlock creates a new Block -func NewBlock(parent Hash, cert QuorumCert, cmd Command, view View, proposer ID) *Block { +func NewBlock(parent Hash, cert QuorumCert, cmd Command, view View, proposer ID, pipe Pipe) *Block { b := &Block{ parent: parent, cert: cert, cmd: cmd, view: view, proposer: proposer, + pipe: pipe, } // cache the hash immediately because it is too racy to do it in Hash() b.hash = sha256.Sum256(b.ToBytes()) @@ -33,12 +35,13 @@ func NewBlock(parent Hash, cert QuorumCert, cmd Command, view View, proposer ID) func (b *Block) String() string { return fmt.Sprintf( - "Block{ hash: %.6s parent: %.6s, proposer: %d, view: %d , cert: %v }", + "Block{ hash: %.6s parent: %.6s, proposer: %d, view: %d , cert: %v, pipe: %d }", b.Hash().String(), b.parent.String(), b.proposer, b.view, b.cert, + b.pipe, ) } @@ -72,12 +75,19 @@ func (b *Block) View() View { return b.view } +func (b *Block) Pipe() Pipe { + return b.pipe +} + // ToBytes returns the raw byte form of the Block, to be used for hashing, etc. func (b *Block) ToBytes() []byte { buf := b.parent[:] var proposerBuf [4]byte binary.LittleEndian.PutUint32(proposerBuf[:], uint32(b.proposer)) buf = append(buf, proposerBuf[:]...) + var pipeBuf [4]byte + binary.LittleEndian.PutUint32(pipeBuf[:], uint32(b.pipe)) + buf = append(buf, pipeBuf[:]...) var viewBuf [8]byte binary.LittleEndian.PutUint64(viewBuf[:], uint64(b.view)) buf = append(buf, viewBuf[:]...) diff --git a/blockchain/blockchain.go b/blockchain/blockchain.go index 9fad8241..e171cc39 100644 --- a/blockchain/blockchain.go +++ b/blockchain/blockchain.go @@ -3,6 +3,7 @@ package blockchain import ( "context" + "fmt" "sync" "github.com/relab/hotstuff" @@ -16,21 +17,20 @@ import ( // blocks are evicted in LRU order. type blockChain struct { configuration modules.Configuration - consensus modules.Consensus - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop logger logging.Logger - mut sync.Mutex - pruneHeight hotstuff.View - blocks map[hotstuff.Hash]*hotstuff.Block - blockAtHeight map[hotstuff.View]*hotstuff.Block - pendingFetch map[hotstuff.Hash]context.CancelFunc // allows a pending fetch operation to be canceled + mut sync.Mutex + pruneHeight hotstuff.View + blocks map[hotstuff.Hash]*hotstuff.Block + // blocksAtHeight map[hotstuff.View][]*hotstuff.Block + blocksAtHeight map[hotstuff.View][]*hotstuff.Block + pendingFetch map[hotstuff.Hash]context.CancelFunc // allows a pending fetch operation to be canceled } -func (chain *blockChain) InitModule(mods *modules.Core) { +func (chain *blockChain) InitModule(mods *modules.Core, _ modules.ScopeInfo) { mods.Get( &chain.configuration, - &chain.consensus, &chain.eventLoop, &chain.logger, ) @@ -40,9 +40,9 @@ func (chain *blockChain) InitModule(mods *modules.Core) { // Blocks are dropped in least recently used order. func New() modules.BlockChain { bc := &blockChain{ - blocks: make(map[hotstuff.Hash]*hotstuff.Block), - blockAtHeight: make(map[hotstuff.View]*hotstuff.Block), - pendingFetch: make(map[hotstuff.Hash]context.CancelFunc), + blocks: make(map[hotstuff.Hash]*hotstuff.Block), + blocksAtHeight: make(map[hotstuff.View][]*hotstuff.Block), + pendingFetch: make(map[hotstuff.Hash]context.CancelFunc), } bc.Store(hotstuff.GetGenesis()) return bc @@ -54,7 +54,7 @@ func (chain *blockChain) Store(block *hotstuff.Block) { defer chain.mut.Unlock() chain.blocks[block.Hash()] = block - chain.blockAtHeight[block.View()] = block + chain.blocksAtHeight[block.View()] = append(chain.blocksAtHeight[block.View()], block) // cancel any pending fetch operations if cancel, ok := chain.pendingFetch[block.Hash()]; ok { @@ -75,9 +75,28 @@ func (chain *blockChain) LocalGet(hash hotstuff.Hash) (*hotstuff.Block, bool) { return block, true } +func (chain *blockChain) DeleteAtHeight(height hotstuff.View, blockHash hotstuff.Hash) error { + blocks, ok := chain.blocksAtHeight[height] + if !ok { + return fmt.Errorf("no blocks at height %d", height) + } + + strHash := blockHash.String() + for i, block := range blocks { + if block.Hash().String() == strHash { + chain.blocksAtHeight[height] = append(chain.blocksAtHeight[height][:i], chain.blocksAtHeight[height][i+1:]...) + if len(blocks) == 0 { + delete(chain.blocksAtHeight, height) + } + return nil + } + } + return fmt.Errorf("block not found at height %d", height) +} + // Get retrieves a block given its hash. Get will try to find the block locally. // If it is not available locally, it will try to fetch the block. -func (chain *blockChain) Get(hash hotstuff.Hash) (block *hotstuff.Block, ok bool) { +func (chain *blockChain) Get(hash hotstuff.Hash, pipe hotstuff.Pipe) (block *hotstuff.Block, ok bool) { // need to declare vars early, or else we won't be able to use goto var ( ctx context.Context @@ -90,7 +109,7 @@ func (chain *blockChain) Get(hash hotstuff.Hash) (block *hotstuff.Block, ok bool goto done } - ctx, cancel = synchronizer.TimeoutContext(chain.eventLoop.Context(), chain.eventLoop) + ctx, cancel = synchronizer.ScopedTimeoutContext(chain.eventLoop.Context(), chain.eventLoop, pipe) chain.pendingFetch[hash] = cancel chain.mut.Unlock() @@ -108,7 +127,7 @@ func (chain *blockChain) Get(hash hotstuff.Hash) (block *hotstuff.Block, ok bool chain.logger.Debugf("Successfully fetched block: %.8s", hash) chain.blocks[hash] = block - chain.blockAtHeight[block.View()] = block + chain.blocksAtHeight[block.View()] = append(chain.blocksAtHeight[block.View()], block) done: chain.mut.Unlock() @@ -125,43 +144,36 @@ func (chain *blockChain) Extends(block, target *hotstuff.Block) bool { current := block ok := true for ok && current.View() > target.View() { - current, ok = chain.Get(current.Parent()) + current, ok = chain.Get(current.Parent(), block.Pipe()) } return ok && current.Hash() == target.Hash() } -func (chain *blockChain) PruneToHeight(height hotstuff.View) (forkedBlocks []*hotstuff.Block) { +func (chain *blockChain) PruneToHeight(height hotstuff.View) (prunedBlocks map[hotstuff.View][]*hotstuff.Block) { chain.mut.Lock() defer chain.mut.Unlock() + prunedBlocks = make(map[hotstuff.View][]*hotstuff.Block) - committedHeight := chain.consensus.CommittedBlock().View() - committedViews := make(map[hotstuff.View]bool) - committedViews[committedHeight] = true - for h := committedHeight; h >= chain.pruneHeight; { - block, ok := chain.blockAtHeight[h] + for h := height; h > chain.pruneHeight; h-- { + blocks, ok := chain.blocksAtHeight[h] if !ok { - break + continue } - parent, ok := chain.blocks[block.Parent()] - if !ok || parent.View() < chain.pruneHeight { - break + for _, block := range blocks { + // Add pruned blocks to list and go back a height + // prunedBlocks = append(prunedBlocks, block) + prunedBlocks[block.View()] = append(prunedBlocks[block.View()], block) } - h = parent.View() - committedViews[h] = true + delete(chain.blocksAtHeight, h) } - for h := height; h > chain.pruneHeight; h-- { - if !committedViews[h] { - block, ok := chain.blockAtHeight[h] - if ok { - chain.logger.Debugf("PruneToHeight: found forked block: %v", block) - forkedBlocks = append(forkedBlocks, block) - } - } - delete(chain.blockAtHeight, h) - } chain.pruneHeight = height - return forkedBlocks + + return +} + +func (chain *blockChain) PruneHeight() hotstuff.View { + return chain.pruneHeight } var _ modules.BlockChain = (*blockChain)(nil) diff --git a/client/client.go b/client/client.go index c46bf676..c561649d9 100644 --- a/client/client.go +++ b/client/client.go @@ -60,7 +60,7 @@ type Config struct { // Client is a hotstuff client. type Client struct { - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop logger logging.Logger opts *modules.Options @@ -80,7 +80,7 @@ type Client struct { } // InitModule initializes the client. -func (c *Client) InitModule(mods *modules.Core) { +func (c *Client) InitModule(mods *modules.Core, _ modules.ScopeInfo) { mods.Get( &c.eventLoop, &c.logger, diff --git a/cmd/plot/main.go b/cmd/plot/main.go index 01e74a61..c26422f5 100644 --- a/cmd/plot/main.go +++ b/cmd/plot/main.go @@ -37,7 +37,6 @@ func main() { if err != nil { log.Fatalln(err) } - latencyPlot := plotting.NewClientLatencyPlot() throughputPlot := plotting.NewThroughputPlot() throughputVSLatencyPlot := plotting.NewThroughputVSLatencyPlot() @@ -57,6 +56,7 @@ func main() { if err := throughputPlot.PlotAverage(*throughput, *interval); err != nil { log.Fatalln(err) } + } if *throughputVSLatency != "" { diff --git a/committer/basic.go b/committer/basic.go new file mode 100644 index 00000000..c7183fa6 --- /dev/null +++ b/committer/basic.go @@ -0,0 +1,126 @@ +package committer + +import ( + "fmt" + "sync" + + "github.com/relab/hotstuff" + "github.com/relab/hotstuff/logging" + "github.com/relab/hotstuff/modules" +) + +type basic struct { + consensus modules.Consensus + blockChain modules.BlockChain + executor modules.ExecutorExt + forkHandler modules.ForkHandlerExt + logger logging.Logger + + mut sync.Mutex + bExec *hotstuff.Block +} + +// Basic committer implements commit logic without pipelining. +func New() modules.Committer { + return &basic{ + bExec: hotstuff.GetGenesis(), + } +} + +func (bb *basic) InitModule(mods *modules.Core, info modules.ScopeInfo) { + if info.IsPipeliningEnabled { + panic("pipelining not supported for this module") + } + mods.Get( + &bb.executor, + &bb.blockChain, + &bb.forkHandler, + &bb.logger, + &bb.consensus, + ) +} + +// Stores the block before further execution. +func (bb *basic) Commit(block *hotstuff.Block) { + err := bb.commit(block) + if err != nil { + bb.logger.Warnf("failed to commit: %v", err) + return + } + + // prune the blockchain and handle forked blocks + prunedBlocks := bb.blockChain.PruneToHeight(block.View()) + forkedBlocks := bb.findForks(block.View(), prunedBlocks) + for _, blocks := range forkedBlocks { + bb.forkHandler.Fork(blocks) + } +} + +func (bb *basic) commit(block *hotstuff.Block) error { + bb.mut.Lock() + // can't recurse due to requiring the mutex, so we use a helper instead. + err := bb.commitInner(block) + bb.mut.Unlock() + return err +} + +// recursive helper for commit +func (bb *basic) commitInner(block *hotstuff.Block) error { + if bb.bExec.View() >= block.View() { + return nil + } + if parent, ok := bb.blockChain.Get(block.Parent(), block.Pipe()); ok { + err := bb.commitInner(parent) + if err != nil { + return err + } + } else { + return fmt.Errorf("failed to locate block: %s", block.Parent()) + } + bb.logger.Debug("EXEC: ", block) + bb.executor.Exec(block) + bb.bExec = block + return nil +} + +// Retrieve the last block which was committed on a pipe. Use zero if pipelining is not used. +func (bb *basic) CommittedBlock(_ hotstuff.Pipe) *hotstuff.Block { + bb.mut.Lock() + defer bb.mut.Unlock() + return bb.bExec +} + +func (bb *basic) findForks(height hotstuff.View, blocksAtHeight map[hotstuff.View][]*hotstuff.Block) (forkedBlocks []*hotstuff.Block) { + + committedViews := make(map[hotstuff.View]bool) + committedHeight := bb.consensus.CommittedBlock().View() + + // This is a hacky value: chain.prevPruneHeight, but it works. + for h := committedHeight; h >= bb.blockChain.PruneHeight(); { + blocks, ok := blocksAtHeight[h] + if !ok { + break + } + block := blocks[0] + parent, ok := bb.blockChain.LocalGet(block.Parent()) + if !ok || parent.View() < bb.blockChain.PruneHeight() { + break + } + h = parent.View() + committedViews[h] = true + } + + for h := height; h > bb.blockChain.PruneHeight(); h-- { + if !committedViews[h] { + blocks, ok := blocksAtHeight[h] + if ok { + bb.logger.Debugf("PruneToHeight: found forked blocks: %v", blocks) + block := blocks[0] + forkedBlocks = append(forkedBlocks, block) + } + } + } + return +} + +var _ modules.Committer = (*basic)(nil) diff --git a/committer/piped.go b/committer/piped.go new file mode 100644 index 00000000..3dfc9da4 --- /dev/null +++ b/committer/piped.go @@ -0,0 +1,170 @@ +package committer + +import ( + "fmt" + "sync" + + "github.com/relab/hotstuff" + "github.com/relab/hotstuff/debug" + "github.com/relab/hotstuff/eventloop" + "github.com/relab/hotstuff/logging" + "github.com/relab/hotstuff/modules" +) + +func init() { + modules.RegisterModule("sequential", NewSequentialPiped) +} + +type sequentialPipedCommitter struct { + blockChain modules.BlockChain + eventLoop *eventloop.ScopedEventLoop + executor modules.ExecutorExt + forkHandler modules.ForkHandlerExt + logger logging.Logger + + mut sync.Mutex + bExecAtCi map[hotstuff.Pipe]*hotstuff.Block + waitingBlocksAtPipe map[hotstuff.Pipe][]*hotstuff.Block + pipeCount int + currentView hotstuff.View + currentPipe hotstuff.Pipe +} + +// pipedCommitter orders commits from multiple pipes by 1..n. +func NewSequentialPiped() modules.Committer { + return &sequentialPipedCommitter{ + bExecAtCi: make(map[hotstuff.Pipe]*hotstuff.Block), + waitingBlocksAtPipe: make(map[hotstuff.Pipe][]*hotstuff.Block), + currentView: 1, + currentPipe: 1, + } +} + +func (c *sequentialPipedCommitter) InitModule(mods *modules.Core, info modules.ScopeInfo) { + mods.Get( + &c.eventLoop, + &c.executor, + &c.blockChain, + &c.forkHandler, + &c.logger, + ) + + c.pipeCount = info.ScopeCount + if info.IsPipeliningEnabled { + for _, pipe := range mods.Scopes() { + c.bExecAtCi[pipe] = hotstuff.GetGenesis() + c.waitingBlocksAtPipe[pipe] = nil + } + return + } + + c.bExecAtCi[hotstuff.NullPipe] = hotstuff.GetGenesis() + c.waitingBlocksAtPipe[hotstuff.NullPipe] = nil +} + +// Stores the block before further execution. +func (c *sequentialPipedCommitter) Commit(block *hotstuff.Block) { + c.logger.Debugf("Commit (currentCi: %d, currentView: %d): new incoming block {p:%d, v:%d, h:%s}", + c.currentPipe, c.currentView, + block.Pipe(), block.View(), block.Hash().String()[:4]) + c.mut.Lock() + // can't recurse due to requiring the mutex, so we use a helper instead. + err := c.commitInner(block) + c.mut.Unlock() + + if err != nil { + c.logger.Debug("failed to commit block") + } + + c.mut.Lock() + err = c.tryExec() + c.mut.Unlock() + if err != nil { + c.logger.Debug(err) + } +} + +// Retrieve the last block which was committed on an pipe. Use zero if pipelining is not used. +func (c *sequentialPipedCommitter) CommittedBlock(pipe hotstuff.Pipe) *hotstuff.Block { + c.mut.Lock() + defer c.mut.Unlock() + return c.bExecAtCi[pipe] +} + +// recursive helper for commit +func (c *sequentialPipedCommitter) commitInner(block *hotstuff.Block) error { + if c.bExecAtCi[block.Pipe()].View() >= block.View() { + return nil + } + + // Check if the block was added to the end of the queue. If so, exit. + blockQueue := c.waitingBlocksAtPipe[block.Pipe()] + if len(blockQueue) > 0 && blockQueue[len(blockQueue)-1].View() >= block.View() { + return nil + } + + if parent, ok := c.blockChain.Get(block.Parent(), block.Pipe()); ok { + err := c.commitInner(parent) + if err != nil { + return err + } + } else { + return fmt.Errorf("failed to locate block: %s", block.Parent()) + } + + c.logger.Debugf("commitInner: Queued block: {p:%d, v:%d, h:%s}", block.Pipe(), block.View(), block.Hash().String()[:4]) + c.waitingBlocksAtPipe[block.Pipe()] = append(c.waitingBlocksAtPipe[block.Pipe()], block) + return nil +} + +func (c *sequentialPipedCommitter) handleForks(prunedBlocks map[hotstuff.View][]*hotstuff.Block) { + // All pruned blocks are assumed to be forks after the previous exec logic + for _, blocks := range prunedBlocks { + for _, block := range blocks { + c.forkHandler.Fork(block) + } + } +} + +func (c *sequentialPipedCommitter) tryExec() error { + waitingBlocks := c.waitingBlocksAtPipe[c.currentPipe] + canPeek := len(waitingBlocks) > 0 + if !canPeek { + c.logger.Debugf("tryExec (currentCi: %d, currentView: %d): no block on pipe yet", c.currentPipe, c.currentView) + return nil + } + + peekedBlock := waitingBlocks[0] + if peekedBlock.View() == c.currentView { + // Execute block + c.logger.Debugf("tryExec: block executed: {p=%d, v=%d, h:%s}", peekedBlock.Pipe(), peekedBlock.View(), peekedBlock.Hash().String()[:4]) + c.executor.Exec(peekedBlock) + c.bExecAtCi[peekedBlock.Pipe()] = peekedBlock + // Pop from queue + c.waitingBlocksAtPipe[c.currentPipe] = c.waitingBlocksAtPipe[c.currentPipe][1:] + // Delete from chain. + err := c.blockChain.DeleteAtHeight(peekedBlock.View(), peekedBlock.Hash()) + if err != nil { + return err + } + } else { + c.logger.Debugf("tryExec (currentCi: %d, currentView: %d): block in queue does not match view: {p:%d, v:%d, h:%s}", + c.currentPipe, c.currentView, + peekedBlock.Pipe(), peekedBlock.View(), peekedBlock.Hash().String()[:4]) + c.eventLoop.DebugEvent(debug.CommitHaltEvent{Pipe: c.currentPipe}) + } + + c.currentPipe++ + if c.currentPipe == hotstuff.Pipe(c.pipeCount)+1 { + c.currentPipe = 1 + // Prune out remaining blocks in the chain. Those blocks are guaranteed to be forks. + prunedBlocks := c.blockChain.PruneToHeight(c.currentView) + c.handleForks(prunedBlocks) + c.currentView++ + c.logger.Debugf("tryExec (currentCi: %d): advance to view %d", c.currentPipe, c.currentView) + } + + return c.tryExec() +} + +var _ modules.Committer = (*sequentialPipedCommitter)(nil) diff --git a/consensus/byzantine/byzantine.go b/consensus/byzantine/byzantine.go index 4fddbcc1..6dcc2349 100644 --- a/consensus/byzantine/byzantine.go +++ b/consensus/byzantine/byzantine.go @@ -22,9 +22,9 @@ type silence struct { consensus.Rules } -func (s *silence) InitModule(mods *modules.Core) { +func (s *silence) InitModule(mods *modules.Core, info modules.ScopeInfo) { if mod, ok := s.Rules.(modules.Module); ok { - mod.InitModule(mods) + mod.InitModule(mods, info) } } @@ -49,7 +49,7 @@ type fork struct { consensus.Rules } -func (f *fork) InitModule(mods *modules.Core) { +func (f *fork) InitModule(mods *modules.Core, info modules.ScopeInfo) { mods.Get( &f.blockChain, &f.synchronizer, @@ -57,20 +57,20 @@ func (f *fork) InitModule(mods *modules.Core) { ) if mod, ok := f.Rules.(modules.Module); ok { - mod.InitModule(mods) + mod.InitModule(mods, info) } } func (f *fork) ProposeRule(cert hotstuff.SyncInfo, cmd hotstuff.Command) (proposal hotstuff.ProposeMsg, ok bool) { - block, ok := f.blockChain.Get(f.synchronizer.HighQC().BlockHash()) + block, ok := f.blockChain.Get(f.synchronizer.HighQC().BlockHash(), cert.Pipe()) if !ok { return proposal, false } - parent, ok := f.blockChain.Get(block.Parent()) + parent, ok := f.blockChain.Get(block.Parent(), cert.Pipe()) if !ok { return proposal, false } - grandparent, ok := f.blockChain.Get(parent.Hash()) + grandparent, ok := f.blockChain.Get(parent.Hash(), cert.Pipe()) if !ok { return proposal, false } @@ -83,6 +83,7 @@ func (f *fork) ProposeRule(cert hotstuff.SyncInfo, cmd hotstuff.Command) (propos cmd, f.synchronizer.View(), f.opts.ID(), + 0, ), } if aggQC, ok := cert.AggQC(); f.opts.ShouldUseAggQC() && ok { diff --git a/consensus/chainedhotstuff/chainedhotstuff.go b/consensus/chainedhotstuff/chainedhotstuff.go index d5b34e62..a25efb29 100644 --- a/consensus/chainedhotstuff/chainedhotstuff.go +++ b/consensus/chainedhotstuff/chainedhotstuff.go @@ -16,6 +16,7 @@ func init() { type ChainedHotStuff struct { blockChain modules.BlockChain logger logging.Logger + pipe hotstuff.Pipe // protocol variables @@ -30,7 +31,8 @@ func New() consensus.Rules { } // InitModule initializes the module. -func (hs *ChainedHotStuff) InitModule(mods *modules.Core) { +func (hs *ChainedHotStuff) InitModule(mods *modules.Core, initinfo modules.ScopeInfo) { + hs.pipe = initinfo.ModuleScope mods.Get(&hs.blockChain, &hs.logger) } @@ -38,11 +40,15 @@ func (hs *ChainedHotStuff) qcRef(qc hotstuff.QuorumCert) (*hotstuff.Block, bool) if (hotstuff.Hash{}) == qc.BlockHash() { return nil, false } - return hs.blockChain.Get(qc.BlockHash()) + return hs.blockChain.Get(qc.BlockHash(), qc.Pipe()) } // CommitRule decides whether an ancestor of the block should be committed. func (hs *ChainedHotStuff) CommitRule(block *hotstuff.Block) *hotstuff.Block { + if hs.pipe != block.Pipe() { + panic("incorrect pipe") + } + block1, ok := hs.qcRef(block.QuorumCert()) if !ok { return nil @@ -50,7 +56,7 @@ func (hs *ChainedHotStuff) CommitRule(block *hotstuff.Block) *hotstuff.Block { // Note that we do not call UpdateHighQC here. // This is done through AdvanceView, which the Consensus implementation will call. - hs.logger.Debug("PRE_COMMIT: ", block1) + hs.logger.Debugf("PRE_COMMIT[p=%d, view=%d]: %s", hs.pipe, hs.bLock.View(), block1) block2, ok := hs.qcRef(block1.QuorumCert()) if !ok { @@ -58,7 +64,7 @@ func (hs *ChainedHotStuff) CommitRule(block *hotstuff.Block) *hotstuff.Block { } if block2.View() > hs.bLock.View() { - hs.logger.Debug("COMMIT: ", block2) + hs.logger.Debugf("COMMIT[p=%d, view=%d]: %s", hs.pipe, hs.bLock.View(), block2) hs.bLock = block2 } @@ -68,7 +74,7 @@ func (hs *ChainedHotStuff) CommitRule(block *hotstuff.Block) *hotstuff.Block { } if block1.Parent() == block2.Hash() && block2.Parent() == block3.Hash() { - hs.logger.Debug("DECIDE: ", block3) + hs.logger.Debugf("DECIDE[p=%d, view=%d]: ", hs.pipe, hs.bLock.View(), block3) return block3 } @@ -77,9 +83,12 @@ func (hs *ChainedHotStuff) CommitRule(block *hotstuff.Block) *hotstuff.Block { // VoteRule decides whether to vote for the proposal or not. func (hs *ChainedHotStuff) VoteRule(proposal hotstuff.ProposeMsg) bool { + if hs.pipe != proposal.Pipe { + panic("incorrect pipe") + } block := proposal.Block - qcBlock, haveQCBlock := hs.blockChain.Get(block.QuorumCert().BlockHash()) + qcBlock, haveQCBlock := hs.blockChain.Get(block.QuorumCert().BlockHash(), block.Pipe()) safe := false if haveQCBlock && qcBlock.View() > hs.bLock.View() { diff --git a/consensus/consensus.go b/consensus/consensus.go index 518083df..19b4a019 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/relab/hotstuff" + "github.com/relab/hotstuff/debug" "github.com/relab/hotstuff/eventloop" "github.com/relab/hotstuff/logging" "github.com/relab/hotstuff/modules" @@ -40,11 +41,11 @@ type consensusBase struct { acceptor modules.Acceptor blockChain modules.BlockChain + committer modules.Committer commandQueue modules.CommandQueue configuration modules.Configuration crypto modules.Crypto - eventLoop *eventloop.EventLoop - executor modules.ExecutorExt + eventLoop *eventloop.ScopedEventLoop forkHandler modules.ForkHandlerExt leaderRotation modules.LeaderRotation logger logging.Logger @@ -54,81 +55,85 @@ type consensusBase struct { handel modules.Handel lastVote hotstuff.View + pipe hotstuff.Pipe - mut sync.Mutex - bExec *hotstuff.Block + mut sync.Mutex } // New returns a new Consensus instance based on the given Rules implementation. -func New(impl Rules) modules.Consensus { +func New() modules.Consensus { return &consensusBase{ - impl: impl, lastVote: 0, - bExec: hotstuff.GetGenesis(), } } // InitModule initializes the module. -func (cs *consensusBase) InitModule(mods *modules.Core) { - mods.Get( +func (cs *consensusBase) InitModule(mods *modules.Core, info modules.ScopeInfo) { + cs.pipe = info.ModuleScope + + mods.GetScoped(cs, &cs.acceptor, &cs.blockChain, &cs.commandQueue, + &cs.committer, &cs.configuration, &cs.crypto, &cs.eventLoop, - &cs.executor, &cs.forkHandler, &cs.leaderRotation, &cs.logger, &cs.opts, + &cs.impl, &cs.synchronizer, ) mods.TryGet(&cs.handel) - if mod, ok := cs.impl.(modules.Module); ok { - mod.InitModule(mods) - } + // if mod, ok := cs.impl.(modules.Module); ok { + // mod.InitModule(mods, initOpt) + // } cs.eventLoop.RegisterHandler(hotstuff.ProposeMsg{}, func(event any) { cs.OnPropose(event.(hotstuff.ProposeMsg)) - }) + }, eventloop.RespondToScope(info.ModuleScope)) } func (cs *consensusBase) CommittedBlock() *hotstuff.Block { - cs.mut.Lock() - defer cs.mut.Unlock() - return cs.bExec + return cs.committer.CommittedBlock(cs.pipe) } // StopVoting ensures that no voting happens in a view earlier than `view`. func (cs *consensusBase) StopVoting(view hotstuff.View) { if cs.lastVote < view { + cs.logger.Debugf("stopped voting on view %d and changed view to %d", cs.lastVote, view) cs.lastVote = view } } // Propose creates a new proposal. func (cs *consensusBase) Propose(cert hotstuff.SyncInfo) { - cs.logger.Debug("Propose") + cs.logger.Debugf("Propose[p=%d]", cs.pipe) + + if cs.pipe != cert.Pipe() { + panic("incorrect pipe") + } qc, ok := cert.QC() if ok { // tell the acceptor that the previous proposal succeeded. - if qcBlock, ok := cs.blockChain.Get(qc.BlockHash()); ok { + if qcBlock, ok := cs.blockChain.Get(qc.BlockHash(), cs.pipe); ok { cs.acceptor.Proposed(qcBlock.Command()) } else { cs.logger.Errorf("Could not find block for QC: %s", qc) } } - ctx, cancel := synchronizer.TimeoutContext(cs.eventLoop.Context(), cs.eventLoop) + ctx, cancel := synchronizer.ScopedTimeoutContext(cs.eventLoop.Context(), cs.eventLoop, cs.pipe) defer cancel() cmd, ok := cs.commandQueue.Get(ctx) if !ok { - cs.logger.Debug("Propose: No command") + cs.logger.Debugf("Propose[p=%d, view=%d]: No command", cs.pipe, cs.synchronizer.View()) return } @@ -136,7 +141,7 @@ func (cs *consensusBase) Propose(cert hotstuff.SyncInfo) { if proposer, ok := cs.impl.(ProposeRuler); ok { proposal, ok = proposer.ProposeRule(cert, cmd) if !ok { - cs.logger.Debug("Propose: No block") + cs.logger.Debug("Propose[p=%d]: No block", cs.pipe) return } } else { @@ -148,7 +153,9 @@ func (cs *consensusBase) Propose(cert hotstuff.SyncInfo) { cmd, cs.synchronizer.View(), cs.opts.ID(), + cs.pipe, ), + Pipe: cs.pipe, } if aggQC, ok := cert.AggQC(); ok && cs.opts.ShouldUseAggQC() { @@ -165,66 +172,73 @@ func (cs *consensusBase) Propose(cert hotstuff.SyncInfo) { func (cs *consensusBase) OnPropose(proposal hotstuff.ProposeMsg) { //nolint:gocyclo // TODO: extract parts of this method into helper functions maybe? - cs.logger.Debugf("OnPropose: %v", proposal.Block) + cs.logger.Debugf("OnPropose[p=%d, view=%d]: %.8s -> %.8x", cs.pipe, cs.synchronizer.View(), proposal.Block.Hash(), proposal.Block.Command()) + if cs.pipe != proposal.Pipe { + panic("OnPropose: incorrect pipe") + } block := proposal.Block if cs.opts.ShouldUseAggQC() && proposal.AggregateQC != nil { highQC, ok := cs.crypto.VerifyAggregateQC(*proposal.AggregateQC) if !ok { - cs.logger.Warn("OnPropose: failed to verify aggregate QC") + cs.logger.Warnf("OnPropose[p=%d, view=%d]: failed to verify aggregate QC", cs.pipe, cs.synchronizer.View()) return } // NOTE: for simplicity, we require that the highQC found in the AggregateQC equals the QC embedded in the block. if !block.QuorumCert().Equals(highQC) { - cs.logger.Warn("OnPropose: block QC does not equal highQC") + cs.logger.Warnf("OnPropose[p=%d, view=%d]: block QC does not equal highQC", cs.pipe, cs.synchronizer.View()) return } } if !cs.crypto.VerifyQuorumCert(block.QuorumCert()) { - cs.logger.Info("OnPropose: invalid QC") + cs.logger.Infof("OnPropose[p=%d, view=%d]: invalid QC", cs.pipe, cs.synchronizer.View()) return } // ensure the block came from the leader. if proposal.ID != cs.leaderRotation.GetLeader(block.View()) { - cs.logger.Info("OnPropose: block was not proposed by the expected leader") + cs.logger.Infof("OnPropose[p=%d, view=%d]: block was not proposed by the expected leader", cs.pipe, cs.synchronizer.View()) return } if !cs.impl.VoteRule(proposal) { - cs.logger.Info("OnPropose: Block not voted for") + cs.logger.Infof("OnPropose[p=%d, view=%d]: Block not voted for", cs.pipe, cs.synchronizer.View()) return } - if qcBlock, ok := cs.blockChain.Get(block.QuorumCert().BlockHash()); ok { + if qcBlock, ok := cs.blockChain.Get(block.QuorumCert().BlockHash(), cs.pipe); ok { cs.acceptor.Proposed(qcBlock.Command()) } else { - cs.logger.Info("OnPropose: Failed to fetch qcBlock") + cs.logger.Infof("OnPropose[p=%d, view=%d]: Failed to fetch qcBlock", cs.pipe, cs.synchronizer.View()) } - if !cs.acceptor.Accept(block.Command()) { - cs.logger.Info("OnPropose: command not accepted") + cmd := block.Command() + if !cs.acceptor.Accept(cmd) { + cs.logger.Infof("OnPropose[p=%d, view=%d]: block rejected: %.8s -> %.8x", cs.pipe, cs.synchronizer.View(), block.Hash(), block.Command()) + cs.eventLoop.DebugEvent(debug.CommandRejectedEvent{Pipe: cs.pipe, View: cs.synchronizer.View()}) return } + cs.logger.Debugf("OnPropose[p=%d, view=%d]: block accepted: %.8s -> %.8x", cs.pipe, cs.synchronizer.View(), block.Hash(), block.Command()) + // block is safe and was accepted cs.blockChain.Store(block) if b := cs.impl.CommitRule(block); b != nil { - cs.commit(b) + cs.committer.Commit(block) } - cs.synchronizer.AdvanceView(hotstuff.NewSyncInfo().WithQC(block.QuorumCert())) + cs.synchronizer.AdvanceView(hotstuff.NewSyncInfo(cs.pipe).WithQC(block.QuorumCert())) if block.View() <= cs.lastVote { - cs.logger.Info("OnPropose: block view too old") + cs.logger.Info(fmt.Sprintf("OnPropose[p=%d, view=%d]: block view too old for %.8s -> %.8x (diff=%d)", cs.pipe, cs.synchronizer.View(), block.Hash(), block.Command(), cs.lastVote-block.View())) return } pc, err := cs.crypto.CreatePartialCert(block) if err != nil { - cs.logger.Error("OnPropose: failed to sign block: ", err) + cs.logger.Errorf("OnPropose[p=%d, view=%d]: failed to sign block: ", cs.pipe, cs.synchronizer.View(), err) return } @@ -238,7 +252,7 @@ func (cs *consensusBase) OnPropose(proposal hotstuff.ProposeMsg) { //nolint:gocy leaderID := cs.leaderRotation.GetLeader(cs.lastVote + 1) if leaderID == cs.opts.ID() { - cs.eventLoop.AddEvent(hotstuff.VoteMsg{ID: cs.opts.ID(), PartialCert: pc}) + cs.eventLoop.AddScopedEvent(cs.pipe, hotstuff.VoteMsg{ID: cs.opts.ID(), PartialCert: pc}) return } @@ -248,46 +262,10 @@ func (cs *consensusBase) OnPropose(proposal hotstuff.ProposeMsg) { //nolint:gocy return } + cs.logger.Debugf("OnPropose[p=%d, view=%d]: voting for %.8s -> %.8x", cs.pipe, cs.synchronizer.View(), block.Hash(), block.Command()) leader.Vote(pc) } -func (cs *consensusBase) commit(block *hotstuff.Block) { - cs.mut.Lock() - // can't recurse due to requiring the mutex, so we use a helper instead. - err := cs.commitInner(block) - cs.mut.Unlock() - - if err != nil { - cs.logger.Warnf("failed to commit: %v", err) - return - } - - // prune the blockchain and handle forked blocks - forkedBlocks := cs.blockChain.PruneToHeight(block.View()) - for _, block := range forkedBlocks { - cs.forkHandler.Fork(block) - } -} - -// recursive helper for commit -func (cs *consensusBase) commitInner(block *hotstuff.Block) error { - if cs.bExec.View() >= block.View() { - return nil - } - if parent, ok := cs.blockChain.Get(block.Parent()); ok { - err := cs.commitInner(parent) - if err != nil { - return err - } - } else { - return fmt.Errorf("failed to locate block: %s", block.Parent()) - } - cs.logger.Debug("EXEC: ", block) - cs.executor.Exec(block) - cs.bExec = block - return nil -} - // ChainLength returns the number of blocks that need to be chained together in order to commit. func (cs *consensusBase) ChainLength() int { return cs.impl.ChainLength() diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go index f5e33f5c..2d7d6b75 100644 --- a/consensus/consensus_test.go +++ b/consensus/consensus_test.go @@ -20,18 +20,19 @@ func TestVote(t *testing.T) { ctrl := gomock.NewController(t) bl := testutil.CreateBuilders(t, ctrl, n) cs := mocks.NewMockConsensus(ctrl) - bl[0].Add(synchronizer.New(testutil.FixedTimeout(1*time.Millisecond)), cs) + d := testutil.FixedTimeout(1 * time.Millisecond) + bl[0].Add(synchronizer.New(), d, cs) hl := bl.Build() hs := hl[0] var ( - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop blockChain modules.BlockChain ) hs.Get(&eventLoop, &blockChain) - cs.EXPECT().Propose(gomock.AssignableToTypeOf(hotstuff.NewSyncInfo())) + cs.EXPECT().Propose(gomock.AssignableToTypeOf(hotstuff.NewSyncInfo(hotstuff.NullPipe))) ok := false ctx, cancel := context.WithCancel(context.Background()) @@ -42,7 +43,11 @@ func TestVote(t *testing.T) { b := testutil.NewProposeMsg( hotstuff.GetGenesis().Hash(), - hotstuff.NewQuorumCert(nil, 1, hotstuff.GetGenesis().Hash()), + hotstuff.NewQuorumCert( + nil, + 1, + hotstuff.NullPipe, // TODO: Verify if this code conflicts with pipelining + hotstuff.GetGenesis().Hash()), "test", 1, 1, ) blockChain.Store(b.Block) diff --git a/consensus/fasthotstuff/fasthotstuff.go b/consensus/fasthotstuff/fasthotstuff.go index 098362b8..5dbffac7 100644 --- a/consensus/fasthotstuff/fasthotstuff.go +++ b/consensus/fasthotstuff/fasthotstuff.go @@ -17,6 +17,8 @@ type FastHotStuff struct { blockChain modules.BlockChain logger logging.Logger synchronizer modules.Synchronizer + + pipe hotstuff.Pipe } // New returns a new FastHotStuff instance. @@ -25,10 +27,16 @@ func New() consensus.Rules { } // InitModule initializes the module. -func (fhs *FastHotStuff) InitModule(mods *modules.Core) { +func (fhs *FastHotStuff) InitModule(mods *modules.Core, info modules.ScopeInfo) { var opts *modules.Options - mods.Get(&opts, &fhs.blockChain, &fhs.logger, &fhs.synchronizer) + fhs.pipe = info.ModuleScope + + mods.GetScoped(fhs, + &fhs.blockChain, + &fhs.logger, + &opts, + &fhs.synchronizer) opts.SetShouldUseAggQC() } @@ -37,11 +45,15 @@ func (fhs *FastHotStuff) qcRef(qc hotstuff.QuorumCert) (*hotstuff.Block, bool) { if (hotstuff.Hash{}) == qc.BlockHash() { return nil, false } - return fhs.blockChain.Get(qc.BlockHash()) + return fhs.blockChain.Get(qc.BlockHash(), qc.Pipe()) } // CommitRule decides whether an ancestor of the block can be committed. func (fhs *FastHotStuff) CommitRule(block *hotstuff.Block) *hotstuff.Block { + if fhs.pipe != block.Pipe() { + panic("incorrect pipe") + } + parent, ok := fhs.qcRef(block.QuorumCert()) if !ok { return nil @@ -61,10 +73,13 @@ func (fhs *FastHotStuff) CommitRule(block *hotstuff.Block) *hotstuff.Block { // VoteRule decides whether to vote for the proposal or not. func (fhs *FastHotStuff) VoteRule(proposal hotstuff.ProposeMsg) bool { + if fhs.pipe != proposal.Pipe { + panic("incorrect consensus instance") + } // The base implementation verifies both regular QCs and AggregateQCs, and asserts that the QC embedded in the // block is the same as the highQC found in the aggregateQC. if proposal.AggregateQC != nil { - hqcBlock, ok := fhs.blockChain.Get(proposal.Block.QuorumCert().BlockHash()) + hqcBlock, ok := fhs.blockChain.Get(proposal.Block.QuorumCert().BlockHash(), proposal.Pipe) return ok && fhs.blockChain.Extends(proposal.Block, hqcBlock) } return proposal.Block.View() >= fhs.synchronizer.View() && diff --git a/consensus/simplehotstuff/simplehotstuff.go b/consensus/simplehotstuff/simplehotstuff.go index 3f6e973e..e119c50f 100644 --- a/consensus/simplehotstuff/simplehotstuff.go +++ b/consensus/simplehotstuff/simplehotstuff.go @@ -21,6 +21,7 @@ type SimpleHotStuff struct { logger logging.Logger synchronizer modules.Synchronizer + pipe hotstuff.Pipe locked *hotstuff.Block } @@ -32,12 +33,20 @@ func New() consensus.Rules { } // InitModule initializes the module. -func (hs *SimpleHotStuff) InitModule(mods *modules.Core) { - mods.Get(&hs.blockChain, &hs.logger, &hs.synchronizer) +func (hs *SimpleHotStuff) InitModule(mods *modules.Core, info modules.ScopeInfo) { + hs.pipe = info.ModuleScope + mods.GetScoped(hs, + &hs.blockChain, + &hs.logger, + &hs.synchronizer) } // VoteRule decides if the replica should vote for the given block. func (hs *SimpleHotStuff) VoteRule(proposal hotstuff.ProposeMsg) bool { + if proposal.Pipe != hs.pipe { + panic("incorrect pipe") + } + block := proposal.Block // Rule 1: can only vote in increasing rounds @@ -46,7 +55,7 @@ func (hs *SimpleHotStuff) VoteRule(proposal hotstuff.ProposeMsg) bool { return false } - parent, ok := hs.blockChain.Get(block.QuorumCert().BlockHash()) + parent, ok := hs.blockChain.Get(block.QuorumCert().BlockHash(), block.Pipe()) if !ok { hs.logger.Info("VoteRule: missing parent block: ", block.QuorumCert().BlockHash()) return false @@ -63,13 +72,16 @@ func (hs *SimpleHotStuff) VoteRule(proposal hotstuff.ProposeMsg) bool { // CommitRule decides if an ancestor of the block can be committed, and returns the ancestor, otherwise returns nil. func (hs *SimpleHotStuff) CommitRule(block *hotstuff.Block) *hotstuff.Block { + if block.Pipe() != hs.pipe { + panic("incorrect pipe") + } // will consider if the great-grandparent of the new block can be committed. - p, ok := hs.blockChain.Get(block.QuorumCert().BlockHash()) + p, ok := hs.blockChain.Get(block.QuorumCert().BlockHash(), block.Pipe()) if !ok { return nil } - gp, ok := hs.blockChain.Get(p.QuorumCert().BlockHash()) + gp, ok := hs.blockChain.Get(p.QuorumCert().BlockHash(), block.Pipe()) if ok && gp.View() > hs.locked.View() { hs.locked = gp hs.logger.Debug("Locked: ", gp) @@ -77,7 +89,7 @@ func (hs *SimpleHotStuff) CommitRule(block *hotstuff.Block) *hotstuff.Block { return nil } - ggp, ok := hs.blockChain.Get(gp.QuorumCert().BlockHash()) + ggp, ok := hs.blockChain.Get(gp.QuorumCert().BlockHash(), block.Pipe()) // we commit the great-grandparent of the block if its grandchild is certified, // which we already know is true because the new block contains the grandchild's certificate, // and if the great-grandparent's view + 2 equals the grandchild's view. diff --git a/consensus/votingmachine.go b/consensus/votingmachine.go index 7d576d82..d9781243 100644 --- a/consensus/votingmachine.go +++ b/consensus/votingmachine.go @@ -1,6 +1,7 @@ package consensus import ( + "fmt" "sync" "github.com/relab/hotstuff" @@ -14,11 +15,12 @@ type VotingMachine struct { blockChain modules.BlockChain configuration modules.Configuration crypto modules.Crypto - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop logger logging.Logger synchronizer modules.Synchronizer opts *modules.Options + pipe hotstuff.Pipe mut sync.Mutex verifiedVotes map[hotstuff.Hash][]hotstuff.PartialCert // verified votes that could become a QC } @@ -31,8 +33,8 @@ func NewVotingMachine() *VotingMachine { } // InitModule initializes the VotingMachine. -func (vm *VotingMachine) InitModule(mods *modules.Core) { - mods.Get( +func (vm *VotingMachine) InitModule(mods *modules.Core, info modules.ScopeInfo) { + mods.GetScoped(vm, &vm.blockChain, &vm.configuration, &vm.crypto, @@ -42,13 +44,20 @@ func (vm *VotingMachine) InitModule(mods *modules.Core) { &vm.opts, ) - vm.eventLoop.RegisterHandler(hotstuff.VoteMsg{}, func(event any) { vm.OnVote(event.(hotstuff.VoteMsg)) }) + vm.pipe = info.ModuleScope + vm.eventLoop.RegisterHandler(hotstuff.VoteMsg{}, func(event any) { + vm.OnVote(event.(hotstuff.VoteMsg)) + }, eventloop.RespondToScope(info.ModuleScope)) } // OnVote handles an incoming vote. func (vm *VotingMachine) OnVote(vote hotstuff.VoteMsg) { cert := vote.PartialCert - vm.logger.Debugf("OnVote(%d): %.8s", vote.ID, cert.BlockHash()) + if vm.pipe != cert.Pipe() { + panic("incorrect pipe") + } + + vm.logger.Debugf("OnVote[p=%d, view=%d](vote=%d): %.8s", vm.pipe, vm.synchronizer.View(), vote.ID, cert.BlockHash()) var ( block *hotstuff.Block @@ -61,16 +70,16 @@ func (vm *VotingMachine) OnVote(vote hotstuff.VoteMsg) { if !ok { // if that does not work, we will try to handle this event later. // hopefully, the block has arrived by then. - vm.logger.Debugf("Local cache miss for block: %.8s", cert.BlockHash()) + vm.logger.Debugf("Local cache miss for block [p=%d, view=%d]: %.8s", vm.pipe, vm.synchronizer.View(), cert.BlockHash()) vote.Deferred = true - vm.eventLoop.DelayUntil(hotstuff.ProposeMsg{}, vote) + vm.eventLoop.DelayScoped(vm.pipe, hotstuff.ProposeMsg{}, vote) return } } else { // if the block has not arrived at this point we will try to fetch it. - block, ok = vm.blockChain.Get(cert.BlockHash()) + block, ok = vm.blockChain.Get(cert.BlockHash(), cert.Pipe()) if !ok { - vm.logger.Debugf("Could not find block for vote: %.8s.", cert.BlockHash()) + vm.logger.Debugf("Could not find block for vote [p=%d, view=%d]", vm.pipe, vm.synchronizer.View()) return } } @@ -89,7 +98,7 @@ func (vm *VotingMachine) OnVote(vote hotstuff.VoteMsg) { func (vm *VotingMachine) verifyCert(cert hotstuff.PartialCert, block *hotstuff.Block) { if !vm.crypto.VerifyPartialCert(cert) { - vm.logger.Info("OnVote: Vote could not be verified!") + vm.logger.Infof("OnVote[p=%d, view=%d]: Vote could not be verified!", vm.pipe, vm.synchronizer.View()) return } @@ -120,10 +129,12 @@ func (vm *VotingMachine) verifyCert(cert hotstuff.PartialCert, block *hotstuff.B qc, err := vm.crypto.CreateQuorumCert(block, votes) if err != nil { - vm.logger.Info("OnVote: could not create QC for block: ", err) + vm.logger.Info(fmt.Sprintf("OnVote[p=%d, view=%d]: could not create QC for block: ", vm.pipe, vm.synchronizer.View()), err) return } delete(vm.verifiedVotes, cert.BlockHash()) - vm.eventLoop.AddEvent(hotstuff.NewViewMsg{ID: vm.opts.ID(), SyncInfo: hotstuff.NewSyncInfo().WithQC(qc)}) + vm.eventLoop.AddScopedEvent(vm.pipe, hotstuff.NewViewMsg{ + ID: vm.opts.ID(), + SyncInfo: hotstuff.NewSyncInfo(block.Pipe()).WithQC(qc)}) } diff --git a/crypto/bls12/bls12.go b/crypto/bls12/bls12.go index ffcc5793..c2c05179 100644 --- a/crypto/bls12/bls12.go +++ b/crypto/bls12/bls12.go @@ -158,7 +158,7 @@ func New() modules.CryptoBase { // InitModule gives the module a reference to the Core object. // It also allows the module to set module options using the OptionsBuilder. -func (bls *bls12Base) InitModule(mods *modules.Core) { +func (bls *bls12Base) InitModule(mods *modules.Core, _ modules.ScopeInfo) { mods.Get( &bls.configuration, &bls.logger, diff --git a/crypto/cache.go b/crypto/cache.go index c7d2df8b..4438423b 100644 --- a/crypto/cache.go +++ b/crypto/cache.go @@ -32,9 +32,9 @@ func NewCache(impl modules.CryptoBase, capacity int) modules.Crypto { // InitModule gives the module a reference to the Core object. // It also allows the module to set module options using the OptionsBuilder. -func (cache *cache) InitModule(mods *modules.Core) { +func (cache *cache) InitModule(mods *modules.Core, info modules.ScopeInfo) { if mod, ok := cache.impl.(modules.Module); ok { - mod.InitModule(mods) + mod.InitModule(mods, info) } } diff --git a/crypto/crypto.go b/crypto/crypto.go index 34ac67be..4b8e64f8 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -21,14 +21,14 @@ func New(impl modules.CryptoBase) modules.Crypto { // InitModule gives the module a reference to the Core object. // It also allows the module to set module options using the OptionsBuilder. -func (c *crypto) InitModule(mods *modules.Core) { +func (c *crypto) InitModule(mods *modules.Core, info modules.ScopeInfo) { mods.Get( &c.blockChain, &c.configuration, ) if mod, ok := c.CryptoBase.(modules.Module); ok { - mod.InitModule(mods) + mod.InitModule(mods, info) } } @@ -38,14 +38,14 @@ func (c crypto) CreatePartialCert(block *hotstuff.Block) (cert hotstuff.PartialC if err != nil { return hotstuff.PartialCert{}, err } - return hotstuff.NewPartialCert(sig, block.Hash()), nil + return hotstuff.NewPartialCert(block.Pipe(), sig, block.Hash()), nil } // CreateQuorumCert creates a quorum certificate from a list of partial certificates. func (c crypto) CreateQuorumCert(block *hotstuff.Block, signatures []hotstuff.PartialCert) (cert hotstuff.QuorumCert, err error) { // genesis QC is always valid. if block.Hash() == hotstuff.GetGenesis().Hash() { - return hotstuff.NewQuorumCert(nil, 0, hotstuff.GetGenesis().Hash()), nil + return hotstuff.NewQuorumCert(nil, 0, block.Pipe(), hotstuff.GetGenesis().Hash()), nil } sigs := make([]hotstuff.QuorumSignature, 0, len(signatures)) for _, sig := range signatures { @@ -55,7 +55,7 @@ func (c crypto) CreateQuorumCert(block *hotstuff.Block, signatures []hotstuff.Pa if err != nil { return hotstuff.QuorumCert{}, err } - return hotstuff.NewQuorumCert(sig, block.View(), block.Hash()), nil + return hotstuff.NewQuorumCert(sig, block.View(), block.Pipe(), block.Hash()), nil } // CreateTimeoutCert creates a timeout certificate from a list of timeout messages. @@ -96,7 +96,7 @@ func (c crypto) CreateAggregateQC(view hotstuff.View, timeouts []hotstuff.Timeou // VerifyPartialCert verifies a single partial certificate. func (c crypto) VerifyPartialCert(cert hotstuff.PartialCert) bool { - block, ok := c.blockChain.Get(cert.BlockHash()) + block, ok := c.blockChain.Get(cert.BlockHash(), cert.Pipe()) if !ok { return false } @@ -112,7 +112,7 @@ func (c crypto) VerifyQuorumCert(qc hotstuff.QuorumCert) bool { if qc.Signature().Participants().Len() < c.configuration.QuorumSize() { return false } - block, ok := c.blockChain.Get(qc.BlockHash()) + block, ok := c.blockChain.Get(qc.BlockHash(), qc.Pipe()) if !ok { return false } @@ -142,7 +142,8 @@ func (c crypto) VerifyAggregateQC(aggQC hotstuff.AggregateQC) (highQC hotstuff.Q messages[id] = hotstuff.TimeoutMsg{ ID: id, View: aggQC.View(), - SyncInfo: hotstuff.NewSyncInfo().WithQC(qc), + SyncInfo: hotstuff.NewSyncInfo(qc.Pipe()).WithQC(qc), + Pipe: qc.Pipe(), }.ToBytes() } if aggQC.Sig().Participants().Len() < c.configuration.QuorumSize() { diff --git a/crypto/crypto_test.go b/crypto/crypto_test.go index 0f12f8b9..bd49bd0c 100644 --- a/crypto/crypto_test.go +++ b/crypto/crypto_test.go @@ -209,7 +209,7 @@ func createBlock(t *testing.T, signer modules.Crypto) *hotstuff.Block { t.Errorf("Could not create empty QC for genesis: %v", err) } - b := hotstuff.NewBlock(hotstuff.GetGenesis().Hash(), qc, "foo", 42, 1) + b := hotstuff.NewBlock(hotstuff.GetGenesis().Hash(), qc, "foo", 42, 1, 0) return b } diff --git a/crypto/ecdsa/ecdsa.go b/crypto/ecdsa/ecdsa.go index a4333de2..1e57595f 100644 --- a/crypto/ecdsa/ecdsa.go +++ b/crypto/ecdsa/ecdsa.go @@ -79,7 +79,7 @@ func New() modules.CryptoBase { // InitModule gives the module a reference to the Core object. // It also allows the module to set module options using the OptionsBuilder. -func (ec *ecdsaBase) InitModule(mods *modules.Core) { +func (ec *ecdsaBase) InitModule(mods *modules.Core, _ modules.ScopeInfo) { mods.Get( &ec.configuration, &ec.logger, diff --git a/crypto/eddsa/eddsa.go b/crypto/eddsa/eddsa.go index c886731f..9acf4a0d 100644 --- a/crypto/eddsa/eddsa.go +++ b/crypto/eddsa/eddsa.go @@ -65,7 +65,7 @@ func New() modules.CryptoBase { // InitModule gives the module a reference to the Core object. // It also allows the module to set module options using the OptionsBuilder. -func (ed *eddsaBase) InitModule(mods *modules.Core) { +func (ed *eddsaBase) InitModule(mods *modules.Core, _ modules.ScopeInfo) { mods.Get( &ed.configuration, &ed.logger, diff --git a/debug/events.go b/debug/events.go new file mode 100644 index 00000000..f48a426b --- /dev/null +++ b/debug/events.go @@ -0,0 +1,14 @@ +package debug + +import ( + "github.com/relab/hotstuff" +) + +type CommitHaltEvent struct { + Pipe hotstuff.Pipe +} + +type CommandRejectedEvent struct { + Pipe hotstuff.Pipe + View hotstuff.View +} diff --git a/docs/experimentation.md b/docs/experimentation.md index 6ea6c5df..5837dfea 100644 --- a/docs/experimentation.md +++ b/docs/experimentation.md @@ -68,7 +68,7 @@ The example below shows a complete initialization function for the throughput me ```go // InitModule implements the modules.Module interface -func (t *Throughput) InitModule(mods *modules.Core) { +func (t *Throughput) InitModule(mods *modules.Core, opt modules.InitOptions) { var ( eventLoop *eventloop.EventLoop logger logging.Logger diff --git a/docs/modules.md b/docs/modules.md index 65842648..0f1d2e84 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -77,7 +77,7 @@ but it is not sensible to use this as a solution to the circular dependency prob ### Deferred Initialization -This idea is simple: First create the instances of all the modules. +This idea is simple: First create the cis of all the modules. Afterwards, initialize the modules using the now existing, albeit uninitialized, modules. Consider the following code example: @@ -163,7 +163,7 @@ For example: ```go type A1 struct{ b B } -func (a *A1) InitModule(mods *modules.Core) { +func (a *A1) InitModule(mods *modules.Core, opt modules.InitOptions) { mods.Get(&a.b) } ``` diff --git a/eventloop/eventloop.go b/eventloop/eventloop.go index 95861166..97ea730d 100644 --- a/eventloop/eventloop.go +++ b/eventloop/eventloop.go @@ -1,9 +1,3 @@ -// Package eventloop provides an event loop which is widely used by modules. -// The use of the event loop enables many of the modules to run synchronously, thus removing the need for thread safety. -// This simplifies the implementation of modules and reduces the risks of race conditions. -// -// The event loop can accept events of any type. -// It uses reflection to determine what handler function to execute based on the type of an event. package eventloop import ( @@ -15,40 +9,6 @@ import ( "github.com/relab/hotstuff/util/gpool" ) -type handlerOpts struct { - runInAddEvent bool - priority bool -} - -// HandlerOption sets configuration options for event handlers. -type HandlerOption func(*handlerOpts) - -// UnsafeRunInAddEvent instructs the eventloop to run the handler as a part of AddEvent. -// Handlers that use this option can process events before they are added to the event queue. -// Because AddEvent could be running outside the event loop, it is unsafe. -// Only thread-safe modules can be used safely from a handler using this option. -func UnsafeRunInAddEvent() HandlerOption { - return func(ho *handlerOpts) { - ho.runInAddEvent = true - } -} - -// Prioritize instructs the event loop to run the handler before handlers that do not have priority. -// It should only be used if you must look at an event before other handlers get to look at it. -func Prioritize() HandlerOption { - return func(ho *handlerOpts) { - ho.priority = true - } -} - -// EventHandler processes an event. -type EventHandler func(event any) - -type handler struct { - callback EventHandler - opts handlerOpts -} - // EventLoop accepts events of any type and executes registered event handlers. type EventLoop struct { eventQ queue @@ -289,16 +249,6 @@ func (el *EventLoop) DelayUntil(eventType, event any) { el.mut.Unlock() } -type ticker struct { - interval time.Duration - callback func(time.Time) any - cancel context.CancelFunc -} - -type startTickerEvent struct { - tickerID int -} - // AddTicker adds a ticker with the specified interval and returns the ticker id. // The ticker will send the specified event on the event loop at regular intervals. // The returned ticker id can be used to remove the ticker with RemoveTicker. diff --git a/eventloop/eventloop_test.go b/eventloop/eventloop_test.go index c322f10d..d6366c42 100644 --- a/eventloop/eventloop_test.go +++ b/eventloop/eventloop_test.go @@ -6,13 +6,14 @@ import ( "testing" "time" + "github.com/relab/hotstuff" "github.com/relab/hotstuff/eventloop" ) type testEvent int func TestHandler(t *testing.T) { - el := eventloop.New(10) + el := eventloop.NewScoped(10, 0) c := make(chan any) el.RegisterHandler(testEvent(0), func(event any) { c <- event @@ -45,13 +46,58 @@ func TestHandler(t *testing.T) { } } +func TestHandlerScoped(t *testing.T) { + listeningScope := hotstuff.Pipe(1) + incorrectScope := hotstuff.Pipe(2) + scopes := 1 + el := eventloop.NewScoped(10, scopes) + c := make(chan any) + el.RegisterHandler(testEvent(0), func(event any) { + c <- event + }, eventloop.RespondToScope(listeningScope)) + + el.RegisterHandler(testEvent(0), func(_ any) { + panic("wrong scope") + }, eventloop.RespondToScope(incorrectScope)) + + el.RegisterHandler(testEvent(0), func(_ any) { + panic("non-scoped handler should not respond") + }) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + go el.Run(ctx) + + // wait for the event loop to start + time.Sleep(1 * time.Millisecond) + + want := testEvent(42) + el.AddScopedEvent(listeningScope, want) + + var event any + select { + case <-ctx.Done(): + t.Fatal("timed out") + case event = <-c: + } + + e, ok := event.(testEvent) + if !ok { + t.Fatalf("wrong type for event: got: %T, want: %T", event, want) + } + + if e != want { + t.Fatalf("wrong value for event: got: %v, want: %v", e, want) + } +} + func TestObserver(t *testing.T) { type eventData struct { event any handler bool } - el := eventloop.New(10) + el := eventloop.NewScoped(10, 0) c := make(chan eventData) el.RegisterHandler(testEvent(0), func(event any) { c <- eventData{event: event, handler: true} @@ -100,7 +146,7 @@ func TestTicker(t *testing.T) { return } - el := eventloop.New(10) + el := eventloop.NewScoped(10, 0) count := 0 el.RegisterHandler(testEvent(0), func(event any) { count += int(event.(testEvent)) @@ -130,8 +176,49 @@ func TestTicker(t *testing.T) { } } +func TestDelayedEventScoped(t *testing.T) { + scopes := 1 + listeningScope := hotstuff.Pipe(1) + incorrectScope := hotstuff.Pipe(2) + el := eventloop.NewScoped(10, scopes) + c := make(chan testEvent) + + el.RegisterHandler(testEvent(0), func(event any) { + c <- event.(testEvent) + }, eventloop.RespondToScope(listeningScope)) + + el.RegisterHandler(testEvent(0), func(_ any) { + panic("wrong scope") + }, eventloop.RespondToScope(incorrectScope)) + + el.RegisterHandler(testEvent(0), func(_ any) { + panic("non-scoped handler should not respond") + }) + + // delay the "2" and "3" events until after the first instance of testEvent + el.DelayScoped(listeningScope, testEvent(0), testEvent(2)) + el.DelayScoped(listeningScope, testEvent(0), testEvent(3)) + // then send the "1" event + el.AddScopedEvent(listeningScope, testEvent(1)) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + go el.Run(ctx) + + for i := 1; i <= 3; i++ { + select { + case event := <-c: + if testEvent(i) != event { + t.Errorf("events arrived in the wrong order: want: %d, got: %d", i, event) + } + case <-ctx.Done(): + t.Fatalf("timed out") + } + } +} + func TestDelayedEvent(t *testing.T) { - el := eventloop.New(10) + el := eventloop.NewScoped(10, 0) c := make(chan testEvent) el.RegisterHandler(testEvent(0), func(event any) { @@ -161,7 +248,7 @@ func TestDelayedEvent(t *testing.T) { } func BenchmarkEventLoopWithObservers(b *testing.B) { - el := eventloop.New(100) + el := eventloop.NewScoped(100, 0) for i := 0; i < 100; i++ { el.RegisterObserver(testEvent(0), func(event any) { @@ -178,7 +265,7 @@ func BenchmarkEventLoopWithObservers(b *testing.B) { } func BenchmarkEventLoopWithUnsafeRunInAddEventHandlers(b *testing.B) { - el := eventloop.New(100) + el := eventloop.NewScoped(100, 0) for i := 0; i < 100; i++ { el.RegisterHandler(testEvent(0), func(event any) { @@ -203,7 +290,7 @@ func BenchmarkEventLoopWithUnsafeRunInAddEventHandlers(b *testing.B) { } func BenchmarkDelay(b *testing.B) { - el := eventloop.New(100) + el := eventloop.NewScoped(100, 0) for i := 0; i < b.N; i++ { el.DelayUntil(testEvent(0), testEvent(2)) diff --git a/eventloop/scoped.go b/eventloop/scoped.go new file mode 100644 index 00000000..c12209cd --- /dev/null +++ b/eventloop/scoped.go @@ -0,0 +1,397 @@ +// Package eventloop provides an event loop which is widely used by modules. +// The use of the event loop enables many of the modules to run synchronously, thus removing the need for thread safety. +// This simplifies the implementation of modules and reduces the risks of race conditions. +// +// The event loop can accept events of any type. +// It uses reflection to determine what handler function to execute based on the type of an event. +package eventloop + +import ( + "context" + "reflect" + "sync" + "time" + + "github.com/relab/hotstuff" + "github.com/relab/hotstuff/util/gpool" +) + +type scopedEventWrapper struct { + pipe hotstuff.Pipe + event any +} + +// ScopedEventLoop accepts events of any type and executes registered event handlers. +type ScopedEventLoop struct { + eventQ queue + + mut sync.Mutex // protects the following: + + ctx context.Context // set by Run + + waitingEvents map[hotstuff.Pipe]map[reflect.Type][]any + + handlers map[hotstuff.Pipe]map[reflect.Type][]handler + + tickers map[int]*ticker + tickerID int + pipeCount int // number of pipes +} + +// NewScoped returns a new event loop with the requested buffer size. +func NewScoped(bufferSize uint, pipeCount int) *ScopedEventLoop { + el := &ScopedEventLoop{ + ctx: context.Background(), + eventQ: newQueue(bufferSize), + waitingEvents: make(map[hotstuff.Pipe]map[reflect.Type][]any), + handlers: make(map[hotstuff.Pipe]map[reflect.Type][]handler), + tickers: make(map[int]*ticker), + } + + el.pipeCount = pipeCount + if el.pipeCount == 0 { + el.pipeCount = 1 + } + + return el +} + +// RegisterObserver registers a handler with priority. +// Deprecated: use RegisterHandler and the Prioritize option instead. +func (el *ScopedEventLoop) RegisterObserver(eventType any, handler EventHandler) int { + return el.registerHandler(eventType, []HandlerOption{Prioritize()}, handler) +} + +// UnregisterObserver unregister a handler. +// Deprecated: use UnregisterHandler instead. +func (el *ScopedEventLoop) UnregisterObserver(eventType any, pipe hotstuff.Pipe, id int) { + el.UnregisterHandler(eventType, pipe, id) +} + +// RegisterHandler registers the given event handler for the given event type with the given handler options, if any. +// If no handler options are provided, the default handler options will be used. +func (el *ScopedEventLoop) RegisterHandler(eventType any, handler EventHandler, opts ...HandlerOption) int { + return el.registerHandler(eventType, opts, handler) +} + +func (el *ScopedEventLoop) registerHandler(eventType any, opts []HandlerOption, callback EventHandler) int { + h := handler{callback: callback} + + for _, opt := range opts { + opt(&h.opts) + } + + el.mut.Lock() + defer el.mut.Unlock() + t := reflect.TypeOf(eventType) + + _, ok := el.handlers[h.opts.pipe] + if !ok { + el.handlers[h.opts.pipe] = make(map[reflect.Type][]handler) + } + + handlers := el.handlers[h.opts.pipe][t] + + // search for a free slot for the handler + i := 0 + for ; i < len(handlers); i++ { + if handlers[i].callback == nil { + break + } + } + + // no free slots; have to grow the list + if i == len(handlers) { + handlers = append(handlers, h) + } else { + handlers[i] = h + } + + el.handlers[h.opts.pipe][t] = handlers + + return i +} + +// UnregisterHandler unregisters the handler for the given event type with the given id. +func (el *ScopedEventLoop) UnregisterHandler(eventType any, pipe hotstuff.Pipe, id int) { + el.mut.Lock() + defer el.mut.Unlock() + t := reflect.TypeOf(eventType) + el.handlers[pipe][t][id].callback = nil +} + +// AddEvent adds an event to the event queue. +func (el *ScopedEventLoop) AddEvent(event any) { + if event != nil { + // run handlers with runInAddEvent option + el.processEvent(event, hotstuff.NullPipe, true) + el.eventQ.push(scopedEventWrapper{ + event: event, + pipe: hotstuff.NullPipe, + }) + } +} + +// DebugEvent adds an event to the event queue if debug mode is enabled. +// Does nothing otherwise +func (el *ScopedEventLoop) DebugEvent(event any) { + // TODO: Check for a debug flag + el.AddEvent(event) +} + +// AddScopedEvent adds an event to a specified pipe. +func (el *ScopedEventLoop) AddScopedEvent(pipe hotstuff.Pipe, event any) { + if !pipe.IsNull() { + el.AddEvent(event) + return + } + + if event != nil { + // run handlers with runInAddEvent option + el.processEvent(event, pipe, true) + el.eventQ.push(scopedEventWrapper{ + event: event, + pipe: pipe, + }) + } +} + +// Context returns the context associated with the event loop. +// Usually, this context will be the one passed to Run. +// However, if Tick is used instead of Run, Context will return +// the last context that was passed to Tick. +// If neither Run nor Tick have been called, +// Context returns context.Background. +func (el *ScopedEventLoop) Context() context.Context { + el.mut.Lock() + defer el.mut.Unlock() + + return el.ctx +} + +func (el *ScopedEventLoop) setContext(ctx context.Context) { + el.mut.Lock() + defer el.mut.Unlock() + + el.ctx = ctx +} + +// Run runs the event loop. A context object can be provided to stop the event loop. +func (el *ScopedEventLoop) Run(ctx context.Context) { + el.setContext(ctx) + +loop: + for { + event, ok := el.eventQ.pop() + if !ok { + select { + case <-el.eventQ.ready(): + continue loop + case <-ctx.Done(): + break loop + } + } + wrapper := event.(scopedEventWrapper) + if e, ok := wrapper.event.(startTickerEvent); ok { + el.startTicker(e.tickerID) + continue + } + el.processEvent(wrapper.event, wrapper.pipe, false) + } + + // HACK: when we get canceled, we will handle the events that were in the queue at that time before quitting. + l := el.eventQ.len() + for i := 0; i < l; i++ { + event, _ := el.eventQ.pop() + wrapper := event.(scopedEventWrapper) + el.processEvent(wrapper.event, wrapper.pipe, false) + } +} + +// Tick processes a single event. Returns true if an event was handled. +func (el *ScopedEventLoop) Tick(ctx context.Context) bool { + el.setContext(ctx) + + event, ok := el.eventQ.pop() + if !ok { + return false + } + wrapper := event.(scopedEventWrapper) + if e, ok := wrapper.event.(startTickerEvent); ok { + el.startTicker(e.tickerID) + } else { + el.processEvent(wrapper.event, wrapper.pipe, false) + } + + return true +} + +var scopedHandlerListPool = gpool.New(func() []handler { return make([]handler, 0, 10) }) + +// processEvent dispatches the event to the correct handler. +func (el *ScopedEventLoop) processEvent(event any, pipe hotstuff.Pipe, runningInAddEvent bool) { + t := reflect.TypeOf(event) + + if !runningInAddEvent { + defer el.dispatchDelayedEvents(pipe, t) + } + + // Must copy handlers to a list so that they can be executed after unlocking the mutex. + // This looks like it might be slow, but there should be few handlers (< 10) registered for each event type. + // We use a pool to reduce memory allocations. + priorityList := scopedHandlerListPool.Get() + handlerList := scopedHandlerListPool.Get() + + el.mut.Lock() + handlers := el.handlers[pipe][t] + for _, handler := range handlers { + if handler.opts.runInAddEvent != runningInAddEvent || handler.callback == nil { + continue + } + + if handler.opts.priority { + priorityList = append(priorityList, handler) + } else { + handlerList = append(handlerList, handler) + } + } + el.mut.Unlock() + + for _, handler := range priorityList { + handler.callback(event) + } + + priorityList = priorityList[:0] + scopedHandlerListPool.Put(priorityList) + + for _, handler := range handlerList { + handler.callback(event) + } + + handlerList = handlerList[:0] + scopedHandlerListPool.Put(handlerList) +} + +func (el *ScopedEventLoop) dispatchDelayedEvents(pipe hotstuff.Pipe, t reflect.Type) { + var ( + events []any + ok bool + ) + + if _, ok := el.waitingEvents[pipe]; !ok { + return + } + + el.mut.Lock() + if events, ok = el.waitingEvents[pipe][t]; ok { + delete(el.waitingEvents[pipe], t) + } + el.mut.Unlock() + + for _, event := range events { + el.AddScopedEvent(pipe, event) + } +} + +func (el *ScopedEventLoop) DelayScoped(pipe hotstuff.Pipe, eventType, event any) { + if eventType == nil || event == nil { + return + } + + if _, ok := el.waitingEvents[pipe]; !ok { + el.waitingEvents[pipe] = make(map[reflect.Type][]any) + } + + el.mut.Lock() + t := reflect.TypeOf(eventType) + v := el.waitingEvents[pipe][t] + v = append(v, event) + el.waitingEvents[pipe][t] = v + el.mut.Unlock() +} + +// DelayUntil allows us to delay handling of an event until after another event has happened. +// The eventType parameter decides the type of event to wait for, and it should be the zero value +// of that event type. The event parameter is the event that will be delayed. +func (el *ScopedEventLoop) DelayUntil(eventType, event any) { + el.DelayScoped(hotstuff.NullPipe, eventType, event) +} + +// AddTicker adds a ticker with the specified interval and returns the ticker id. +// The ticker will send the specified event on the event loop at regular intervals. +// The returned ticker id can be used to remove the ticker with RemoveTicker. +// The ticker will not be started before the event loop is running. +func (el *ScopedEventLoop) AddTicker(interval time.Duration, callback func(tick time.Time) (event any)) int { + el.mut.Lock() + + id := el.tickerID + el.tickerID++ + + ticker := ticker{ + interval: interval, + callback: callback, + cancel: func() {}, // initialized to empty function to avoid nil + } + el.tickers[id] = &ticker + + el.mut.Unlock() + + // We want the ticker to inherit the context of the event loop, + // so we need to start the ticker from the run loop. + el.eventQ.push(scopedEventWrapper{ + event: startTickerEvent{id}, + pipe: hotstuff.NullPipe, + }) + + return id +} + +// RemoveTicker removes the ticker with the specified id. +// If the ticker was removed, RemoveTicker will return true. +// If the ticker does not exist, false will be returned instead. +func (el *ScopedEventLoop) RemoveTicker(id int) bool { + el.mut.Lock() + defer el.mut.Unlock() + + ticker, ok := el.tickers[id] + if !ok { + return false + } + ticker.cancel() + delete(el.tickers, id) + return true +} + +func (el *ScopedEventLoop) startTicker(id int) { + // lock the mutex such that the ticker cannot be removed until we have started it + el.mut.Lock() + defer el.mut.Unlock() + ticker, ok := el.tickers[id] + if !ok { + return + } + ctx := el.ctx + ctx, ticker.cancel = context.WithCancel(ctx) + go el.runTicker(ctx, ticker) +} + +func (el *ScopedEventLoop) runTicker(ctx context.Context, ticker *ticker) { + t := time.NewTicker(ticker.interval) + defer t.Stop() + + if ctx.Err() != nil { + return + } + + // send the first event immediately + el.AddEvent(ticker.callback(time.Now())) + + for { + select { + case tick := <-t.C: + el.AddEvent(ticker.callback(tick)) + case <-ctx.Done(): + return + } + } +} diff --git a/eventloop/types.go b/eventloop/types.go new file mode 100644 index 00000000..34f6587d --- /dev/null +++ b/eventloop/types.go @@ -0,0 +1,63 @@ +package eventloop + +import ( + "context" + "reflect" + "time" + + "github.com/relab/hotstuff" +) + +type ticker struct { + interval time.Duration + callback func(time.Time) any + cancel context.CancelFunc +} + +type startTickerEvent struct { + tickerID int +} + +// EventHandler processes an event. +type EventHandler func(event any) + +type handler struct { + callback EventHandler + opts handlerOpts + eventType reflect.Type +} + +type handlerOpts struct { + runInAddEvent bool + priority bool + pipe hotstuff.Pipe +} + +// HandlerOption sets configuration options for event handlers. +type HandlerOption func(*handlerOpts) + +// UnsafeRunInAddEvent instructs the eventloop to run the handler as a part of AddEvent. +// Handlers that use this option can process events before they are added to the event queue. +// Because AddEvent could be running outside the event loop, it is unsafe. +// Only thread-safe modules can be used safely from a handler using this option. +func UnsafeRunInAddEvent() HandlerOption { + return func(ho *handlerOpts) { + ho.runInAddEvent = true + } +} + +// Prioritize instructs the event loop to run the handler before handlers that do not have priority. +// It should only be used if you must look at an event before other handlers get to look at it. +func Prioritize() HandlerOption { + return func(ho *handlerOpts) { + ho.priority = true + } +} + +// RespondToScope assigns which pipe (scope) to respond to when ScopeEvent is used to add an event to the +// eventloop. If the NullPipe (0) is passed, this handler option will not take effect. +func RespondToScope(pipe hotstuff.Pipe) HandlerOption { + return func(ho *handlerOpts) { + ho.pipe = pipe + } +} diff --git a/events.go b/events.go index 31bb9971..6d321cfc 100644 --- a/events.go +++ b/events.go @@ -10,6 +10,7 @@ type ProposeMsg struct { ID ID // The ID of the replica who sent the message. Block *Block // The block that is proposed. AggregateQC *AggregateQC // Optional AggregateQC + Pipe Pipe // } func (p ProposeMsg) String() string { @@ -29,8 +30,9 @@ func (v VoteMsg) String() string { // TimeoutMsg is broadcast whenever a replica has a local timeout. type TimeoutMsg struct { - ID ID // The ID of the replica who sent the message. - View View // The view that the replica wants to enter. + ID ID // The ID of the replica who sent the message. + View View // The view that the replica wants to enter. + Pipe Pipe ViewSignature QuorumSignature // A signature of the view MsgSignature QuorumSignature // A signature of the view, QC.BlockHash, and the replica ID SyncInfo SyncInfo // The highest QC/TC known to the sender. @@ -41,6 +43,7 @@ func (timeout TimeoutMsg) ToBytes() []byte { var b bytes.Buffer _, _ = b.Write(timeout.ID.ToBytes()) _, _ = b.Write(timeout.View.ToBytes()) + _, _ = b.Write(timeout.Pipe.ToBytes()) if qc, ok := timeout.SyncInfo.QC(); ok { _, _ = b.Write(qc.ToBytes()) } @@ -48,7 +51,7 @@ func (timeout TimeoutMsg) ToBytes() []byte { } func (timeout TimeoutMsg) String() string { - return fmt.Sprintf("ID: %d, View: %d, SyncInfo: %v", timeout.ID, timeout.View, timeout.SyncInfo) + return fmt.Sprintf("ID: %d, CI: %d, View: %d, SyncInfo: %v", timeout.ID, timeout.Pipe, timeout.View, timeout.SyncInfo) } // NewViewMsg is sent to the leader whenever a replica decides to advance to the next view. @@ -61,5 +64,6 @@ type NewViewMsg struct { // CommitEvent is raised whenever a block is committed, // and includes the number of client commands that were executed. type CommitEvent struct { + Pipe Pipe Commands int } diff --git a/genesis.go b/genesis.go index 5a6b5ad0..0699fb0a 100644 --- a/genesis.go +++ b/genesis.go @@ -1,6 +1,9 @@ package hotstuff -var genesisBlock = NewBlock(Hash{}, QuorumCert{}, "", 0, 0) +// Genesis block can be on pipe 1, even if no pipelining is enabled +const firstPipe = 1 + +var genesisBlock = NewBlock(Hash{}, QuorumCert{}, "", 0, 0, firstPipe) // GetGenesis returns a pointer to the genesis block, the starting point for the hotstuff blockchain. func GetGenesis() *Block { diff --git a/go.mod b/go.mod index 71eda768..3b68d958 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.22.1 require ( github.com/felixge/fgprof v0.9.4 github.com/golang/mock v1.6.0 - github.com/golang/protobuf v1.5.4 github.com/kilic/bls12-381 v0.1.1-0.20210208205449-6045b0235e36 github.com/mattn/go-isatty v0.0.20 github.com/mitchellh/go-homedir v1.1.0 @@ -45,6 +44,7 @@ require ( github.com/go-pdf/fpdf v0.9.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/gonuts/binary v0.2.0 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect diff --git a/handel/handel.go b/handel/handel.go index ea04fa3c..9d0fe137 100644 --- a/handel/handel.go +++ b/handel/handel.go @@ -51,7 +51,7 @@ type Handel struct { blockChain modules.BlockChain crypto modules.Crypto - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop logger logging.Logger opts *modules.Options synchronizer modules.Synchronizer @@ -71,7 +71,7 @@ func New() modules.Handel { } // InitModule initializes the Handel module. -func (h *Handel) InitModule(mods *modules.Core) { +func (h *Handel) InitModule(mods *modules.Core, _ modules.ScopeInfo) { mods.Get( &h.configuration, &h.server, diff --git a/handel/session.go b/handel/session.go index 041a31b7..7a9bf697 100644 --- a/handel/session.go +++ b/handel/session.go @@ -376,9 +376,11 @@ func (s *session) updateOutgoing(levelIndex int) { s.h.logger.Debugf("Done with session: %.8s", s.hash) s.h.eventLoop.AddEvent(hotstuff.NewViewMsg{ - SyncInfo: hotstuff.NewSyncInfo().WithQC(hotstuff.NewQuorumCert( + // TODO: Check if null pipe is okay to use here + SyncInfo: hotstuff.NewSyncInfo(hotstuff.NullPipe).WithQC(hotstuff.NewQuorumCert( outgoing, s.h.synchronizer.View(), + hotstuff.NullPipe, // TODO: Verify if this code conflicts with pipelining s.hash, )), }) @@ -625,7 +627,8 @@ func (s *session) improveSignature(contribution contribution) hotstuff.QuorumSig } func (s *session) verifyContribution(c contribution, sig hotstuff.QuorumSignature, verifyIndiv bool) { - block, ok := s.h.blockChain.Get(s.hash) + // TODO (Alan): Verify issues with pipelining + block, ok := s.h.blockChain.Get(s.hash, hotstuff.NullPipe) if !ok { return } diff --git a/internal/cli/run.go b/internal/cli/run.go index b3e5e1e9..0b91434d 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -63,6 +63,10 @@ func init() { runCmd.Flags().String("leader-rotation", "round-robin", "name of the leader rotation algorithm") runCmd.Flags().Int64("shared-seed", 0, "Shared random number generator seed") runCmd.Flags().StringSlice("modules", nil, "Name additional modules to be loaded.") + runCmd.Flags().Uint32("pipes", 0, "number of pipes in pipelining mode, where zero means pipelining is disabled.") + runCmd.Flags().String("pipeline-ordering", "sequential", "block execution ordering logic for pipelining mode.") + runCmd.Flags().String("viewduration-method", "fixed", "Calculation method for computing view durations.") + runCmd.Flags().Duration("hacky-replica-latency", 0, "Hacky way to induce replica latency. All replicas will have this latency.") runCmd.Flags().Bool("worker", false, "run a local worker") runCmd.Flags().StringSlice("hosts", nil, "the remote hosts to run the experiment on via ssh") @@ -105,21 +109,25 @@ func runController() { Duration: viper.GetDuration("duration"), Output: outputDir, ReplicaOpts: &orchestrationpb.ReplicaOpts{ - UseTLS: true, - BatchSize: viper.GetUint32("batch-size"), - TimeoutMultiplier: float32(viper.GetFloat64("timeout-multiplier")), - Consensus: viper.GetString("consensus"), - Crypto: viper.GetString("crypto"), - LeaderRotation: viper.GetString("leader-rotation"), - ConnectTimeout: durationpb.New(viper.GetDuration("connect-timeout")), - InitialTimeout: durationpb.New(viper.GetDuration("view-timeout")), - TimeoutSamples: viper.GetUint32("duration-samples"), - MaxTimeout: durationpb.New(viper.GetDuration("max-timeout")), - SharedSeed: viper.GetInt64("shared-seed"), - Modules: viper.GetStringSlice("modules"), + UseTLS: false, // TODO (Alan): this was true, + BatchSize: viper.GetUint32("batch-size"), + TimeoutMultiplier: float32(viper.GetFloat64("timeout-multiplier")), + Consensus: viper.GetString("consensus"), + Crypto: viper.GetString("crypto"), + LeaderRotation: viper.GetString("leader-rotation"), + ConnectTimeout: durationpb.New(viper.GetDuration("connect-timeout")), + InitialTimeout: durationpb.New(viper.GetDuration("view-timeout")), + TimeoutSamples: viper.GetUint32("duration-samples"), + MaxTimeout: durationpb.New(viper.GetDuration("max-timeout")), + SharedSeed: viper.GetInt64("shared-seed"), + Modules: viper.GetStringSlice("modules"), + Pipes: viper.GetUint32("pipes"), + PipelineOrdering: viper.GetString("pipeline-ordering"), + ViewDurationMethod: viper.GetString("viewduration-method"), + HackyLatency: durationpb.New(viper.GetDuration("hacky-replica-latency")), }, ClientOpts: &orchestrationpb.ClientOpts{ - UseTLS: true, + UseTLS: false, // TODO (Alan): this was true, ConnectTimeout: durationpb.New(viper.GetDuration("connect-timeout")), PayloadSize: viper.GetUint32("payload-size"), MaxConcurrent: viper.GetUint32("max-concurrent"), @@ -136,7 +144,6 @@ func runController() { worker := viper.GetBool("worker") hosts := viper.GetStringSlice("hosts") exePath := viper.GetString("exe") - g, err := iago.NewSSHGroup(hosts, viper.GetString("ssh-config")) checkf("failed to connect to remote hosts: %v", err) diff --git a/internal/mocks/executor_mock.go b/internal/mocks/executor_mock.go index 7139b44f..8f18d4f7 100644 --- a/internal/mocks/executor_mock.go +++ b/internal/mocks/executor_mock.go @@ -35,13 +35,13 @@ func (m *MockExecutor) EXPECT() *MockExecutorMockRecorder { } // Exec mocks base method. -func (m *MockExecutor) Exec(arg0 hotstuff.Command) { +func (m *MockExecutor) Exec(arg0 hotstuff.Pipe, arg1 hotstuff.Command) { m.ctrl.T.Helper() - m.ctrl.Call(m, "Exec", arg0) + m.ctrl.Call(m, "Exec", arg0, arg1) } // Exec indicates an expected call of Exec. -func (mr *MockExecutorMockRecorder) Exec(arg0 interface{}) *gomock.Call { +func (mr *MockExecutorMockRecorder) Exec(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exec", reflect.TypeOf((*MockExecutor)(nil).Exec), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exec", reflect.TypeOf((*MockExecutor)(nil).Exec), arg0, arg1) } diff --git a/internal/mocks/synchronizer_mock.go b/internal/mocks/synchronizer_mock.go index beb1ff3a..9ac77083 100644 --- a/internal/mocks/synchronizer_mock.go +++ b/internal/mocks/synchronizer_mock.go @@ -66,7 +66,7 @@ func (m *MockSynchronizer) LeafBlock() *hotstuff.Block { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "LeafBlock") ret0, _ := ret[0].(*hotstuff.Block) - return ret0 + return ret0 } // LeafBlock indicates an expected call of LeafBlock. diff --git a/internal/orchestration/worker.go b/internal/orchestration/worker.go index 290d66d6..cbcaa8c0 100644 --- a/internal/orchestration/worker.go +++ b/internal/orchestration/worker.go @@ -15,6 +15,7 @@ import ( "github.com/relab/hotstuff/backend" "github.com/relab/hotstuff/blockchain" "github.com/relab/hotstuff/client" + "github.com/relab/hotstuff/committer" "github.com/relab/hotstuff/consensus" "github.com/relab/hotstuff/consensus/byzantine" "github.com/relab/hotstuff/crypto" @@ -166,14 +167,15 @@ func (w *Worker) createReplica(opts *orchestrationpb.ReplicaOpts) (*replica.Repl // prepare modules builder := modules.NewBuilder(hotstuff.ID(opts.GetID()), privKey) - consensusRules, ok := modules.GetModule[consensus.Rules](opts.GetConsensus()) + newConsensusRules, ok := modules.GetModuleCtor[consensus.Rules](opts.GetConsensus()) if !ok { return nil, fmt.Errorf("invalid consensus name: '%s'", opts.GetConsensus()) } + var newByz func() byzantine.Byzantine = nil if opts.GetByzantineStrategy() != "" { - if byz, ok := modules.GetModule[byzantine.Byzantine](opts.GetByzantineStrategy()); ok { - consensusRules = byz.Wrap(consensusRules) + if ctor, ok := modules.GetModuleCtor[byzantine.Byzantine](opts.GetByzantineStrategy()); ok { + newByz = ctor } else { return nil, fmt.Errorf("invalid byzantine strategy: '%s'", opts.GetByzantineStrategy()) } @@ -184,28 +186,72 @@ func (w *Worker) createReplica(opts *orchestrationpb.ReplicaOpts) (*replica.Repl return nil, fmt.Errorf("invalid crypto name: '%s'", opts.GetCrypto()) } - leaderRotation, ok := modules.GetModule[modules.LeaderRotation](opts.GetLeaderRotation()) + newLeaderRotation, ok := modules.GetModuleCtor[modules.LeaderRotation](opts.GetLeaderRotation()) if !ok { return nil, fmt.Errorf("invalid leader-rotation algorithm: '%s'", opts.GetLeaderRotation()) } - sync := synchronizer.New(synchronizer.NewViewDuration( - uint64(opts.GetTimeoutSamples()), - float64(opts.GetInitialTimeout().AsDuration().Nanoseconds())/float64(time.Millisecond), - float64(opts.GetMaxTimeout().AsDuration().Nanoseconds())/float64(time.Millisecond), - float64(opts.GetTimeoutMultiplier()), - )) + var comm modules.Committer + if opts.GetPipes() > 0 { + builder.EnablePipelining(int(opts.GetPipes())) + comm, ok = modules.GetModule[modules.Committer](opts.GetPipelineOrdering()) + if !ok { + return nil, fmt.Errorf("invalid pipeline ordering scheme: %s", opts.GetPipelineOrdering()) + } + } else { + comm = committer.New() + } + + consensusRules := builder.CreateScope(newConsensusRules) + if newByz != nil { + for pipe, rules := range consensusRules { + byz := newByz() + consensusRules[pipe] = byz.Wrap(rules.(consensus.Rules)) + } + } + + consensuses := builder.CreateScope(consensus.New) + votingMachines := builder.CreateScope(consensus.NewVotingMachine) + leaderRotations := builder.CreateScope(newLeaderRotation) + + // View duration for "dynamic" + newViewDuration := func() synchronizer.ViewDuration { + return synchronizer.NewViewDuration( + uint64(opts.GetTimeoutSamples()), + float64(opts.GetInitialTimeout().AsDuration().Nanoseconds())/float64(time.Millisecond), + float64(opts.GetMaxTimeout().AsDuration().Nanoseconds())/float64(time.Millisecond), + float64(opts.GetTimeoutMultiplier()), + ) + } + + if opts.GetViewDurationMethod() == "fixed" { + newViewDuration = func() synchronizer.ViewDuration { + return synchronizer.NewFixedDuration(opts.GetInitialTimeout().AsDuration()) + } + } + + viewDurations := builder.CreateScope(newViewDuration) + synchronizers := builder.CreateScope(synchronizer.New) + + logger := logging.New("hs" + strconv.Itoa(int(opts.GetID()))) + builder.Add( - eventloop.New(1000), - consensus.New(consensusRules), - consensus.NewVotingMachine(), + eventloop.NewScoped(1000, int(opts.GetPipes())), crypto.NewCache(cryptoImpl, 100), // TODO: consider making this configurable - leaderRotation, - sync, w.metricsLogger, blockchain.New(), - logging.New("hs"+strconv.Itoa(int(opts.GetID()))), + comm, + logger, ) + + builder.AddScoped( + consensusRules, + consensuses, + votingMachines, + leaderRotations, + synchronizers, + viewDurations) + builder.Options().SetSharedRandomSeed(opts.GetSharedSeed()) if w.measurementInterval > 0 { replicaMetrics := metrics.GetReplicaMetrics(w.metrics...) @@ -233,6 +279,8 @@ func (w *Worker) createReplica(opts *orchestrationpb.ReplicaOpts) (*replica.Repl RootCAs: rootCAs, LocationInfo: locationInfo, BatchSize: opts.GetBatchSize(), + PipeCount: opts.GetPipes(), + HackyLatency: *opts.GetHackyLatency(), ManagerOptions: []gorums.ManagerOption{ gorums.WithDialTimeout(opts.GetConnectTimeout().AsDuration()), gorums.WithGrpcDialOptions(grpc.WithReturnConnectionError()), @@ -305,7 +353,7 @@ func (w *Worker) startClients(req *orchestrationpb.StartClientRequest) (*orchest Timeout: opts.GetTimeout().AsDuration(), } mods := modules.NewBuilder(hotstuff.ID(opts.GetID()), nil) - mods.Add(eventloop.New(1000)) + mods.Add(eventloop.NewScoped(1000, 0)) if w.measurementInterval > 0 { clientMetrics := metrics.GetClientMetrics(w.metrics...) diff --git a/internal/proto/hotstuffpb/convert.go b/internal/proto/hotstuffpb/convert.go index 71c3292e..9ac34458 100644 --- a/internal/proto/hotstuffpb/convert.go +++ b/internal/proto/hotstuffpb/convert.go @@ -82,6 +82,7 @@ func PartialCertToProto(cert hotstuff.PartialCert) *PartialCert { return &PartialCert{ Sig: QuorumSignatureToProto(cert.Signature()), Hash: hash[:], + Pipe: uint32(cert.Pipe()), } } @@ -89,7 +90,7 @@ func PartialCertToProto(cert hotstuff.PartialCert) *PartialCert { func PartialCertFromProto(cert *PartialCert) hotstuff.PartialCert { var h hotstuff.Hash copy(h[:], cert.GetHash()) - return hotstuff.NewPartialCert(QuorumSignatureFromProto(cert.GetSig()), h) + return hotstuff.NewPartialCert(hotstuff.Pipe(cert.Pipe), QuorumSignatureFromProto(cert.GetSig()), h) } // QuorumCertToProto converts a consensus.QuorumCert to a hotstuffpb.QuorumCert. @@ -99,6 +100,7 @@ func QuorumCertToProto(qc hotstuff.QuorumCert) *QuorumCert { Sig: QuorumSignatureToProto(qc.Signature()), Hash: hash[:], View: uint64(qc.View()), + Pipe: uint32(qc.Pipe()), } } @@ -106,12 +108,17 @@ func QuorumCertToProto(qc hotstuff.QuorumCert) *QuorumCert { func QuorumCertFromProto(qc *QuorumCert) hotstuff.QuorumCert { var h hotstuff.Hash copy(h[:], qc.GetHash()) - return hotstuff.NewQuorumCert(QuorumSignatureFromProto(qc.GetSig()), hotstuff.View(qc.GetView()), h) + return hotstuff.NewQuorumCert( + QuorumSignatureFromProto(qc.GetSig()), + hotstuff.View(qc.GetView()), + hotstuff.Pipe(qc.GetPipe()), + h) } // ProposalToProto converts a ProposeMsg to a protobuf message. func ProposalToProto(proposal hotstuff.ProposeMsg) *Proposal { p := &Proposal{ + Pipe: uint32(proposal.Pipe), Block: BlockToProto(proposal.Block), } if proposal.AggregateQC != nil { @@ -123,6 +130,7 @@ func ProposalToProto(proposal hotstuff.ProposeMsg) *Proposal { // ProposalFromProto converts a protobuf message to a ProposeMsg. func ProposalFromProto(p *Proposal) (proposal hotstuff.ProposeMsg) { proposal.Block = BlockFromProto(p.GetBlock()) + proposal.Pipe = hotstuff.Pipe(p.Pipe) if p.GetAggQC() != nil { aggQC := AggregateQCFromProto(p.GetAggQC()) proposal.AggregateQC = &aggQC @@ -139,6 +147,7 @@ func BlockToProto(block *hotstuff.Block) *Block { QC: QuorumCertToProto(block.QuorumCert()), View: uint64(block.View()), Proposer: uint32(block.Proposer()), + Pipe: uint32(block.Pipe()), } } @@ -152,6 +161,7 @@ func BlockFromProto(block *Block) *hotstuff.Block { hotstuff.Command(block.GetCommand()), hotstuff.View(block.GetView()), hotstuff.ID(block.GetProposer()), + hotstuff.Pipe(block.GetPipe()), ) } @@ -161,6 +171,7 @@ func TimeoutMsgFromProto(m *TimeoutMsg) hotstuff.TimeoutMsg { View: hotstuff.View(m.GetView()), SyncInfo: SyncInfoFromProto(m.GetSyncInfo()), ViewSignature: QuorumSignatureFromProto(m.GetViewSig()), + Pipe: hotstuff.Pipe(m.GetPipe()), } if m.GetViewSig() != nil { timeoutMsg.MsgSignature = QuorumSignatureFromProto(m.GetMsgSig()) @@ -174,6 +185,7 @@ func TimeoutMsgToProto(timeoutMsg hotstuff.TimeoutMsg) *TimeoutMsg { View: uint64(timeoutMsg.View), SyncInfo: SyncInfoToProto(timeoutMsg.SyncInfo), ViewSig: QuorumSignatureToProto(timeoutMsg.ViewSignature), + Pipe: uint32(timeoutMsg.Pipe), } if timeoutMsg.MsgSignature != nil { tm.MsgSig = QuorumSignatureToProto(timeoutMsg.MsgSignature) @@ -183,7 +195,10 @@ func TimeoutMsgToProto(timeoutMsg hotstuff.TimeoutMsg) *TimeoutMsg { // TimeoutCertFromProto converts a timeout certificate from the protobuf type to the hotstuff type. func TimeoutCertFromProto(m *TimeoutCert) hotstuff.TimeoutCert { - return hotstuff.NewTimeoutCert(QuorumSignatureFromProto(m.GetSig()), hotstuff.View(m.GetView())) + return hotstuff.NewTimeoutCert( + QuorumSignatureFromProto(m.GetSig()), + hotstuff.View(m.GetView()), + ) } // TimeoutCertToProto converts a timeout certificate from the hotstuff type to the protobuf type. @@ -214,7 +229,7 @@ func AggregateQCToProto(aggQC hotstuff.AggregateQC) *AggQC { // SyncInfoFromProto converts a SyncInfo struct from the protobuf type to the hotstuff type. func SyncInfoFromProto(m *SyncInfo) hotstuff.SyncInfo { - si := hotstuff.NewSyncInfo() + si := hotstuff.NewSyncInfo(hotstuff.Pipe(m.GetPipe())) if qc := m.GetQC(); qc != nil { si = si.WithQC(QuorumCertFromProto(qc)) } @@ -230,6 +245,7 @@ func SyncInfoFromProto(m *SyncInfo) hotstuff.SyncInfo { // SyncInfoToProto converts a SyncInfo struct from the hotstuff type to the protobuf type. func SyncInfoToProto(syncInfo hotstuff.SyncInfo) *SyncInfo { m := &SyncInfo{} + m.Pipe = uint32(syncInfo.Pipe()) if qc, ok := syncInfo.QC(); ok { m.QC = QuorumCertToProto(qc) } diff --git a/internal/proto/hotstuffpb/convert_test.go b/internal/proto/hotstuffpb/convert_test.go index 8540d6a3..56980167 100644 --- a/internal/proto/hotstuffpb/convert_test.go +++ b/internal/proto/hotstuffpb/convert_test.go @@ -43,7 +43,13 @@ func TestConvertQuorumCert(t *testing.T) { builders := testutil.CreateBuilders(t, ctrl, 4) hl := builders.Build() - b1 := hotstuff.NewBlock(hotstuff.GetGenesis().Hash(), hotstuff.NewQuorumCert(nil, 0, hotstuff.GetGenesis().Hash()), "", 1, 1) + b1 := hotstuff.NewBlock( + hotstuff.GetGenesis().Hash(), + hotstuff.NewQuorumCert( + nil, + 0, + hotstuff.NullPipe, + hotstuff.GetGenesis().Hash()), "", 1, 1, 0) signatures := testutil.CreatePCs(t, b1, hl.Signers()) @@ -64,8 +70,12 @@ func TestConvertQuorumCert(t *testing.T) { } func TestConvertBlock(t *testing.T) { - qc := hotstuff.NewQuorumCert(nil, 0, hotstuff.Hash{}) - want := hotstuff.NewBlock(hotstuff.GetGenesis().Hash(), qc, "", 1, 1) + qc := hotstuff.NewQuorumCert( + nil, + 0, + hotstuff.NullPipe, // TODO: Verify if this code conflicts with pipelining + hotstuff.Hash{}) + want := hotstuff.NewBlock(hotstuff.GetGenesis().Hash(), qc, "", 1, 1, 0) pb := BlockToProto(want) got := BlockFromProto(pb) diff --git a/internal/proto/hotstuffpb/hotstuff.pb.go b/internal/proto/hotstuffpb/hotstuff.pb.go index b6cf4c5b..8bf65024 100644 --- a/internal/proto/hotstuffpb/hotstuff.pb.go +++ b/internal/proto/hotstuffpb/hotstuff.pb.go @@ -1,16 +1,16 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.32.0 -// protoc v3.12.4 +// protoc-gen-go v1.34.2 +// protoc v5.28.2 // source: internal/proto/hotstuffpb/hotstuff.proto package hotstuffpb import ( - empty "github.com/golang/protobuf/ptypes/empty" _ "github.com/relab/gorums" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" reflect "reflect" sync "sync" ) @@ -29,6 +29,7 @@ type Proposal struct { Block *Block `protobuf:"bytes,1,opt,name=Block,proto3" json:"Block,omitempty"` AggQC *AggQC `protobuf:"bytes,2,opt,name=AggQC,proto3" json:"AggQC,omitempty"` + Pipe uint32 `protobuf:"varint,6,opt,name=Pipe,proto3" json:"Pipe,omitempty"` } func (x *Proposal) Reset() { @@ -77,12 +78,20 @@ func (x *Proposal) GetAggQC() *AggQC { return nil } +func (x *Proposal) GetPipe() uint32 { + if x != nil { + return x.Pipe + } + return 0 +} + type BlockHash struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Hash []byte `protobuf:"bytes,1,opt,name=Hash,proto3" json:"Hash,omitempty"` + Pipe uint32 `protobuf:"varint,2,opt,name=Pipe,proto3" json:"Pipe,omitempty"` } func (x *BlockHash) Reset() { @@ -124,6 +133,13 @@ func (x *BlockHash) GetHash() []byte { return nil } +func (x *BlockHash) GetPipe() uint32 { + if x != nil { + return x.Pipe + } + return 0 +} + type Block struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -134,6 +150,7 @@ type Block struct { View uint64 `protobuf:"varint,3,opt,name=View,proto3" json:"View,omitempty"` Command []byte `protobuf:"bytes,4,opt,name=Command,proto3" json:"Command,omitempty"` Proposer uint32 `protobuf:"varint,5,opt,name=Proposer,proto3" json:"Proposer,omitempty"` + Pipe uint32 `protobuf:"varint,6,opt,name=Pipe,proto3" json:"Pipe,omitempty"` } func (x *Block) Reset() { @@ -203,6 +220,13 @@ func (x *Block) GetProposer() uint32 { return 0 } +func (x *Block) GetPipe() uint32 { + if x != nil { + return x.Pipe + } + return 0 +} + type ECDSASignature struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -374,6 +398,7 @@ type Signature struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Sig: + // // *Signature_ECDSASig // *Signature_BLS12Sig // *Signature_EDDSASig @@ -469,6 +494,7 @@ type PartialCert struct { Sig *QuorumSignature `protobuf:"bytes,1,opt,name=Sig,proto3" json:"Sig,omitempty"` Hash []byte `protobuf:"bytes,2,opt,name=Hash,proto3" json:"Hash,omitempty"` + Pipe uint32 `protobuf:"varint,3,opt,name=Pipe,proto3" json:"Pipe,omitempty"` } func (x *PartialCert) Reset() { @@ -517,6 +543,13 @@ func (x *PartialCert) GetHash() []byte { return nil } +func (x *PartialCert) GetPipe() uint32 { + if x != nil { + return x.Pipe + } + return 0 +} + type ECDSAMultiSignature struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -672,6 +705,7 @@ type QuorumSignature struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Sig: + // // *QuorumSignature_ECDSASigs // *QuorumSignature_BLS12Sig // *QuorumSignature_EDDSASigs @@ -768,6 +802,7 @@ type QuorumCert struct { Sig *QuorumSignature `protobuf:"bytes,1,opt,name=Sig,proto3" json:"Sig,omitempty"` View uint64 `protobuf:"varint,2,opt,name=View,proto3" json:"View,omitempty"` Hash []byte `protobuf:"bytes,3,opt,name=Hash,proto3" json:"Hash,omitempty"` + Pipe uint32 `protobuf:"varint,4,opt,name=Pipe,proto3" json:"Pipe,omitempty"` } func (x *QuorumCert) Reset() { @@ -823,6 +858,13 @@ func (x *QuorumCert) GetHash() []byte { return nil } +func (x *QuorumCert) GetPipe() uint32 { + if x != nil { + return x.Pipe + } + return 0 +} + type TimeoutCert struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -887,6 +929,7 @@ type TimeoutMsg struct { SyncInfo *SyncInfo `protobuf:"bytes,2,opt,name=SyncInfo,proto3" json:"SyncInfo,omitempty"` ViewSig *QuorumSignature `protobuf:"bytes,3,opt,name=ViewSig,proto3" json:"ViewSig,omitempty"` MsgSig *QuorumSignature `protobuf:"bytes,4,opt,name=MsgSig,proto3" json:"MsgSig,omitempty"` + Pipe uint32 `protobuf:"varint,5,opt,name=Pipe,proto3" json:"Pipe,omitempty"` } func (x *TimeoutMsg) Reset() { @@ -949,6 +992,13 @@ func (x *TimeoutMsg) GetMsgSig() *QuorumSignature { return nil } +func (x *TimeoutMsg) GetPipe() uint32 { + if x != nil { + return x.Pipe + } + return 0 +} + type SyncInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -957,6 +1007,7 @@ type SyncInfo struct { QC *QuorumCert `protobuf:"bytes,1,opt,name=QC,proto3" json:"QC,omitempty"` TC *TimeoutCert `protobuf:"bytes,2,opt,name=TC,proto3" json:"TC,omitempty"` AggQC *AggQC `protobuf:"bytes,3,opt,name=AggQC,proto3" json:"AggQC,omitempty"` + Pipe uint32 `protobuf:"varint,4,opt,name=Pipe,proto3" json:"Pipe,omitempty"` } func (x *SyncInfo) Reset() { @@ -1012,6 +1063,13 @@ func (x *SyncInfo) GetAggQC() *AggQC { return nil } +func (x *SyncInfo) GetPipe() uint32 { + if x != nil { + return x.Pipe + } + return 0 +} + type AggQC struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1020,6 +1078,7 @@ type AggQC struct { QCs map[uint32]*QuorumCert `protobuf:"bytes,1,rep,name=QCs,proto3" json:"QCs,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` Sig *QuorumSignature `protobuf:"bytes,2,opt,name=Sig,proto3" json:"Sig,omitempty"` View uint64 `protobuf:"varint,3,opt,name=View,proto3" json:"View,omitempty"` + Pipe uint32 `protobuf:"varint,4,opt,name=Pipe,proto3" json:"Pipe,omitempty"` } func (x *AggQC) Reset() { @@ -1075,6 +1134,13 @@ func (x *AggQC) GetView() uint64 { return 0 } +func (x *AggQC) GetPipe() uint32 { + if x != nil { + return x.Pipe + } + return 0 +} + var File_internal_proto_hotstuffpb_hotstuff_proto protoreflect.FileDescriptor var file_internal_proto_hotstuffpb_hotstuff_proto_rawDesc = []byte{ @@ -1084,51 +1150,56 @@ var file_internal_proto_hotstuffpb_hotstuff_proto_rawDesc = []byte{ 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x1a, 0x0c, 0x67, 0x6f, 0x72, 0x75, 0x6d, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0x5c, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x70, 0x6f, 0x73, 0x61, 0x6c, 0x12, 0x27, 0x0a, + 0x6f, 0x22, 0x70, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x70, 0x6f, 0x73, 0x61, 0x6c, 0x12, 0x27, 0x0a, 0x05, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x05, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x27, 0x0a, 0x05, 0x41, 0x67, 0x67, 0x51, 0x43, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, - 0x70, 0x62, 0x2e, 0x41, 0x67, 0x67, 0x51, 0x43, 0x52, 0x05, 0x41, 0x67, 0x67, 0x51, 0x43, 0x22, - 0x1f, 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x12, 0x12, 0x0a, 0x04, - 0x48, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x48, 0x61, 0x73, 0x68, - 0x22, 0x91, 0x01, 0x0a, 0x05, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x50, 0x61, 0x72, 0x65, - 0x6e, 0x74, 0x12, 0x26, 0x0a, 0x02, 0x51, 0x43, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, - 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x6f, 0x72, - 0x75, 0x6d, 0x43, 0x65, 0x72, 0x74, 0x52, 0x02, 0x51, 0x43, 0x12, 0x12, 0x0a, 0x04, 0x56, 0x69, - 0x65, 0x77, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x56, 0x69, 0x65, 0x77, 0x12, 0x18, - 0x0a, 0x07, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x07, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x70, - 0x6f, 0x73, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x70, - 0x6f, 0x73, 0x65, 0x72, 0x22, 0x44, 0x0a, 0x0e, 0x45, 0x43, 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x12, 0x0c, - 0x0a, 0x01, 0x52, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x52, 0x12, 0x0c, 0x0a, 0x01, - 0x53, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x53, 0x22, 0x22, 0x0a, 0x0e, 0x42, 0x4c, - 0x53, 0x31, 0x32, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x10, 0x0a, 0x03, - 0x53, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x53, 0x69, 0x67, 0x22, 0x3a, - 0x0a, 0x0e, 0x45, 0x44, 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x12, 0x16, 0x0a, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x53, 0x69, 0x67, 0x22, 0xc0, 0x01, 0x0a, 0x09, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x38, 0x0a, 0x08, 0x45, 0x43, 0x44, 0x53, - 0x41, 0x53, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x68, 0x6f, 0x74, - 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x45, 0x43, 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x48, 0x00, 0x52, 0x08, 0x45, 0x43, 0x44, 0x53, 0x41, 0x53, - 0x69, 0x67, 0x12, 0x38, 0x0a, 0x08, 0x42, 0x4c, 0x53, 0x31, 0x32, 0x53, 0x69, 0x67, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, - 0x62, 0x2e, 0x42, 0x4c, 0x53, 0x31, 0x32, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x48, 0x00, 0x52, 0x08, 0x42, 0x4c, 0x53, 0x31, 0x32, 0x53, 0x69, 0x67, 0x12, 0x38, 0x0a, 0x08, - 0x45, 0x44, 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x45, 0x44, 0x44, 0x53, - 0x41, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x48, 0x00, 0x52, 0x08, 0x45, 0x44, - 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, 0x42, 0x05, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x22, 0x50, 0x0a, - 0x0b, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x43, 0x65, 0x72, 0x74, 0x12, 0x2d, 0x0a, 0x03, - 0x53, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x68, 0x6f, 0x74, 0x73, - 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x03, 0x53, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x48, - 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x48, 0x61, 0x73, 0x68, 0x22, + 0x70, 0x62, 0x2e, 0x41, 0x67, 0x67, 0x51, 0x43, 0x52, 0x05, 0x41, 0x67, 0x67, 0x51, 0x43, 0x12, + 0x12, 0x0a, 0x04, 0x50, 0x69, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x50, + 0x69, 0x70, 0x65, 0x22, 0x33, 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x61, 0x73, 0x68, + 0x12, 0x12, 0x0a, 0x04, 0x48, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, + 0x48, 0x61, 0x73, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x69, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x04, 0x50, 0x69, 0x70, 0x65, 0x22, 0xa5, 0x01, 0x0a, 0x05, 0x42, 0x6c, 0x6f, + 0x63, 0x6b, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x06, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x26, 0x0a, 0x02, 0x51, 0x43, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, + 0x66, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d, 0x43, 0x65, 0x72, 0x74, 0x52, 0x02, + 0x51, 0x43, 0x12, 0x12, 0x0a, 0x04, 0x56, 0x69, 0x65, 0x77, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x04, 0x56, 0x69, 0x65, 0x77, 0x12, 0x18, 0x0a, 0x07, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, + 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x12, 0x1a, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x70, 0x6f, 0x73, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x70, 0x6f, 0x73, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, + 0x50, 0x69, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x50, 0x69, 0x70, 0x65, + 0x22, 0x44, 0x0a, 0x0e, 0x45, 0x43, 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x12, 0x0c, 0x0a, 0x01, 0x52, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x01, 0x52, 0x12, 0x0c, 0x0a, 0x01, 0x53, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x01, 0x53, 0x22, 0x22, 0x0a, 0x0e, 0x42, 0x4c, 0x53, 0x31, 0x32, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x53, 0x69, 0x67, 0x22, 0x3a, 0x0a, 0x0e, 0x45, 0x44, + 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x16, 0x0a, 0x06, + 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x53, 0x69, + 0x67, 0x6e, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x03, 0x53, 0x69, 0x67, 0x22, 0xc0, 0x01, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x12, 0x38, 0x0a, 0x08, 0x45, 0x43, 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, + 0x66, 0x70, 0x62, 0x2e, 0x45, 0x43, 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x48, 0x00, 0x52, 0x08, 0x45, 0x43, 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, 0x12, 0x38, + 0x0a, 0x08, 0x42, 0x4c, 0x53, 0x31, 0x32, 0x53, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x42, 0x4c, + 0x53, 0x31, 0x32, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x48, 0x00, 0x52, 0x08, + 0x42, 0x4c, 0x53, 0x31, 0x32, 0x53, 0x69, 0x67, 0x12, 0x38, 0x0a, 0x08, 0x45, 0x44, 0x44, 0x53, + 0x41, 0x53, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x68, 0x6f, 0x74, + 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x45, 0x44, 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x48, 0x00, 0x52, 0x08, 0x45, 0x44, 0x44, 0x53, 0x41, 0x53, + 0x69, 0x67, 0x42, 0x05, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x22, 0x64, 0x0a, 0x0b, 0x50, 0x61, 0x72, + 0x74, 0x69, 0x61, 0x6c, 0x43, 0x65, 0x72, 0x74, 0x12, 0x2d, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, + 0x70, 0x62, 0x2e, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x03, 0x53, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x48, 0x61, 0x73, 0x68, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x48, 0x61, 0x73, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x50, + 0x69, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x50, 0x69, 0x70, 0x65, 0x22, 0x45, 0x0a, 0x13, 0x45, 0x43, 0x44, 0x53, 0x41, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x2e, 0x0a, 0x04, 0x53, 0x69, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, @@ -1156,47 +1227,52 @@ var file_internal_proto_hotstuffpb_hotstuff_proto_rawDesc = []byte{ 0x69, 0x67, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x45, 0x44, 0x44, 0x53, 0x41, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x48, 0x00, 0x52, 0x09, 0x45, 0x44, - 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, 0x73, 0x42, 0x05, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x22, 0x63, + 0x44, 0x53, 0x41, 0x53, 0x69, 0x67, 0x73, 0x42, 0x05, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x22, 0x77, 0x0a, 0x0a, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d, 0x43, 0x65, 0x72, 0x74, 0x12, 0x2d, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x03, 0x53, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x56, 0x69, 0x65, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x56, 0x69, 0x65, 0x77, 0x12, 0x12, 0x0a, 0x04, 0x48, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x48, - 0x61, 0x73, 0x68, 0x22, 0x50, 0x0a, 0x0b, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x43, 0x65, - 0x72, 0x74, 0x12, 0x2d, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x61, 0x73, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x69, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x04, 0x50, 0x69, 0x70, 0x65, 0x22, 0x50, 0x0a, 0x0b, 0x54, 0x69, 0x6d, 0x65, 0x6f, + 0x75, 0x74, 0x43, 0x65, 0x72, 0x74, 0x12, 0x2d, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, + 0x2e, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x52, 0x03, 0x53, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x56, 0x69, 0x65, 0x77, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x04, 0x56, 0x69, 0x65, 0x77, 0x22, 0xd2, 0x01, 0x0a, 0x0a, 0x54, 0x69, + 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x73, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x56, 0x69, 0x65, 0x77, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x56, 0x69, 0x65, 0x77, 0x12, 0x30, 0x0a, 0x08, + 0x53, 0x79, 0x6e, 0x63, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x53, 0x79, 0x6e, 0x63, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x08, 0x53, 0x79, 0x6e, 0x63, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x35, + 0x0a, 0x07, 0x56, 0x69, 0x65, 0x77, 0x53, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x6f, - 0x72, 0x75, 0x6d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x03, 0x53, 0x69, - 0x67, 0x12, 0x12, 0x0a, 0x04, 0x56, 0x69, 0x65, 0x77, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x04, 0x56, 0x69, 0x65, 0x77, 0x22, 0xbe, 0x01, 0x0a, 0x0a, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, - 0x74, 0x4d, 0x73, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x56, 0x69, 0x65, 0x77, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x04, 0x56, 0x69, 0x65, 0x77, 0x12, 0x30, 0x0a, 0x08, 0x53, 0x79, 0x6e, 0x63, - 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x68, 0x6f, 0x74, - 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x08, 0x53, 0x79, 0x6e, 0x63, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x35, 0x0a, 0x07, 0x56, 0x69, - 0x65, 0x77, 0x53, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x68, 0x6f, - 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x07, 0x56, 0x69, 0x65, 0x77, 0x53, 0x69, - 0x67, 0x12, 0x33, 0x0a, 0x06, 0x4d, 0x73, 0x67, 0x53, 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1b, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x51, - 0x75, 0x6f, 0x72, 0x75, 0x6d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x06, - 0x4d, 0x73, 0x67, 0x53, 0x69, 0x67, 0x22, 0x84, 0x01, 0x0a, 0x08, 0x53, 0x79, 0x6e, 0x63, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x26, 0x0a, 0x02, 0x51, 0x43, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x16, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x6f, - 0x72, 0x75, 0x6d, 0x43, 0x65, 0x72, 0x74, 0x52, 0x02, 0x51, 0x43, 0x12, 0x27, 0x0a, 0x02, 0x54, - 0x43, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, - 0x66, 0x66, 0x70, 0x62, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x43, 0x65, 0x72, 0x74, - 0x52, 0x02, 0x54, 0x43, 0x12, 0x27, 0x0a, 0x05, 0x41, 0x67, 0x67, 0x51, 0x43, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, - 0x2e, 0x41, 0x67, 0x67, 0x51, 0x43, 0x52, 0x05, 0x41, 0x67, 0x67, 0x51, 0x43, 0x22, 0xc8, 0x01, - 0x0a, 0x05, 0x41, 0x67, 0x67, 0x51, 0x43, 0x12, 0x2c, 0x0a, 0x03, 0x51, 0x43, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, - 0x62, 0x2e, 0x41, 0x67, 0x67, 0x51, 0x43, 0x2e, 0x51, 0x43, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x03, 0x51, 0x43, 0x73, 0x12, 0x2d, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, - 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, - 0x03, 0x53, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x56, 0x69, 0x65, 0x77, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x04, 0x56, 0x69, 0x65, 0x77, 0x1a, 0x4e, 0x0a, 0x08, 0x51, 0x43, 0x73, 0x45, + 0x72, 0x75, 0x6d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x07, 0x56, 0x69, + 0x65, 0x77, 0x53, 0x69, 0x67, 0x12, 0x33, 0x0a, 0x06, 0x4d, 0x73, 0x67, 0x53, 0x69, 0x67, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, + 0x70, 0x62, 0x2e, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x52, 0x06, 0x4d, 0x73, 0x67, 0x53, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x69, + 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x50, 0x69, 0x70, 0x65, 0x22, 0x98, + 0x01, 0x0a, 0x08, 0x53, 0x79, 0x6e, 0x63, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x26, 0x0a, 0x02, 0x51, + 0x43, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, + 0x66, 0x66, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d, 0x43, 0x65, 0x72, 0x74, 0x52, + 0x02, 0x51, 0x43, 0x12, 0x27, 0x0a, 0x02, 0x54, 0x43, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x17, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x43, 0x65, 0x72, 0x74, 0x52, 0x02, 0x54, 0x43, 0x12, 0x27, 0x0a, 0x05, + 0x41, 0x67, 0x67, 0x51, 0x43, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x68, 0x6f, + 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x41, 0x67, 0x67, 0x51, 0x43, 0x52, 0x05, + 0x41, 0x67, 0x67, 0x51, 0x43, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x69, 0x70, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x04, 0x50, 0x69, 0x70, 0x65, 0x22, 0xdc, 0x01, 0x0a, 0x05, 0x41, 0x67, + 0x67, 0x51, 0x43, 0x12, 0x2c, 0x0a, 0x03, 0x51, 0x43, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x41, 0x67, + 0x67, 0x51, 0x43, 0x2e, 0x51, 0x43, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x51, 0x43, + 0x73, 0x12, 0x2d, 0x0a, 0x03, 0x53, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, + 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x6f, 0x72, + 0x75, 0x6d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x03, 0x53, 0x69, 0x67, + 0x12, 0x12, 0x0a, 0x04, 0x56, 0x69, 0x65, 0x77, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, + 0x56, 0x69, 0x65, 0x77, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x69, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x04, 0x50, 0x69, 0x70, 0x65, 0x1a, 0x4e, 0x0a, 0x08, 0x51, 0x43, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, @@ -1241,7 +1317,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_rawDescGZIP() []byte { } var file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes = make([]protoimpl.MessageInfo, 18) -var file_internal_proto_hotstuffpb_hotstuff_proto_goTypes = []interface{}{ +var file_internal_proto_hotstuffpb_hotstuff_proto_goTypes = []any{ (*Proposal)(nil), // 0: hotstuffpb.Proposal (*BlockHash)(nil), // 1: hotstuffpb.BlockHash (*Block)(nil), // 2: hotstuffpb.Block @@ -1260,7 +1336,7 @@ var file_internal_proto_hotstuffpb_hotstuff_proto_goTypes = []interface{}{ (*SyncInfo)(nil), // 15: hotstuffpb.SyncInfo (*AggQC)(nil), // 16: hotstuffpb.AggQC nil, // 17: hotstuffpb.AggQC.QCsEntry - (*empty.Empty)(nil), // 18: google.protobuf.Empty + (*emptypb.Empty)(nil), // 18: google.protobuf.Empty } var file_internal_proto_hotstuffpb_hotstuff_proto_depIdxs = []int32{ 2, // 0: hotstuffpb.Proposal.Block:type_name -> hotstuffpb.Block @@ -1309,7 +1385,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*Proposal); i { case 0: return &v.state @@ -1321,7 +1397,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*BlockHash); i { case 0: return &v.state @@ -1333,7 +1409,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*Block); i { case 0: return &v.state @@ -1345,7 +1421,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*ECDSASignature); i { case 0: return &v.state @@ -1357,7 +1433,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*BLS12Signature); i { case 0: return &v.state @@ -1369,7 +1445,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*EDDSASignature); i { case 0: return &v.state @@ -1381,7 +1457,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[6].Exporter = func(v any, i int) any { switch v := v.(*Signature); i { case 0: return &v.state @@ -1393,7 +1469,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[7].Exporter = func(v any, i int) any { switch v := v.(*PartialCert); i { case 0: return &v.state @@ -1405,7 +1481,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[8].Exporter = func(v any, i int) any { switch v := v.(*ECDSAMultiSignature); i { case 0: return &v.state @@ -1417,7 +1493,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[9].Exporter = func(v any, i int) any { switch v := v.(*EDDSAMultiSignature); i { case 0: return &v.state @@ -1429,7 +1505,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[10].Exporter = func(v any, i int) any { switch v := v.(*BLS12AggregateSignature); i { case 0: return &v.state @@ -1441,7 +1517,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[11].Exporter = func(v any, i int) any { switch v := v.(*QuorumSignature); i { case 0: return &v.state @@ -1453,7 +1529,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[12].Exporter = func(v any, i int) any { switch v := v.(*QuorumCert); i { case 0: return &v.state @@ -1465,7 +1541,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[13].Exporter = func(v any, i int) any { switch v := v.(*TimeoutCert); i { case 0: return &v.state @@ -1477,7 +1553,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[14].Exporter = func(v any, i int) any { switch v := v.(*TimeoutMsg); i { case 0: return &v.state @@ -1489,7 +1565,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[15].Exporter = func(v any, i int) any { switch v := v.(*SyncInfo); i { case 0: return &v.state @@ -1501,7 +1577,7 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { return nil } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[16].Exporter = func(v any, i int) any { switch v := v.(*AggQC); i { case 0: return &v.state @@ -1514,12 +1590,12 @@ func file_internal_proto_hotstuffpb_hotstuff_proto_init() { } } } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[6].OneofWrappers = []interface{}{ + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[6].OneofWrappers = []any{ (*Signature_ECDSASig)(nil), (*Signature_BLS12Sig)(nil), (*Signature_EDDSASig)(nil), } - file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[11].OneofWrappers = []interface{}{ + file_internal_proto_hotstuffpb_hotstuff_proto_msgTypes[11].OneofWrappers = []any{ (*QuorumSignature_ECDSASigs)(nil), (*QuorumSignature_BLS12Sig)(nil), (*QuorumSignature_EDDSASigs)(nil), diff --git a/internal/proto/hotstuffpb/hotstuff.proto b/internal/proto/hotstuffpb/hotstuff.proto index 27cc8739..84d22dbb 100644 --- a/internal/proto/hotstuffpb/hotstuff.proto +++ b/internal/proto/hotstuffpb/hotstuff.proto @@ -31,9 +31,13 @@ service Hotstuff { message Proposal { Block Block = 1; AggQC AggQC = 2; + uint32 Pipe = 6; } -message BlockHash { bytes Hash = 1; } +message BlockHash { + bytes Hash = 1; + uint32 Pipe = 2; +} message Block { bytes Parent = 1; @@ -41,6 +45,7 @@ message Block { uint64 View = 3; bytes Command = 4; uint32 Proposer = 5; + uint32 Pipe = 6; } message ECDSASignature { @@ -67,6 +72,7 @@ message Signature { message PartialCert { QuorumSignature Sig = 1; bytes Hash = 2; + uint32 Pipe = 3; } message ECDSAMultiSignature { repeated ECDSASignature Sigs = 1; } @@ -90,6 +96,7 @@ message QuorumCert { QuorumSignature Sig = 1; uint64 View = 2; bytes Hash = 3; + uint32 Pipe = 4; } message TimeoutCert { @@ -102,16 +109,19 @@ message TimeoutMsg { SyncInfo SyncInfo = 2; QuorumSignature ViewSig = 3; QuorumSignature MsgSig = 4; + uint32 Pipe = 5; } message SyncInfo { QuorumCert QC = 1; TimeoutCert TC = 2; AggQC AggQC = 3; + uint32 Pipe = 4; } message AggQC { map QCs = 1; QuorumSignature Sig = 2; uint64 View = 3; + uint32 Pipe = 4; } diff --git a/internal/proto/hotstuffpb/hotstuff_gorums.pb.go b/internal/proto/hotstuffpb/hotstuff_gorums.pb.go index 59c5f38c..c6a4a08e 100644 --- a/internal/proto/hotstuffpb/hotstuff_gorums.pb.go +++ b/internal/proto/hotstuffpb/hotstuff_gorums.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-gorums. DO NOT EDIT. // versions: // protoc-gen-gorums v0.7.0-devel -// protoc v3.12.4 +// protoc v5.28.2 // source: internal/proto/hotstuffpb/hotstuff.proto package hotstuffpb @@ -9,10 +9,10 @@ package hotstuffpb import ( context "context" fmt "fmt" - empty "github.com/golang/protobuf/ptypes/empty" gorums "github.com/relab/gorums" encoding "google.golang.org/grpc/encoding" protoreflect "google.golang.org/protobuf/reflect/protoreflect" + emptypb "google.golang.org/protobuf/types/known/emptypb" ) const ( @@ -33,8 +33,9 @@ type Configuration struct { // ConfigurationFromRaw returns a new Configuration from the given raw configuration and QuorumSpec. // // This function may for example be used to "clone" a configuration but install a different QuorumSpec: -// cfg1, err := mgr.NewConfiguration(qspec1, opts...) -// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) +// +// cfg1, err := mgr.NewConfiguration(qspec1, opts...) +// cfg2 := ConfigurationFromRaw(cfg1.RawConfig, qspec2) func ConfigurationFromRaw(rawCfg gorums.RawConfiguration, qspec QuorumSpec) *Configuration { // return an error if the QuorumSpec interface is not empty and no implementation was provided. var test interface{} = struct{}{} @@ -145,7 +146,7 @@ type Node struct { } // Reference imports to suppress errors if they are not otherwise used. -var _ empty.Empty +var _ emptypb.Empty // Propose is a quorum call invoked on all nodes in configuration c, // with the same argument in, and returns a combined result. @@ -159,7 +160,7 @@ func (c *Configuration) Propose(ctx context.Context, in *Proposal, opts ...gorum } // Reference imports to suppress errors if they are not otherwise used. -var _ empty.Empty +var _ emptypb.Empty // Timeout is a quorum call invoked on all nodes in configuration c, // with the same argument in, and returns a combined result. @@ -251,7 +252,7 @@ type internalBlock struct { } // Reference imports to suppress errors if they are not otherwise used. -var _ empty.Empty +var _ emptypb.Empty // Vote is a quorum call invoked on all nodes in configuration c, // with the same argument in, and returns a combined result. @@ -265,7 +266,7 @@ func (n *Node) Vote(ctx context.Context, in *PartialCert, opts ...gorums.CallOpt } // Reference imports to suppress errors if they are not otherwise used. -var _ empty.Empty +var _ emptypb.Empty // NewView is a quorum call invoked on all nodes in configuration c, // with the same argument in, and returns a combined result. diff --git a/internal/proto/orchestrationpb/orchestration.pb.go b/internal/proto/orchestrationpb/orchestration.pb.go index a170f143..1e07d7fa 100644 --- a/internal/proto/orchestrationpb/orchestration.pb.go +++ b/internal/proto/orchestrationpb/orchestration.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 -// protoc v3.19.4 +// protoc-gen-go v1.34.2 +// protoc v5.28.2 // source: internal/proto/orchestrationpb/orchestration.proto package orchestrationpb @@ -68,6 +68,14 @@ type ReplicaOpts struct { Modules []string `protobuf:"bytes,21,rep,name=Modules,proto3" json:"Modules,omitempty"` // locations of the replicas LocationInfo map[uint32]string `protobuf:"bytes,22,rep,name=LocationInfo,proto3" json:"LocationInfo,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // The number of pipes in pipelining mode. Leave as zero to disable pipelining mode. + Pipes uint32 `protobuf:"varint,23,opt,name=Pipes,proto3" json:"Pipes,omitempty"` + // The ordering scheme for pipelining mode. Default is "sequential". + PipelineOrdering string `protobuf:"bytes,24,opt,name=PipelineOrdering,proto3" json:"PipelineOrdering,omitempty"` + // Fixed or dynamic view duration per pipe. Default is "fixed". + ViewDurationMethod string `protobuf:"bytes,25,opt,name=ViewDurationMethod,proto3" json:"ViewDurationMethod,omitempty"` + // Latency induced by all replicas. + HackyLatency *durationpb.Duration `protobuf:"bytes,26,opt,name=HackyLatency,proto3" json:"HackyLatency,omitempty"` } func (x *ReplicaOpts) Reset() { @@ -242,6 +250,34 @@ func (x *ReplicaOpts) GetLocationInfo() map[uint32]string { return nil } +func (x *ReplicaOpts) GetPipes() uint32 { + if x != nil { + return x.Pipes + } + return 0 +} + +func (x *ReplicaOpts) GetPipelineOrdering() string { + if x != nil { + return x.PipelineOrdering + } + return "" +} + +func (x *ReplicaOpts) GetViewDurationMethod() string { + if x != nil { + return x.ViewDurationMethod + } + return "" +} + +func (x *ReplicaOpts) GetHackyLatency() *durationpb.Duration { + if x != nil { + return x.HackyLatency + } + return nil +} + // ReplicaInfo is the information that the replicas need about each other. type ReplicaInfo struct { state protoimpl.MessageState @@ -1023,7 +1059,7 @@ var file_internal_proto_orchestrationpb_orchestration_proto_rawDesc = []byte{ 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x81, 0x07, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb2, 0x08, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x4f, 0x70, 0x74, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x49, 0x44, 0x12, 0x1e, 0x0a, 0x0a, 0x50, 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x50, 0x72, 0x69, 0x76, 0x61, @@ -1075,155 +1111,166 @@ var file_internal_proto_orchestrationpb_orchestration_proto_rawDesc = []byte{ 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x4f, 0x70, 0x74, 0x73, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x3f, 0x0a, 0x11, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x97, 0x01, 0x0a, 0x0b, 0x52, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x41, 0x64, 0x64, - 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x41, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, - 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x50, 0x6f, 0x72, 0x74, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x50, - 0x6f, 0x72, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x72, - 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x50, - 0x6f, 0x72, 0x74, 0x22, 0xf5, 0x02, 0x0a, 0x0a, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4f, 0x70, - 0x74, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, - 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x55, 0x73, 0x65, 0x54, 0x4c, 0x53, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x06, 0x55, 0x73, 0x65, 0x54, 0x4c, 0x53, 0x12, 0x24, 0x0a, 0x0d, 0x4d, 0x61, - 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x0d, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, - 0x12, 0x20, 0x0a, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x18, - 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, - 0x7a, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, - 0x65, 0x6f, 0x75, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, + 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x50, 0x69, 0x70, 0x65, + 0x73, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x50, 0x69, 0x70, 0x65, 0x73, 0x12, 0x2a, + 0x0a, 0x10, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x69, + 0x6e, 0x67, 0x18, 0x18, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, + 0x6e, 0x65, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x2e, 0x0a, 0x12, 0x56, 0x69, + 0x65, 0x77, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, + 0x18, 0x19, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x56, 0x69, 0x65, 0x77, 0x44, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x3d, 0x0a, 0x0c, 0x48, 0x61, + 0x63, 0x6b, 0x79, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x48, 0x61, 0x63, + 0x6b, 0x79, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x1a, 0x3f, 0x0a, 0x11, 0x4c, 0x6f, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x97, 0x01, 0x0a, 0x0b, 0x52, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x41, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x41, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, + 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, + 0x65, 0x79, 0x12, 0x20, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x50, 0x6f, 0x72, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x50, 0x6f, 0x72, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x50, 0x6f, + 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x50, 0x6f, 0x72, 0x74, 0x22, 0xf5, 0x02, 0x0a, 0x0a, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4f, + 0x70, 0x74, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x02, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x55, 0x73, 0x65, 0x54, 0x4c, 0x53, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x06, 0x55, 0x73, 0x65, 0x54, 0x4c, 0x53, 0x12, 0x24, 0x0a, 0x0d, 0x4d, + 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x0d, 0x4d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, + 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, 0x7a, 0x65, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, + 0x69, 0x7a, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, + 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, + 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, + 0x6d, 0x69, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x52, 0x61, 0x74, 0x65, 0x4c, + 0x69, 0x6d, 0x69, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x52, 0x61, 0x74, 0x65, 0x53, 0x74, 0x65, 0x70, + 0x18, 0x0c, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x52, 0x61, 0x74, 0x65, 0x53, 0x74, 0x65, 0x70, + 0x12, 0x45, 0x0a, 0x10, 0x52, 0x61, 0x74, 0x65, 0x53, 0x74, 0x65, 0x70, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x76, 0x61, 0x6c, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, - 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, 0x6d, - 0x69, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x52, 0x61, 0x74, 0x65, 0x4c, 0x69, - 0x6d, 0x69, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x52, 0x61, 0x74, 0x65, 0x53, 0x74, 0x65, 0x70, 0x18, - 0x0c, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x52, 0x61, 0x74, 0x65, 0x53, 0x74, 0x65, 0x70, 0x12, - 0x45, 0x0a, 0x10, 0x52, 0x61, 0x74, 0x65, 0x53, 0x74, 0x65, 0x70, 0x49, 0x6e, 0x74, 0x65, 0x72, - 0x76, 0x61, 0x6c, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x52, 0x61, 0x74, 0x65, 0x53, 0x74, 0x65, 0x70, 0x49, 0x6e, - 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x33, 0x0a, 0x07, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, - 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x07, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0xc2, 0x01, 0x0a, 0x14, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4f, 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x73, 0x1a, 0x59, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x52, 0x61, 0x74, 0x65, 0x53, 0x74, 0x65, 0x70, 0x49, + 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x33, 0x0a, 0x07, 0x54, 0x69, 0x6d, 0x65, 0x6f, + 0x75, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0xc2, 0x01, 0x0a, + 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4f, 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, - 0x61, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0xc2, 0x01, 0x0a, 0x14, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, - 0x63, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4f, 0x0a, 0x08, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x6f, 0x72, - 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x1a, 0x59, 0x0a, 0x0d, 0x52, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, - 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x52, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x4f, 0x70, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xc4, 0x01, 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x50, 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, - 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, - 0x61, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, - 0x73, 0x1a, 0x59, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe6, 0x01, 0x0a, - 0x13, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0d, 0x52, 0x03, 0x49, 0x44, 0x73, 0x12, 0x5d, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, + 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x52, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x1a, 0x59, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, + 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0xc2, 0x01, 0x0a, 0x14, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4f, 0x0a, 0x08, 0x52, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x6f, + 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x1a, 0x59, 0x0a, 0x0d, 0x52, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, - 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x5e, 0x0a, 0x12, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, - 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x52, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x16, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x26, 0x0a, - 0x12, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0d, - 0x52, 0x03, 0x49, 0x44, 0x73, 0x22, 0x9f, 0x02, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, - 0x06, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, + 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x4f, 0x70, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xc4, 0x01, 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x50, 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x73, 0x1a, 0x59, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe6, 0x01, + 0x0a, 0x13, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0d, 0x52, 0x03, 0x49, 0x44, 0x73, 0x12, 0x5d, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, + 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, + 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x5e, 0x0a, 0x12, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, - 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x06, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x48, 0x0a, 0x06, 0x43, 0x6f, 0x75, 0x6e, 0x74, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, - 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x43, 0x6f, 0x75, 0x6e, 0x74, - 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x39, 0x0a, 0x0b, - 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xab, 0x03, 0x0a, 0x12, 0x53, 0x74, 0x61, 0x72, - 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4a, - 0x0a, 0x07, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x30, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, - 0x62, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x07, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x43, 0x65, - 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, - 0x74, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x14, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, - 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, 0x5c, - 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6c, 0x69, - 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x57, 0x0a, 0x0c, - 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x31, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, + 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x16, 0x0a, 0x14, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x26, + 0x0a, 0x12, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0d, 0x52, 0x03, 0x49, 0x44, 0x73, 0x22, 0x9f, 0x02, 0x0a, 0x13, 0x53, 0x74, 0x6f, 0x70, 0x52, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, + 0x0a, 0x06, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, + 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, + 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x06, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x48, 0x0a, 0x06, 0x43, 0x6f, 0x75, 0x6e, + 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, + 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x52, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x43, 0x6f, 0x75, 0x6e, + 0x74, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x39, 0x0a, + 0x0b, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xab, 0x03, 0x0a, 0x12, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x4a, 0x0a, 0x07, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x30, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x07, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x43, + 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, + 0x69, 0x74, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x14, 0x43, 0x65, 0x72, 0x74, 0x69, + 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, + 0x5c, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x57, 0x0a, + 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x31, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, + 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, + 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4f, 0x70, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5e, 0x0a, 0x12, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, - 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4f, 0x70, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5e, 0x0a, 0x12, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, - 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x2e, 0x52, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x25, 0x0a, 0x11, - 0x53, 0x74, 0x6f, 0x70, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x10, 0x0a, 0x03, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x03, - 0x49, 0x44, 0x73, 0x22, 0x14, 0x0a, 0x12, 0x53, 0x74, 0x6f, 0x70, 0x43, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x0d, 0x0a, 0x0b, 0x51, 0x75, 0x69, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x65, 0x6c, 0x61, 0x62, 0x2f, 0x68, 0x6f, 0x74, - 0x73, 0x74, 0x75, 0x66, 0x66, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x25, 0x0a, + 0x11, 0x53, 0x74, 0x6f, 0x70, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0d, 0x52, + 0x03, 0x49, 0x44, 0x73, 0x22, 0x14, 0x0a, 0x12, 0x53, 0x74, 0x6f, 0x70, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x0d, 0x0a, 0x0b, 0x51, 0x75, + 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x65, 0x6c, 0x61, 0x62, 0x2f, 0x68, 0x6f, + 0x74, 0x73, 0x74, 0x75, 0x66, 0x66, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1239,7 +1286,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_rawDescGZIP() []byt } var file_internal_proto_orchestrationpb_orchestration_proto_msgTypes = make([]protoimpl.MessageInfo, 24) -var file_internal_proto_orchestrationpb_orchestration_proto_goTypes = []interface{}{ +var file_internal_proto_orchestrationpb_orchestration_proto_goTypes = []any{ (*ReplicaOpts)(nil), // 0: orchestrationpb.ReplicaOpts (*ReplicaInfo)(nil), // 1: orchestrationpb.ReplicaInfo (*ClientOpts)(nil), // 2: orchestrationpb.ClientOpts @@ -1271,28 +1318,29 @@ var file_internal_proto_orchestrationpb_orchestration_proto_depIdxs = []int32{ 24, // 1: orchestrationpb.ReplicaOpts.InitialTimeout:type_name -> google.protobuf.Duration 24, // 2: orchestrationpb.ReplicaOpts.MaxTimeout:type_name -> google.protobuf.Duration 15, // 3: orchestrationpb.ReplicaOpts.LocationInfo:type_name -> orchestrationpb.ReplicaOpts.LocationInfoEntry - 24, // 4: orchestrationpb.ClientOpts.ConnectTimeout:type_name -> google.protobuf.Duration - 24, // 5: orchestrationpb.ClientOpts.RateStepInterval:type_name -> google.protobuf.Duration - 24, // 6: orchestrationpb.ClientOpts.Timeout:type_name -> google.protobuf.Duration - 16, // 7: orchestrationpb.ReplicaConfiguration.Replicas:type_name -> orchestrationpb.ReplicaConfiguration.ReplicasEntry - 17, // 8: orchestrationpb.CreateReplicaRequest.Replicas:type_name -> orchestrationpb.CreateReplicaRequest.ReplicasEntry - 18, // 9: orchestrationpb.CreateReplicaResponse.Replicas:type_name -> orchestrationpb.CreateReplicaResponse.ReplicasEntry - 19, // 10: orchestrationpb.StartReplicaRequest.Configuration:type_name -> orchestrationpb.StartReplicaRequest.ConfigurationEntry - 20, // 11: orchestrationpb.StopReplicaResponse.Hashes:type_name -> orchestrationpb.StopReplicaResponse.HashesEntry - 21, // 12: orchestrationpb.StopReplicaResponse.Counts:type_name -> orchestrationpb.StopReplicaResponse.CountsEntry - 22, // 13: orchestrationpb.StartClientRequest.Clients:type_name -> orchestrationpb.StartClientRequest.ClientsEntry - 23, // 14: orchestrationpb.StartClientRequest.Configuration:type_name -> orchestrationpb.StartClientRequest.ConfigurationEntry - 1, // 15: orchestrationpb.ReplicaConfiguration.ReplicasEntry.value:type_name -> orchestrationpb.ReplicaInfo - 0, // 16: orchestrationpb.CreateReplicaRequest.ReplicasEntry.value:type_name -> orchestrationpb.ReplicaOpts - 1, // 17: orchestrationpb.CreateReplicaResponse.ReplicasEntry.value:type_name -> orchestrationpb.ReplicaInfo - 1, // 18: orchestrationpb.StartReplicaRequest.ConfigurationEntry.value:type_name -> orchestrationpb.ReplicaInfo - 2, // 19: orchestrationpb.StartClientRequest.ClientsEntry.value:type_name -> orchestrationpb.ClientOpts - 1, // 20: orchestrationpb.StartClientRequest.ConfigurationEntry.value:type_name -> orchestrationpb.ReplicaInfo - 21, // [21:21] is the sub-list for method output_type - 21, // [21:21] is the sub-list for method input_type - 21, // [21:21] is the sub-list for extension type_name - 21, // [21:21] is the sub-list for extension extendee - 0, // [0:21] is the sub-list for field type_name + 24, // 4: orchestrationpb.ReplicaOpts.HackyLatency:type_name -> google.protobuf.Duration + 24, // 5: orchestrationpb.ClientOpts.ConnectTimeout:type_name -> google.protobuf.Duration + 24, // 6: orchestrationpb.ClientOpts.RateStepInterval:type_name -> google.protobuf.Duration + 24, // 7: orchestrationpb.ClientOpts.Timeout:type_name -> google.protobuf.Duration + 16, // 8: orchestrationpb.ReplicaConfiguration.Replicas:type_name -> orchestrationpb.ReplicaConfiguration.ReplicasEntry + 17, // 9: orchestrationpb.CreateReplicaRequest.Replicas:type_name -> orchestrationpb.CreateReplicaRequest.ReplicasEntry + 18, // 10: orchestrationpb.CreateReplicaResponse.Replicas:type_name -> orchestrationpb.CreateReplicaResponse.ReplicasEntry + 19, // 11: orchestrationpb.StartReplicaRequest.Configuration:type_name -> orchestrationpb.StartReplicaRequest.ConfigurationEntry + 20, // 12: orchestrationpb.StopReplicaResponse.Hashes:type_name -> orchestrationpb.StopReplicaResponse.HashesEntry + 21, // 13: orchestrationpb.StopReplicaResponse.Counts:type_name -> orchestrationpb.StopReplicaResponse.CountsEntry + 22, // 14: orchestrationpb.StartClientRequest.Clients:type_name -> orchestrationpb.StartClientRequest.ClientsEntry + 23, // 15: orchestrationpb.StartClientRequest.Configuration:type_name -> orchestrationpb.StartClientRequest.ConfigurationEntry + 1, // 16: orchestrationpb.ReplicaConfiguration.ReplicasEntry.value:type_name -> orchestrationpb.ReplicaInfo + 0, // 17: orchestrationpb.CreateReplicaRequest.ReplicasEntry.value:type_name -> orchestrationpb.ReplicaOpts + 1, // 18: orchestrationpb.CreateReplicaResponse.ReplicasEntry.value:type_name -> orchestrationpb.ReplicaInfo + 1, // 19: orchestrationpb.StartReplicaRequest.ConfigurationEntry.value:type_name -> orchestrationpb.ReplicaInfo + 2, // 20: orchestrationpb.StartClientRequest.ClientsEntry.value:type_name -> orchestrationpb.ClientOpts + 1, // 21: orchestrationpb.StartClientRequest.ConfigurationEntry.value:type_name -> orchestrationpb.ReplicaInfo + 22, // [22:22] is the sub-list for method output_type + 22, // [22:22] is the sub-list for method input_type + 22, // [22:22] is the sub-list for extension type_name + 22, // [22:22] is the sub-list for extension extendee + 0, // [0:22] is the sub-list for field type_name } func init() { file_internal_proto_orchestrationpb_orchestration_proto_init() } @@ -1301,7 +1349,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*ReplicaOpts); i { case 0: return &v.state @@ -1313,7 +1361,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*ReplicaInfo); i { case 0: return &v.state @@ -1325,7 +1373,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*ClientOpts); i { case 0: return &v.state @@ -1337,7 +1385,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*ReplicaConfiguration); i { case 0: return &v.state @@ -1349,7 +1397,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*CreateReplicaRequest); i { case 0: return &v.state @@ -1361,7 +1409,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*CreateReplicaResponse); i { case 0: return &v.state @@ -1373,7 +1421,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[6].Exporter = func(v any, i int) any { switch v := v.(*StartReplicaRequest); i { case 0: return &v.state @@ -1385,7 +1433,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[7].Exporter = func(v any, i int) any { switch v := v.(*StartReplicaResponse); i { case 0: return &v.state @@ -1397,7 +1445,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[8].Exporter = func(v any, i int) any { switch v := v.(*StopReplicaRequest); i { case 0: return &v.state @@ -1409,7 +1457,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[9].Exporter = func(v any, i int) any { switch v := v.(*StopReplicaResponse); i { case 0: return &v.state @@ -1421,7 +1469,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[10].Exporter = func(v any, i int) any { switch v := v.(*StartClientRequest); i { case 0: return &v.state @@ -1433,7 +1481,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[11].Exporter = func(v any, i int) any { switch v := v.(*StartClientResponse); i { case 0: return &v.state @@ -1445,7 +1493,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[12].Exporter = func(v any, i int) any { switch v := v.(*StopClientRequest); i { case 0: return &v.state @@ -1457,7 +1505,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[13].Exporter = func(v any, i int) any { switch v := v.(*StopClientResponse); i { case 0: return &v.state @@ -1469,7 +1517,7 @@ func file_internal_proto_orchestrationpb_orchestration_proto_init() { return nil } } - file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + file_internal_proto_orchestrationpb_orchestration_proto_msgTypes[14].Exporter = func(v any, i int) any { switch v := v.(*QuitRequest); i { case 0: return &v.state diff --git a/internal/proto/orchestrationpb/orchestration.proto b/internal/proto/orchestrationpb/orchestration.proto index 1ea35340..7467fee8 100644 --- a/internal/proto/orchestrationpb/orchestration.proto +++ b/internal/proto/orchestrationpb/orchestration.proto @@ -53,6 +53,14 @@ message ReplicaOpts { repeated string Modules = 21; // locations of the replicas map LocationInfo = 22; + // The number of pipes in pipelining mode. Leave as zero to disable pipelining mode. + uint32 Pipes = 23; + // The ordering scheme for pipelining mode. Default is "sequential". + string PipelineOrdering = 24; + // Fixed or dynamic view duration per pipe. Default is "fixed". + string ViewDurationMethod = 25; + // Latency induced by all replicas. + google.protobuf.Duration HackyLatency = 26; } // ReplicaInfo is the information that the replicas need about each other. diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index c42ea030..21c4f8ae 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/relab/hotstuff/committer" "github.com/relab/hotstuff/consensus" "github.com/relab/hotstuff/eventloop" "github.com/relab/hotstuff/modules" @@ -35,7 +36,7 @@ func TestModules(t *testing.T, ctrl *gomock.Controller, id hotstuff.ID, _ hotstu acceptor.EXPECT().Proposed(gomock.Any()).AnyTimes() executor := mocks.NewMockExecutor(ctrl) - executor.EXPECT().Exec(gomock.AssignableToTypeOf(hotstuff.Command(""))).AnyTimes() + executor.EXPECT().Exec(0, gomock.AssignableToTypeOf(hotstuff.Command(""))).AnyTimes() forkHandler := mocks.NewMockForkHandler(ctrl) @@ -53,9 +54,10 @@ func TestModules(t *testing.T, ctrl *gomock.Controller, id hotstuff.ID, _ hotstu synchronizer.EXPECT().ViewContext().AnyTimes().Return(context.Background()) builder.Add( - eventloop.New(100), + eventloop.NewScoped(100, 0), logging.New(fmt.Sprintf("hs%d", id)), blockchain.New(), + committer.New(), mocks.NewMockConsensus(ctrl), consensus.NewVotingMachine(), leaderrotation.NewFixed(1), @@ -69,6 +71,43 @@ func TestModules(t *testing.T, ctrl *gomock.Controller, id hotstuff.ID, _ hotstu ) } +// TestModules registers default modules for testing to the given builder. +func TestModulesScoped(t *testing.T, ctrl *gomock.Controller, id hotstuff.ID, _ hotstuff.PrivateKey, builder *modules.Builder, pipeCount int) { + t.Helper() + + acceptor := mocks.NewMockAcceptor(ctrl) + acceptor.EXPECT().Accept(gomock.AssignableToTypeOf(hotstuff.Command(""))).AnyTimes().Return(true) + acceptor.EXPECT().Proposed(gomock.Any()).AnyTimes() + + executor := mocks.NewMockExecutor(ctrl) + executor.EXPECT().Exec(0, gomock.AssignableToTypeOf(hotstuff.Command(""))).AnyTimes() + + forkHandler := mocks.NewMockForkHandler(ctrl) + + commandQ := mocks.NewMockCommandQueue(ctrl) + commandQ.EXPECT().Get(gomock.Any()).AnyTimes().Return(hotstuff.Command("foo"), true) + + signer := crypto.NewCache(ecdsa.New(), 10) + + config := mocks.NewMockConfiguration(ctrl) + config.EXPECT().Len().AnyTimes().Return(1) + config.EXPECT().QuorumSize().AnyTimes().Return(3) + + // TODO: Check if this code still runs after implementing pipelining + builder.Add( + eventloop.NewScoped(100, 0), + logging.New(fmt.Sprintf("hs%d", id)), + blockchain.New(), + committer.New(), + config, + signer, + acceptor, + modules.ExtendedExecutor(executor), + commandQ, + modules.ExtendedForkHandler(forkHandler), + ) +} + // BuilderList is a helper type to perform actions on a set of builders. type BuilderList []*modules.Builder @@ -136,6 +175,30 @@ func CreateBuilders(t *testing.T, ctrl *gomock.Controller, n int, keys ...hotstu return builders } +// TODO: Complete the implementation. +func CreateBuildersScoped(t *testing.T, ctrl *gomock.Controller, n int, pipeCount int, keys ...hotstuff.PrivateKey) (builders BuilderList) { + t.Helper() + network := twins.NewSimpleNetwork() + builders = make([]*modules.Builder, n) + for i := 0; i < n; i++ { + id := hotstuff.ID(i + 1) + var key hotstuff.PrivateKey + if i < len(keys) { + key = keys[i] + } else { + key = GenerateECDSAKey(t) + } + + builder := network.GetNodeBuilder(twins.NodeID{ReplicaID: id, NetworkID: uint32(id)}, key) + builder.EnablePipelining(pipeCount) + builder.Add(network.NewConfiguration()) + TestModulesScoped(t, ctrl, id, key, &builder, pipeCount) + builder.Add(network.NewConfiguration()) + builders[i] = &builder + } + return builders +} + // CreateTCPListener creates a net.Listener on a random port. func CreateTCPListener(t *testing.T) net.Listener { t.Helper() @@ -185,7 +248,11 @@ func CreateTimeouts(t *testing.T, view hotstuff.View, signers []modules.Crypto) ID: signer(sig), View: view, ViewSignature: sig, - SyncInfo: hotstuff.NewSyncInfo().WithQC(hotstuff.NewQuorumCert(nil, 0, hotstuff.GetGenesis().Hash())), + SyncInfo: hotstuff.NewSyncInfo(hotstuff.NullPipe).WithQC(hotstuff.NewQuorumCert( + nil, + 0, + hotstuff.NullPipe, // TODO: Verify if this code conflicts with pipelining + hotstuff.GetGenesis().Hash())), }) } for i := range timeouts { @@ -281,7 +348,7 @@ func GenerateKeys(t *testing.T, n int, keyFunc func(t *testing.T) hotstuff.Priva // NewProposeMsg wraps a new block in a ProposeMsg. func NewProposeMsg(parent hotstuff.Hash, qc hotstuff.QuorumCert, cmd hotstuff.Command, view hotstuff.View, id hotstuff.ID) hotstuff.ProposeMsg { - return hotstuff.ProposeMsg{ID: id, Block: hotstuff.NewBlock(parent, qc, cmd, view, id)} + return hotstuff.ProposeMsg{ID: id, Block: hotstuff.NewBlock(parent, qc, cmd, view, id, 0)} } type leaderRotation struct { diff --git a/leaderrotation/carousel.go b/leaderrotation/carousel.go index f7c2a29b..d84f1a86 100644 --- a/leaderrotation/carousel.go +++ b/leaderrotation/carousel.go @@ -22,8 +22,8 @@ type carousel struct { logger logging.Logger } -func (c *carousel) InitModule(mods *modules.Core) { - mods.Get( +func (c *carousel) InitModule(mods *modules.Core, _ modules.ScopeInfo) { + mods.GetScoped(c, &c.blockChain, &c.configuration, &c.consensus, @@ -57,7 +57,7 @@ func (c carousel) GetLeader(round hotstuff.View) hotstuff.ID { for ok && i < f && block != hotstuff.GetGenesis() { lastAuthors.Add(block.Proposer()) - block, ok = c.blockChain.Get(block.Parent()) + block, ok = c.blockChain.Get(block.Parent(), block.Pipe()) i++ } diff --git a/leaderrotation/reputation.go b/leaderrotation/reputation.go index 7e5e2c4c..6a579217 100644 --- a/leaderrotation/reputation.go +++ b/leaderrotation/reputation.go @@ -28,8 +28,8 @@ type repBased struct { // InitModule gives the module a reference to the Core object. // It also allows the module to set module options using the OptionsBuilder -func (r *repBased) InitModule(mods *modules.Core) { - mods.Get( +func (r *repBased) InitModule(mods *modules.Core, _ modules.ScopeInfo) { + mods.GetScoped(r, &r.configuration, &r.consensus, &r.opts, diff --git a/leaderrotation/roundrobin.go b/leaderrotation/roundrobin.go index 8b2523e7..fb91a2b1 100644 --- a/leaderrotation/roundrobin.go +++ b/leaderrotation/roundrobin.go @@ -13,7 +13,7 @@ type roundRobin struct { configuration modules.Configuration } -func (rr *roundRobin) InitModule(mods *modules.Core) { +func (rr *roundRobin) InitModule(mods *modules.Core, info modules.ScopeInfo) { mods.Get(&rr.configuration) } diff --git a/logged_run.sh b/logged_run.sh new file mode 100644 index 00000000..3207c271 --- /dev/null +++ b/logged_run.sh @@ -0,0 +1,7 @@ +DT=$(date '+%Y-%m-%d_%H-%M-%S') +DIR="debug_logs" +FILE="$DIR/$DT.txt" +make +mkdir $DIR +echo "Writing to $FILE" +./hotstuff --log-level=debug $@ run &>> $FILE diff --git a/metrics/clientlatency.go b/metrics/clientlatency.go index d3f73810..ccb75085 100644 --- a/metrics/clientlatency.go +++ b/metrics/clientlatency.go @@ -25,9 +25,9 @@ type ClientLatency struct { } // InitModule gives the module access to the other modules. -func (lr *ClientLatency) InitModule(mods *modules.Core) { +func (lr *ClientLatency) InitModule(mods *modules.Core, _ modules.ScopeInfo) { var ( - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop logger logging.Logger ) diff --git a/metrics/datalogger.go b/metrics/datalogger.go index 22a217c4..2170b666 100644 --- a/metrics/datalogger.go +++ b/metrics/datalogger.go @@ -36,7 +36,7 @@ func NewJSONLogger(wr io.Writer) (Logger, error) { } // InitModule initializes the metrics logger module. -func (dl *jsonLogger) InitModule(mods *modules.Core) { +func (dl *jsonLogger) InitModule(mods *modules.Core, _ modules.ScopeInfo) { mods.Get(&dl.logger) } diff --git a/metrics/debug.go b/metrics/debug.go new file mode 100644 index 00000000..ae26efac --- /dev/null +++ b/metrics/debug.go @@ -0,0 +1,88 @@ +package metrics + +import ( + "time" + + "github.com/relab/hotstuff" + "github.com/relab/hotstuff/debug" + "github.com/relab/hotstuff/eventloop" + "github.com/relab/hotstuff/logging" + "github.com/relab/hotstuff/metrics/types" + "github.com/relab/hotstuff/modules" +) + +func init() { + RegisterReplicaMetric("debug", func() any { + return &DebugMetrics{ + commitHalts: make(map[hotstuff.Pipe]int), + rejectedCommands: make(map[hotstuff.Pipe]int), + } + }) +} + +// ViewTimeouts is a metric that measures the number of view timeouts that happen. +type DebugMetrics struct { + metricsLogger Logger + opts *modules.Options + pipeCount int + + // metrics + commitHalts map[hotstuff.Pipe]int + rejectedCommands map[hotstuff.Pipe]int +} + +// InitModule gives the module access to the other modules. +func (db *DebugMetrics) InitModule(mods *modules.Core, info modules.ScopeInfo) { + var ( + eventLoop *eventloop.ScopedEventLoop + logger logging.Logger + ) + + mods.Get( + &db.metricsLogger, + &db.opts, + &eventLoop, + &logger, + ) + + db.pipeCount = info.ScopeCount + + logger.Info("DebugMetrics enabled.") + + eventLoop.RegisterHandler(debug.CommitHaltEvent{}, func(event any) { + halt := event.(debug.CommitHaltEvent) + db.commitHalts[halt.Pipe]++ + }) + + eventLoop.RegisterHandler(debug.CommandRejectedEvent{}, func(event any) { + reject := event.(debug.CommandRejectedEvent) + db.rejectedCommands[reject.Pipe]++ + }) + + eventLoop.RegisterObserver(types.TickEvent{}, func(event any) { + db.tick(event.(types.TickEvent)) + }) +} + +func (db *DebugMetrics) tick(_ types.TickEvent) { + var maxCi hotstuff.Pipe = 1 + var start hotstuff.Pipe = hotstuff.NullPipe + + if db.pipeCount > 0 { + maxCi = hotstuff.Pipe(db.pipeCount) + 1 + start++ + } + + for pipe := start; pipe < maxCi; pipe++ { + db.metricsLogger.Log(&types.DebugMeasurement{ + Event: types.NewReplicaEvent(uint32(db.opts.ID()), time.Now()), + Pipe: uint32(pipe), + CommitHalts: uint32(db.commitHalts[pipe]), + RejectedCommands: uint32(db.rejectedCommands[pipe]), + }) + + db.commitHalts[pipe] = 0 + db.rejectedCommands[pipe] = 0 + } + +} diff --git a/metrics/plotting/reader.go b/metrics/plotting/reader.go index 620a156e..46183fb9 100644 --- a/metrics/plotting/reader.go +++ b/metrics/plotting/reader.go @@ -42,6 +42,7 @@ func (r *Reader) ReadAll() error { } for decoder.More() { + var b json.RawMessage err = decoder.Decode(&b) if err != nil { diff --git a/metrics/plotting/throughput.go b/metrics/plotting/throughput.go index 44d321e2..3bbfde98 100644 --- a/metrics/plotting/throughput.go +++ b/metrics/plotting/throughput.go @@ -54,6 +54,7 @@ func (p *ThroughputPlot) PlotAverage(filename string, measurementInterval time.D return avgThroughput(p, measurementInterval) }) } + return GonumPlot(filename, xlabel, ylabel, func(plt *plot.Plot) error { if err := plotutil.AddLinePoints(plt, avgThroughput(p, measurementInterval)); err != nil { return fmt.Errorf("failed to add line plot: %w", err) diff --git a/metrics/throughput.go b/metrics/throughput.go index 3e1a30b5..8c00532c 100644 --- a/metrics/throughput.go +++ b/metrics/throughput.go @@ -14,7 +14,10 @@ import ( func init() { RegisterReplicaMetric("throughput", func() any { - return &Throughput{} + return &Throughput{ + commitCount: make(map[hotstuff.Pipe]uint64), + commandCount: make(map[hotstuff.Pipe]uint64), + } }) } @@ -22,18 +25,18 @@ func init() { type Throughput struct { metricsLogger Logger opts *modules.Options + pipeCount int - commitCount uint64 - commandCount uint64 + commitCount map[hotstuff.Pipe]uint64 + commandCount map[hotstuff.Pipe]uint64 } // InitModule gives the module access to the other modules. -func (t *Throughput) InitModule(mods *modules.Core) { +func (t *Throughput) InitModule(mods *modules.Core, info modules.ScopeInfo) { var ( - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop logger logging.Logger ) - mods.Get( &t.metricsLogger, &t.opts, @@ -41,9 +44,11 @@ func (t *Throughput) InitModule(mods *modules.Core) { &logger, ) + t.pipeCount = info.ScopeCount + eventLoop.RegisterHandler(hotstuff.CommitEvent{}, func(event any) { commitEvent := event.(hotstuff.CommitEvent) - t.recordCommit(commitEvent.Commands) + t.recordCommit(commitEvent.Pipe, commitEvent.Commands) }) eventLoop.RegisterObserver(types.TickEvent{}, func(event any) { @@ -53,21 +58,48 @@ func (t *Throughput) InitModule(mods *modules.Core) { logger.Info("Throughput metric enabled") } -func (t *Throughput) recordCommit(commands int) { - t.commitCount++ - t.commandCount += uint64(commands) +func (t *Throughput) recordCommit(pipe hotstuff.Pipe, commands int) { + t.commitCount[pipe]++ + t.commandCount[pipe] += uint64(commands) } func (t *Throughput) tick(tick types.TickEvent) { now := time.Now() - event := &types.ThroughputMeasurement{ - Event: types.NewReplicaEvent(uint32(t.opts.ID()), now), - Commits: t.commitCount, - Commands: t.commandCount, - Duration: durationpb.New(now.Sub(tick.LastTick)), + + var totalCommands uint64 = 0 + var totalCommits uint64 = 0 + var maxCi hotstuff.Pipe = 1 + var start hotstuff.Pipe = hotstuff.NullPipe + + if t.pipeCount > 0 { + maxCi = hotstuff.Pipe(t.pipeCount) + 1 + start++ + } + + for pipe := start; pipe < maxCi; pipe++ { + event := &types.ThroughputMeasurement{ + Event: types.NewReplicaEvent(uint32(t.opts.ID()), now), + Commits: t.commitCount[pipe], + Commands: t.commandCount[pipe], + Duration: durationpb.New(now.Sub(tick.LastTick)), + Pipe: uint32(pipe), + } + t.metricsLogger.Log(event) + totalCommands += t.commandCount[pipe] + totalCommits += t.commitCount[pipe] + // reset count for next tick + t.commandCount[pipe] = 0 + t.commitCount[pipe] = 0 + } + + if t.pipeCount > 0 { + event := &types.TotalThroughputMeasurement{ + Event: types.NewReplicaEvent(uint32(t.opts.ID()), now), + Commits: totalCommits, + Commands: totalCommands, + Duration: durationpb.New(now.Sub(tick.LastTick)), + PipeCount: uint32(t.pipeCount), + } + t.metricsLogger.Log(event) } - t.metricsLogger.Log(event) - // reset count for next tick - t.commandCount = 0 - t.commitCount = 0 } diff --git a/metrics/ticker.go b/metrics/ticker.go index 65b3f860..2ae9791f 100644 --- a/metrics/ticker.go +++ b/metrics/ticker.go @@ -21,8 +21,8 @@ func NewTicker(interval time.Duration) *Ticker { } // InitModule gives the module access to the other modules. -func (t *Ticker) InitModule(mods *modules.Core) { - var eventLoop *eventloop.EventLoop +func (t *Ticker) InitModule(mods *modules.Core, _ modules.ScopeInfo) { + var eventLoop *eventloop.ScopedEventLoop mods.Get(&eventLoop) diff --git a/metrics/timeouts.go b/metrics/timeouts.go index a2ce5a02..7913c3de 100644 --- a/metrics/timeouts.go +++ b/metrics/timeouts.go @@ -3,6 +3,7 @@ package metrics import ( "time" + "github.com/relab/hotstuff" "github.com/relab/hotstuff/eventloop" "github.com/relab/hotstuff/logging" "github.com/relab/hotstuff/metrics/types" @@ -26,9 +27,9 @@ type ViewTimeouts struct { } // InitModule gives the module access to the other modules. -func (vt *ViewTimeouts) InitModule(mods *modules.Core) { +func (vt *ViewTimeouts) InitModule(mods *modules.Core, info modules.ScopeInfo) { var ( - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop logger logging.Logger ) @@ -41,9 +42,18 @@ func (vt *ViewTimeouts) InitModule(mods *modules.Core) { logger.Info("ViewTimeouts metric enabled.") - eventLoop.RegisterHandler(synchronizer.ViewChangeEvent{}, func(event any) { - vt.viewChange(event.(synchronizer.ViewChangeEvent)) - }) + if info.IsPipeliningEnabled { + for pipe := hotstuff.Pipe(1); pipe <= hotstuff.Pipe(info.ScopeCount); pipe++ { + eventLoop.RegisterHandler(synchronizer.ViewChangeEvent{}, func(event any) { + vt.viewChange(event.(synchronizer.ViewChangeEvent)) + }, eventloop.RespondToScope(pipe)) + } + } else { + + eventLoop.RegisterHandler(synchronizer.ViewChangeEvent{}, func(event any) { + vt.viewChange(event.(synchronizer.ViewChangeEvent)) + }) + } eventLoop.RegisterObserver(types.TickEvent{}, func(event any) { vt.tick(event.(types.TickEvent)) diff --git a/metrics/types/types.pb.go b/metrics/types/types.pb.go index 77849739..3f27975f 100644 --- a/metrics/types/types.pb.go +++ b/metrics/types/types.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.19.4 +// protoc-gen-go v1.34.2 +// protoc v5.28.2 // source: metrics/types/types.proto package types @@ -144,6 +144,7 @@ type ThroughputMeasurement struct { Commits uint64 `protobuf:"varint,2,opt,name=Commits,proto3" json:"Commits,omitempty"` Commands uint64 `protobuf:"varint,3,opt,name=Commands,proto3" json:"Commands,omitempty"` Duration *durationpb.Duration `protobuf:"bytes,4,opt,name=Duration,proto3" json:"Duration,omitempty"` + Pipe uint32 `protobuf:"varint,5,opt,name=Pipe,proto3" json:"Pipe,omitempty"` } func (x *ThroughputMeasurement) Reset() { @@ -206,6 +207,92 @@ func (x *ThroughputMeasurement) GetDuration() *durationpb.Duration { return nil } +func (x *ThroughputMeasurement) GetPipe() uint32 { + if x != nil { + return x.Pipe + } + return 0 +} + +type TotalThroughputMeasurement struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Event *Event `protobuf:"bytes,1,opt,name=Event,proto3" json:"Event,omitempty"` + Commits uint64 `protobuf:"varint,2,opt,name=Commits,proto3" json:"Commits,omitempty"` + Commands uint64 `protobuf:"varint,3,opt,name=Commands,proto3" json:"Commands,omitempty"` + Duration *durationpb.Duration `protobuf:"bytes,4,opt,name=Duration,proto3" json:"Duration,omitempty"` + PipeCount uint32 `protobuf:"varint,5,opt,name=PipeCount,proto3" json:"PipeCount,omitempty"` +} + +func (x *TotalThroughputMeasurement) Reset() { + *x = TotalThroughputMeasurement{} + if protoimpl.UnsafeEnabled { + mi := &file_metrics_types_types_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TotalThroughputMeasurement) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TotalThroughputMeasurement) ProtoMessage() {} + +func (x *TotalThroughputMeasurement) ProtoReflect() protoreflect.Message { + mi := &file_metrics_types_types_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TotalThroughputMeasurement.ProtoReflect.Descriptor instead. +func (*TotalThroughputMeasurement) Descriptor() ([]byte, []int) { + return file_metrics_types_types_proto_rawDescGZIP(), []int{3} +} + +func (x *TotalThroughputMeasurement) GetEvent() *Event { + if x != nil { + return x.Event + } + return nil +} + +func (x *TotalThroughputMeasurement) GetCommits() uint64 { + if x != nil { + return x.Commits + } + return 0 +} + +func (x *TotalThroughputMeasurement) GetCommands() uint64 { + if x != nil { + return x.Commands + } + return 0 +} + +func (x *TotalThroughputMeasurement) GetDuration() *durationpb.Duration { + if x != nil { + return x.Duration + } + return nil +} + +func (x *TotalThroughputMeasurement) GetPipeCount() uint32 { + if x != nil { + return x.PipeCount + } + return 0 +} + type LatencyMeasurement struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -220,7 +307,7 @@ type LatencyMeasurement struct { func (x *LatencyMeasurement) Reset() { *x = LatencyMeasurement{} if protoimpl.UnsafeEnabled { - mi := &file_metrics_types_types_proto_msgTypes[3] + mi := &file_metrics_types_types_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -233,7 +320,7 @@ func (x *LatencyMeasurement) String() string { func (*LatencyMeasurement) ProtoMessage() {} func (x *LatencyMeasurement) ProtoReflect() protoreflect.Message { - mi := &file_metrics_types_types_proto_msgTypes[3] + mi := &file_metrics_types_types_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -246,7 +333,7 @@ func (x *LatencyMeasurement) ProtoReflect() protoreflect.Message { // Deprecated: Use LatencyMeasurement.ProtoReflect.Descriptor instead. func (*LatencyMeasurement) Descriptor() ([]byte, []int) { - return file_metrics_types_types_proto_rawDescGZIP(), []int{3} + return file_metrics_types_types_proto_rawDescGZIP(), []int{4} } func (x *LatencyMeasurement) GetEvent() *Event { @@ -292,7 +379,7 @@ type ViewTimeouts struct { func (x *ViewTimeouts) Reset() { *x = ViewTimeouts{} if protoimpl.UnsafeEnabled { - mi := &file_metrics_types_types_proto_msgTypes[4] + mi := &file_metrics_types_types_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -305,7 +392,7 @@ func (x *ViewTimeouts) String() string { func (*ViewTimeouts) ProtoMessage() {} func (x *ViewTimeouts) ProtoReflect() protoreflect.Message { - mi := &file_metrics_types_types_proto_msgTypes[4] + mi := &file_metrics_types_types_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -318,7 +405,7 @@ func (x *ViewTimeouts) ProtoReflect() protoreflect.Message { // Deprecated: Use ViewTimeouts.ProtoReflect.Descriptor instead. func (*ViewTimeouts) Descriptor() ([]byte, []int) { - return file_metrics_types_types_proto_rawDescGZIP(), []int{4} + return file_metrics_types_types_proto_rawDescGZIP(), []int{5} } func (x *ViewTimeouts) GetEvent() *Event { @@ -342,6 +429,77 @@ func (x *ViewTimeouts) GetTimeouts() uint64 { return 0 } +type DebugMeasurement struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Event *Event `protobuf:"bytes,1,opt,name=Event,proto3" json:"Event,omitempty"` + Pipe uint32 `protobuf:"varint,2,opt,name=Pipe,proto3" json:"Pipe,omitempty"` + CommitHalts uint32 `protobuf:"varint,3,opt,name=CommitHalts,proto3" json:"CommitHalts,omitempty"` + RejectedCommands uint32 `protobuf:"varint,4,opt,name=RejectedCommands,proto3" json:"RejectedCommands,omitempty"` +} + +func (x *DebugMeasurement) Reset() { + *x = DebugMeasurement{} + if protoimpl.UnsafeEnabled { + mi := &file_metrics_types_types_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DebugMeasurement) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DebugMeasurement) ProtoMessage() {} + +func (x *DebugMeasurement) ProtoReflect() protoreflect.Message { + mi := &file_metrics_types_types_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DebugMeasurement.ProtoReflect.Descriptor instead. +func (*DebugMeasurement) Descriptor() ([]byte, []int) { + return file_metrics_types_types_proto_rawDescGZIP(), []int{6} +} + +func (x *DebugMeasurement) GetEvent() *Event { + if x != nil { + return x.Event + } + return nil +} + +func (x *DebugMeasurement) GetPipe() uint32 { + if x != nil { + return x.Pipe + } + return 0 +} + +func (x *DebugMeasurement) GetCommitHalts() uint32 { + if x != nil { + return x.CommitHalts + } + return 0 +} + +func (x *DebugMeasurement) GetRejectedCommands() uint32 { + if x != nil { + return x.RejectedCommands + } + return 0 +} + var File_metrics_types_types_proto protoreflect.FileDescriptor var file_metrics_types_types_proto_rawDesc = []byte{ @@ -361,7 +519,7 @@ var file_metrics_types_types_proto_rawDesc = []byte{ 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x22, 0xa8, 0x01, 0x0a, 0x15, 0x54, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x70, 0x75, 0x74, 0x4d, + 0x22, 0xbc, 0x01, 0x0a, 0x15, 0x54, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x70, 0x75, 0x74, 0x4d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x22, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x18, @@ -371,25 +529,49 @@ var file_metrics_types_types_proto_rawDesc = []byte{ 0x61, 0x6e, 0x64, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x84, 0x01, 0x0a, 0x12, - 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x4d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x12, 0x22, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x0c, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, - 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, - 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, - 0x12, 0x1a, 0x0a, 0x08, 0x56, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x01, 0x52, 0x08, 0x56, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, - 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x43, 0x6f, 0x75, - 0x6e, 0x74, 0x22, 0x64, 0x0a, 0x0c, 0x56, 0x69, 0x65, 0x77, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, - 0x74, 0x73, 0x12, 0x22, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x0c, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, - 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x69, 0x65, 0x77, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x56, 0x69, 0x65, 0x77, 0x73, 0x12, 0x1a, 0x0a, 0x08, - 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, - 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x73, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x65, 0x6c, 0x61, 0x62, 0x2f, 0x68, 0x6f, 0x74, - 0x73, 0x74, 0x75, 0x66, 0x66, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, 0x74, 0x79, - 0x70, 0x65, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x52, 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x50, + 0x69, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x50, 0x69, 0x70, 0x65, 0x22, + 0xcb, 0x01, 0x0a, 0x1a, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, + 0x70, 0x75, 0x74, 0x4d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x22, + 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, + 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x07, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x1a, 0x0a, 0x08, + 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, + 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x44, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x1c, 0x0a, 0x09, 0x50, 0x69, 0x70, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x09, 0x50, 0x69, 0x70, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x84, 0x01, + 0x0a, 0x12, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x4d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x22, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x52, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x4c, 0x61, 0x74, 0x65, + 0x6e, 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x4c, 0x61, 0x74, 0x65, 0x6e, + 0x63, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x56, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x56, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x64, 0x0a, 0x0c, 0x56, 0x69, 0x65, 0x77, 0x54, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x73, 0x12, 0x22, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x52, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x56, 0x69, 0x65, 0x77, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x56, 0x69, 0x65, 0x77, 0x73, 0x12, 0x1a, + 0x0a, 0x08, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x08, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x73, 0x22, 0x98, 0x01, 0x0a, 0x10, 0x44, + 0x65, 0x62, 0x75, 0x67, 0x4d, 0x65, 0x61, 0x73, 0x75, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, + 0x22, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, + 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x69, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x04, 0x50, 0x69, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x6d, 0x69, + 0x74, 0x48, 0x61, 0x6c, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x43, 0x6f, + 0x6d, 0x6d, 0x69, 0x74, 0x48, 0x61, 0x6c, 0x74, 0x73, 0x12, 0x2a, 0x0a, 0x10, 0x52, 0x65, 0x6a, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x10, 0x52, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x73, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x65, 0x6c, 0x61, 0x62, 0x2f, 0x68, 0x6f, 0x74, 0x73, 0x74, 0x75, + 0x66, 0x66, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -404,28 +586,33 @@ func file_metrics_types_types_proto_rawDescGZIP() []byte { return file_metrics_types_types_proto_rawDescData } -var file_metrics_types_types_proto_msgTypes = make([]protoimpl.MessageInfo, 5) -var file_metrics_types_types_proto_goTypes = []interface{}{ - (*StartEvent)(nil), // 0: types.StartEvent - (*Event)(nil), // 1: types.Event - (*ThroughputMeasurement)(nil), // 2: types.ThroughputMeasurement - (*LatencyMeasurement)(nil), // 3: types.LatencyMeasurement - (*ViewTimeouts)(nil), // 4: types.ViewTimeouts - (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp - (*durationpb.Duration)(nil), // 6: google.protobuf.Duration +var file_metrics_types_types_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_metrics_types_types_proto_goTypes = []any{ + (*StartEvent)(nil), // 0: types.StartEvent + (*Event)(nil), // 1: types.Event + (*ThroughputMeasurement)(nil), // 2: types.ThroughputMeasurement + (*TotalThroughputMeasurement)(nil), // 3: types.TotalThroughputMeasurement + (*LatencyMeasurement)(nil), // 4: types.LatencyMeasurement + (*ViewTimeouts)(nil), // 5: types.ViewTimeouts + (*DebugMeasurement)(nil), // 6: types.DebugMeasurement + (*timestamppb.Timestamp)(nil), // 7: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 8: google.protobuf.Duration } var file_metrics_types_types_proto_depIdxs = []int32{ 1, // 0: types.StartEvent.Event:type_name -> types.Event - 5, // 1: types.Event.Timestamp:type_name -> google.protobuf.Timestamp + 7, // 1: types.Event.Timestamp:type_name -> google.protobuf.Timestamp 1, // 2: types.ThroughputMeasurement.Event:type_name -> types.Event - 6, // 3: types.ThroughputMeasurement.Duration:type_name -> google.protobuf.Duration - 1, // 4: types.LatencyMeasurement.Event:type_name -> types.Event - 1, // 5: types.ViewTimeouts.Event:type_name -> types.Event - 6, // [6:6] is the sub-list for method output_type - 6, // [6:6] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 8, // 3: types.ThroughputMeasurement.Duration:type_name -> google.protobuf.Duration + 1, // 4: types.TotalThroughputMeasurement.Event:type_name -> types.Event + 8, // 5: types.TotalThroughputMeasurement.Duration:type_name -> google.protobuf.Duration + 1, // 6: types.LatencyMeasurement.Event:type_name -> types.Event + 1, // 7: types.ViewTimeouts.Event:type_name -> types.Event + 1, // 8: types.DebugMeasurement.Event:type_name -> types.Event + 9, // [9:9] is the sub-list for method output_type + 9, // [9:9] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_metrics_types_types_proto_init() } @@ -434,7 +621,7 @@ func file_metrics_types_types_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_metrics_types_types_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_metrics_types_types_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*StartEvent); i { case 0: return &v.state @@ -446,7 +633,7 @@ func file_metrics_types_types_proto_init() { return nil } } - file_metrics_types_types_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_metrics_types_types_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*Event); i { case 0: return &v.state @@ -458,7 +645,7 @@ func file_metrics_types_types_proto_init() { return nil } } - file_metrics_types_types_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_metrics_types_types_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*ThroughputMeasurement); i { case 0: return &v.state @@ -470,7 +657,19 @@ func file_metrics_types_types_proto_init() { return nil } } - file_metrics_types_types_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_metrics_types_types_proto_msgTypes[3].Exporter = func(v any, i int) any { + switch v := v.(*TotalThroughputMeasurement); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metrics_types_types_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*LatencyMeasurement); i { case 0: return &v.state @@ -482,7 +681,7 @@ func file_metrics_types_types_proto_init() { return nil } } - file_metrics_types_types_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_metrics_types_types_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*ViewTimeouts); i { case 0: return &v.state @@ -494,6 +693,18 @@ func file_metrics_types_types_proto_init() { return nil } } + file_metrics_types_types_proto_msgTypes[6].Exporter = func(v any, i int) any { + switch v := v.(*DebugMeasurement); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -501,7 +712,7 @@ func file_metrics_types_types_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_metrics_types_types_proto_rawDesc, NumEnums: 0, - NumMessages: 5, + NumMessages: 7, NumExtensions: 0, NumServices: 0, }, diff --git a/metrics/types/types.proto b/metrics/types/types.proto index 69602029..51c45dcf 100644 --- a/metrics/types/types.proto +++ b/metrics/types/types.proto @@ -23,6 +23,15 @@ message ThroughputMeasurement { uint64 Commits = 2; uint64 Commands = 3; google.protobuf.Duration Duration = 4; + uint32 Pipe = 5; +} + +message TotalThroughputMeasurement { + Event Event = 1; + uint64 Commits = 2; + uint64 Commands = 3; + google.protobuf.Duration Duration = 4; + uint32 PipeCount = 5; } message LatencyMeasurement { @@ -39,3 +48,10 @@ message ViewTimeouts { // Number of view timeouts. uint64 Timeouts = 3; } + +message DebugMeasurement { + Event Event = 1; + uint32 Pipe = 2; + uint32 CommitHalts = 3; + uint32 RejectedCommands = 4; +} diff --git a/modules/core.go b/modules/core.go index 86bdca45..5e1ff94e 100644 --- a/modules/core.go +++ b/modules/core.go @@ -39,15 +39,25 @@ import ( "github.com/relab/hotstuff" ) +type moduleScope []Module + +type ScopeInfo struct { + IsPipeliningEnabled bool + ScopeCount int + ModuleScope hotstuff.Pipe +} + // Module is an interface for initializing modules. type Module interface { - InitModule(mods *Core) + InitModule(mods *Core, opt ScopeInfo) } // Core is the base of the module system. // It contains only a few core modules that are shared between replicas and clients. type Core struct { - modules []any + staticModules []any + scopedModules map[hotstuff.Pipe][]any + isPipeliningEnabled bool } // TryGet attempts to find a module for ptr. @@ -75,7 +85,7 @@ func (mods Core) TryGet(ptr any) bool { panic("only pointer values allowed") } - for _, m := range mods.modules { + for _, m := range mods.staticModules { mv := reflect.ValueOf(m) if mv.Type().AssignableTo(pt.Elem()) { v.Elem().Set(mv) @@ -110,11 +120,165 @@ func (mods *Core) Get(pointers ...any) { } } +// GetScoped does the same as Get and additionally searches for pointers in the same scope as moduleInScope. +// If pipelining is not enabled, Get is called internally instead. +// +// NOTE: pointers must only contain non-nil pointers to types that have been provided to the module system +// as a scoped module. +// GetScoped panics if one of the given arguments is not a pointer, if a compatible module is not found, +// if the module was not in a scope or if the module was not in the same scope as moduleInScope. +// +// Example: +// +// type OtherModule interface { +// Foo() +// } +// +// type MyModuleImpl struct { +// otherModule OtherModule +// } +// +// func (m *MyModuleImpl) InitModule(mods *modules.Core, opt modules.InitOptions) { +// mods.GetScoped(m, &m.otherModule) // Requires an OtherModule from the same pipe +// } +// +// func main() { +// // TODO: Fix this doc +// pipeCount := 3 +// +// builder := modules.NewBuilder(0, nil) +// builder.EnablePipelining(pipeCount) +// builder.AddScoped(NewMyModuleImpl) +// builder.AddScoped(NewOtherModuleImpl) +// builder.Build() // InitModule is called here +// } +func (mods *Core) GetScoped(moduleInScope Module, pointers ...any) { + if len(pointers) == 0 { + panic("no pointers given") + } + + if !mods.isPipeliningEnabled { + mods.Get(pointers...) + return + } + + for _, ptr := range pointers { + if !mods.TryGet(ptr) { + if !mods.tryGetFromScope(moduleInScope, ptr) { + panic(fmt.Sprintf("scoped module of type %s not found", reflect.TypeOf(ptr).Elem())) + } + } + } +} + +// tryGetFromScope attempts to find a module for ptr which also happens to be in the same +// pipe as moduleInScope, false otherwise. +// tryGetFromScope returns true if a module was successflully stored in ptr, false otherwise. +// If pipelining was not enabled, TryGet is called implicitly. +func (mods *Core) tryGetFromScope(moduleInScope Module, ptr any) bool { + if len(mods.scopedModules) == 0 { + return mods.TryGet(ptr) + } + + v := reflect.ValueOf(ptr) + if !v.IsValid() { + panic("ptr value cannot be nil") + } + pt := v.Type() + if pt.Kind() != reflect.Ptr { + panic("only pointer values allowed") + } + + correctPipe := hotstuff.NullPipe + for pipe := range mods.scopedModules { + scope := mods.scopedModules[pipe] + // Check if self is in scope + for _, module := range scope { + // TODO: Verify if equality comparison is correct + if module == moduleInScope { + correctPipe = pipe + break + } + } + // Break outer loop too if pipe was found + if correctPipe != hotstuff.NullPipe { + break + } + } + + // If this variable remained unchanged, return false + if correctPipe == hotstuff.NullPipe { + return false + } + + correctScope := mods.scopedModules[correctPipe] + for _, m := range correctScope { + mv := reflect.ValueOf(m) + if mv.Type().AssignableTo(pt.Elem()) { + v.Elem().Set(mv) + return true + } + } + + return false +} + +// MatchForScope assigns ptr to a matching module in the scope with scope. +func (core *Core) MatchForScope(pipe hotstuff.Pipe, ptr any) { + if len(core.scopedModules) == 0 { + panic("pipelining is not enabled") + } + + v := reflect.ValueOf(ptr) + if !v.IsValid() { + panic("pointer value cannot be nil") + } + + pt := v.Type() + if pt.Kind() != reflect.Ptr { + panic("only pointer value allowed") + } + + scope := core.scopedModules[pipe] + for _, m := range scope { + mv := reflect.ValueOf(m) + if mv.Type().AssignableTo(pt.Elem()) { + v.Elem().Set(mv) + return + } + } + + panic("no match found for " + pt.Elem().Name()) +} + +// Return the number of scopes the builder has generated. +func (core *Core) ScopeCount() int { + return len(core.scopedModules) +} + +// Return a slice of Scopes in the order which the scopes were created by Builder. +func (core *Core) Scopes() (ids []hotstuff.Pipe) { + for id := range core.scopedModules { + ids = append(ids, id) + } + + return +} + +// Return a list of modules from a scope. The order of module types is influenced +// by when AddScoped was called in Builder. +func (core *Core) GetScope(pipe hotstuff.Pipe) []any { + return core.scopedModules[pipe] +} + // Builder is a helper for setting up client modules. type Builder struct { - core Core - modules []Module - opts *Options + core Core + staticModules []Module + moduleScopes map[hotstuff.Pipe]moduleScope + opts *Options + pipeliningEnabled bool + pipeIds []hotstuff.Pipe } // NewBuilder returns a new builder. @@ -125,21 +289,105 @@ func NewBuilder(id hotstuff.ID, pk hotstuff.PrivateKey) Builder { privateKey: pk, connectionMetadata: make(map[string]string), }, + pipeliningEnabled: false, + pipeIds: nil, + moduleScopes: nil, } + return bl } +// EnablePipelining enables pipelining by allocating the module scopes and assigning them the pipe identifiers. +func (bl *Builder) EnablePipelining(pipeCount int) { + if bl.pipeliningEnabled { + panic("pipelining already enabled") + } + + if pipeCount <= 0 { + panic("pipelining requires at least one pipe") + } + + bl.pipeliningEnabled = true + bl.core.scopedModules = make(map[hotstuff.Pipe][]any) + bl.moduleScopes = make(map[hotstuff.Pipe]moduleScope) + + for i := hotstuff.Pipe(1); i <= hotstuff.Pipe(pipeCount); i++ { + bl.pipeIds = append(bl.pipeIds, i) + bl.moduleScopes[i] = make(moduleScope, 0) + bl.core.scopedModules[i] = make([]any, 0) + } + +} + // Options returns the options module. func (b *Builder) Options() *Options { return b.opts } -// Add adds modules to the builder. +// Add adds existing, singular, module instances to the builder. func (b *Builder) Add(modules ...any) { - b.core.modules = append(b.core.modules, modules...) + b.core.staticModules = append(b.core.staticModules, modules...) for _, module := range modules { if m, ok := module.(Module); ok { - b.modules = append(b.modules, m) + b.staticModules = append(b.staticModules, m) + } + } +} + +// CreateScope constructs N modules of the same type based on the constructor function +// and adds them to a map where each module is mapped to a scope. The module's constructor +// is called with the variadic arguments. +// If pipelining is disabled when CreateScope is called, only one module will be constructed and mapped +// to hotstuff.NullPipe. +// To add the modules, use AddScopedModules later. +func (b *Builder) CreateScope(ctor any, ctorArgs ...any) (scopedMods map[hotstuff.Pipe]any) { + if reflect.TypeOf(ctor).Kind() != reflect.Func { + panic("first argument is not a function") + } + + scopedMods = make(map[hotstuff.Pipe]any) + + vargs := make([]reflect.Value, len(ctorArgs)) + for n, v := range ctorArgs { + vargs[n] = reflect.ValueOf(v) + } + + ctorVal := reflect.ValueOf(ctor) + scopeIds := b.pipeIds + if !b.pipeliningEnabled { + scopeIds = []hotstuff.Pipe{hotstuff.NullPipe} + } + for _, id := range scopeIds { + returnResult := ctorVal.Call(vargs) + if len(returnResult) != 1 { + panic("constructor does not return a single value") + } + mod := returnResult[0].Interface() + scopedMods[id] = mod + } + return scopedMods +} + +func (b *Builder) AddScoped(scopedModuleMaps ...map[hotstuff.Pipe]any) { + for _, scopedMods := range scopedModuleMaps { + if !b.pipeliningEnabled { + mod, ok := scopedMods[hotstuff.NullPipe] + if !ok { + panic("map of scoped modules did not contain null pipe key") + } + b.Add(mod) + continue + } + + for id := range scopedMods { + mod := scopedMods[id] + converted, ok := mod.(Module) + + b.core.scopedModules[id] = append(b.core.scopedModules[id], mod) + if !ok { + continue + } + b.moduleScopes[id] = append(b.moduleScopes[id], converted) } } } @@ -147,13 +395,38 @@ func (b *Builder) Add(modules ...any) { // Build initializes all added modules and returns the Core object. func (b *Builder) Build() *Core { // reverse the order of the added modules so that TryGet will find the latest first. - for i, j := 0, len(b.core.modules)-1; i < j; i, j = i+1, j-1 { - b.core.modules[i], b.core.modules[j] = b.core.modules[j], b.core.modules[i] + for i, j := 0, len(b.core.staticModules)-1; i < j; i, j = i+1, j-1 { + b.core.staticModules[i], b.core.staticModules[j] = b.core.staticModules[j], b.core.staticModules[i] } // add the Options last so that it can be overridden by user. b.Add(b.opts) - for _, module := range b.modules { - module.InitModule(&b.core) + opt := ScopeInfo{ + IsPipeliningEnabled: b.pipeliningEnabled, + ModuleScope: hotstuff.NullPipe, + ScopeCount: len(b.pipeIds), + } + for _, module := range b.staticModules { + module.InitModule(&b.core, opt) + } + + if !b.pipeliningEnabled { + b.core.isPipeliningEnabled = false + return &b.core // Exit early + } + + b.core.isPipeliningEnabled = true + + // Initializing later so that modules can reference + // other modules in the same pipe without panicking. + for pipe, scope := range b.moduleScopes { + opt := ScopeInfo{ + IsPipeliningEnabled: b.pipeliningEnabled, + ModuleScope: pipe, + ScopeCount: len(b.pipeIds), + } + for _, module := range scope { + module.(Module).InitModule(&b.core, opt) + } } return &b.core } diff --git a/modules/module_test.go b/modules/module_test.go index d758bd60..e285cb1e 100644 --- a/modules/module_test.go +++ b/modules/module_test.go @@ -42,7 +42,7 @@ func NewGreeter() *greeterImpl { //nolint:revive return &greeterImpl{} } -func (g *greeterImpl) InitModule(mods *modules.Core) { +func (g *greeterImpl) InitModule(mods *modules.Core, _ modules.ScopeInfo) { mods.Get(&g.counter) } diff --git a/modules/modules.go b/modules/modules.go index 98c1f84c..ebc1cf35 100644 --- a/modules/modules.go +++ b/modules/modules.go @@ -2,8 +2,10 @@ package modules import ( "context" + "sync" "github.com/relab/hotstuff" + "github.com/relab/hotstuff/logging" ) // Module interfaces @@ -23,7 +25,7 @@ type CommandQueue interface { // Acceptor decides if a replica should accept a command. type Acceptor interface { // Accept returns true if the replica should accept the command, false otherwise. - Accept(hotstuff.Command) bool + Accept(cmd hotstuff.Command) bool // Proposed tells the acceptor that the propose phase for the given command succeeded, and it should no longer be // accepted in the future. Proposed(hotstuff.Command) @@ -34,7 +36,7 @@ type Acceptor interface { // Executor is responsible for executing the commands that are committed by the consensus protocol. type Executor interface { // Exec executes the command. - Exec(cmd hotstuff.Command) + Exec(onPipe hotstuff.Pipe, cmd hotstuff.Command) } // ExecutorExt is responsible for executing the commands that are committed by the consensus protocol. @@ -64,6 +66,16 @@ type ForkHandlerExt interface { Fork(block *hotstuff.Block) } +// Committer is a helper module which handles block commits and forks. +// NOTE: This module was created to deal with multiple pipes. +type Committer interface { + // Stores the block before further execution. + Commit(block *hotstuff.Block) + + // Retrieve the last block which was committed on a pipe. Use zero if pipelining is not desired in the implementation. + CommittedBlock(pip hotstuff.Pipe) *hotstuff.Block +} + // BlockChain is a datastructure that stores a chain of blocks. // It is not required that a block is stored forever, // but a block must be stored until at least one of its children have been committed. @@ -72,7 +84,7 @@ type BlockChain interface { Store(*hotstuff.Block) // Get retrieves a block given its hash, attempting to fetching it from other replicas if necessary. - Get(hotstuff.Hash) (*hotstuff.Block, bool) + Get(hash hotstuff.Hash, onPipe hotstuff.Pipe) (*hotstuff.Block, bool) // LocalGet retrieves a block given its hash, without fetching it from other replicas. LocalGet(hotstuff.Hash) (*hotstuff.Block, bool) @@ -82,7 +94,11 @@ type BlockChain interface { // Prunes blocks from the in-memory tree up to the specified height. // Returns a set of forked blocks (blocks that were on a different branch, and thus not committed). - PruneToHeight(height hotstuff.View) (forkedBlocks []*hotstuff.Block) + PruneToHeight(height hotstuff.View) (prunedBlocks map[hotstuff.View][]*hotstuff.Block) + + PruneHeight() hotstuff.View + + DeleteAtHeight(height hotstuff.View, blockHash hotstuff.Hash) error } //go:generate mockgen -destination=../internal/mocks/replica_mock.go -package=mocks . Replica @@ -170,21 +186,27 @@ type Handel interface { // ExtendedExecutor turns the given Executor into an ExecutorExt. func ExtendedExecutor(executor Executor) ExecutorExt { - return executorWrapper{executor} + return &executorWrapper{executor: executor, bExec: hotstuff.GetGenesis()} } type executorWrapper struct { - executor Executor + blockChain BlockChain + executor Executor + forkHandler ForkHandlerExt + logger logging.Logger + + mut sync.Mutex + bExec *hotstuff.Block } -func (ew executorWrapper) InitModule(mods *Core) { +func (ew *executorWrapper) InitModule(mods *Core, buildOpt ScopeInfo) { if m, ok := ew.executor.(Module); ok { - m.InitModule(mods) + m.InitModule(mods, buildOpt) } } -func (ew executorWrapper) Exec(block *hotstuff.Block) { - ew.executor.Exec(block.Command()) +func (ew *executorWrapper) Exec(block *hotstuff.Block) { + ew.executor.Exec(block.Pipe(), block.Command()) } // ExtendedForkHandler turns the given ForkHandler into a ForkHandlerExt. @@ -196,9 +218,9 @@ type forkHandlerWrapper struct { forkHandler ForkHandler } -func (fhw forkHandlerWrapper) InitModule(mods *Core) { +func (fhw forkHandlerWrapper) InitModule(mods *Core, buildOpt ScopeInfo) { if m, ok := fhw.forkHandler.(Module); ok { - m.InitModule(mods) + m.InitModule(mods, buildOpt) } } diff --git a/modules/pipelined_test.go b/modules/pipelined_test.go new file mode 100644 index 00000000..53fbde05 --- /dev/null +++ b/modules/pipelined_test.go @@ -0,0 +1,128 @@ +package modules_test + +import ( + "testing" + + "github.com/relab/hotstuff" + "github.com/relab/hotstuff/modules" +) + +type Adder interface { + Add(a, b int) int + LastResult() int +} + +type adderImpl struct { + results []int +} + +func NewAdder() *adderImpl { //nolint:revive + return &adderImpl{ + results: make([]int, 0), + } +} + +func (ad *adderImpl) Add(a, b int) int { + ad.results = append(ad.results, a+b) + return a + b +} + +func (ad *adderImpl) LastResult() int { + return ad.results[len(ad.results)-1] +} + +type Multiplier interface { + Mult(a, b int) int +} + +type multiplierImpl struct { + // declares dependencies on other modules + adder Adder +} + +func (m *multiplierImpl) Mult(a, b int) int { + result := 0 + for i := 0; i < a; i++ { + result = m.adder.Add(result, b) + } + return result +} + +func NewMultiplier() *multiplierImpl { //nolint:revive + return &multiplierImpl{} +} + +func (m *multiplierImpl) InitModule(mods *modules.Core, _ modules.ScopeInfo) { + mods.GetScoped(m, &m.adder) // Requires an adder from the same pipe +} + +func TestPipeliningDisabled(t *testing.T) { + builder := modules.NewBuilder(0, nil) + + adders := builder.CreateScope(NewAdder) + multers := builder.CreateScope(NewMultiplier) + + builder.AddScoped(adders, multers) + + mods := builder.Build() + if mods.ScopeCount() > 0 { + t.Fail() + } + + var adder Adder + var multiplier Multiplier + mods.Get(&adder, &multiplier) + + result := multiplier.Mult(2, 3) + if result != 6 { + t.Fail() + } +} + +func TestPipelined(t *testing.T) { + pipes := 3 + + builder := modules.NewBuilder(0, nil) + builder.EnablePipelining(pipes) + + adders := builder.CreateScope(NewAdder) + multers := builder.CreateScope(NewMultiplier) + + builder.AddScoped(adders, multers) + + core := builder.Build() + if core.ScopeCount() != pipes { + t.Fail() + } + + type AdderMultTestCase struct { + A int + B int + Result int + } + + testCasesMult := map[hotstuff.Pipe]AdderMultTestCase{ + 1: {A: 2, B: 3, Result: 6}, + 2: {A: 2, B: 5, Result: 10}, + 3: {A: 2, B: 6, Result: 12}, + } + + scopeIds := core.Scopes() + for _, id := range scopeIds { + var multer Multiplier + core.MatchForScope(id, &multer) + tc := testCasesMult[id] + actualResult := multer.Mult(tc.A, tc.B) + if tc.Result != actualResult { + t.Fail() + } + + // The last result stored in the adder is the same as the result of multiplier, + // since the multiple addings will add up to the multiplication answer. + var adder Adder + core.MatchForScope(id, &adder) + if adder.LastResult() != tc.Result || adder.LastResult() != actualResult { + t.Fail() + } + } +} diff --git a/modules/registry.go b/modules/registry.go index 86897bf7..a696b840 100644 --- a/modules/registry.go +++ b/modules/registry.go @@ -61,6 +61,25 @@ func GetModule[T any](name string) (out T, ok bool) { return ctor.(func() T)(), true } +func GetModuleCtor[T any](name string) (out func() T, ok bool) { + targetType := reflect.TypeFor[T]() + + registryMut.Lock() + defer registryMut.Unlock() + + modules, ok := byInterface[targetType] + if !ok { + return out, false + } + + ctor, ok := modules[name] + if !ok { + return out, false + } + + return ctor.(func() T), true +} + // GetModuleUntyped returns a new instance of the named module, if it exists. func GetModuleUntyped(name string) (m any, ok bool) { registryMut.Lock() diff --git a/replica/clientsrv.go b/replica/clientsrv.go index cb454271..048439cb 100644 --- a/replica/clientsrv.go +++ b/replica/clientsrv.go @@ -21,36 +21,106 @@ import ( // clientSrv serves a client. type clientSrv struct { - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop logger logging.Logger - mut sync.Mutex - srv *gorums.Server - awaitingCmds map[cmdID]chan<- error - cmdCache *cmdCache - hash hash.Hash - cmdCount uint32 + mut sync.Mutex + srv *gorums.Server + awaitingCmds map[cmdID]chan<- error + cmdCaches map[hotstuff.Pipe]*cmdCache + hash hash.Hash + cmdCount uint32 + pipeCount int + cmdsSentToPipe map[hotstuff.Pipe]int + cmdAddMethodStr string + marshaler proto.MarshalOptions } // newClientServer returns a new client server. -func newClientServer(conf Config, srvOpts []gorums.ServerOption) (srv *clientSrv) { +func newClientServer(cmdAddMethodStr string, srvOpts []gorums.ServerOption) (srv *clientSrv) { srv = &clientSrv{ awaitingCmds: make(map[cmdID]chan<- error), srv: gorums.NewServer(srvOpts...), - cmdCache: newCmdCache(int(conf.BatchSize)), - hash: sha256.New(), + // cmdCache: newCmdCache(int(conf.BatchSize)), + cmdCaches: make(map[hotstuff.Pipe]*cmdCache), + hash: sha256.New(), + cmdsSentToPipe: make(map[hotstuff.Pipe]int), + cmdAddMethodStr: cmdAddMethodStr, + marshaler: proto.MarshalOptions{Deterministic: true}, } + clientpb.RegisterClientServer(srv.srv, srv) return srv } +func (srv *clientSrv) addCommandToSmallestCache(cmd *clientpb.Command) { + smallestCachePipe := hotstuff.NullPipe + smallestCacheCount := 0 + for pipe := range srv.cmdCaches { + count := srv.cmdCaches[pipe].commandCount() + if smallestCacheCount > count || smallestCachePipe == hotstuff.NullPipe { + smallestCachePipe = pipe + smallestCacheCount = count + } + } + srv.cmdCaches[smallestCachePipe].addCommand(cmd) + srv.mut.Lock() + srv.cmdsSentToPipe[smallestCachePipe]++ + srv.mut.Unlock() +} + +func (srv *clientSrv) addCommandHashed(cmd *clientpb.Command) error { + correctPipe := hotstuff.Pipe((uint32(cmd.SequenceNumber) % uint32(srv.pipeCount)) + 1) + cache, ok := srv.cmdCaches[correctPipe] + if ok { + cache.addCommand(cmd) + srv.mut.Lock() + srv.cmdsSentToPipe[correctPipe]++ + srv.mut.Unlock() + } else { + srv.logger.DPanicf("addCommand: pipe not found: %d. count was %d", correctPipe, srv.pipeCount) + } + return nil +} + +func (srv *clientSrv) commandAddingMethod(cmd *clientpb.Command) { + if srv.pipeCount == 0 { + srv.cmdCaches[hotstuff.NullPipe].addCommand(cmd) + return + } + + switch srv.cmdAddMethodStr { + case "hashed": + srv.addCommandHashed(cmd) + break + case "smallest": + srv.addCommandToSmallestCache(cmd) + break + } +} + // InitModule gives the module access to the other modules. -func (srv *clientSrv) InitModule(mods *modules.Core) { +func (srv *clientSrv) InitModule(mods *modules.Core, info modules.ScopeInfo) { mods.Get( &srv.eventLoop, &srv.logger, ) - srv.cmdCache.InitModule(mods) + + srv.pipeCount = info.ScopeCount + if info.IsPipeliningEnabled { + for _, scope := range mods.Scopes() { + var cache *cmdCache + mods.MatchForScope(scope, &cache) + srv.cmdCaches[scope] = cache + } + return + } + + var cache *cmdCache + mods.Get(&cache) + srv.cmdCaches[hotstuff.NullPipe] = cache + + // srv.cmdCache.InitModule(mods, buildOpt) } func (srv *clientSrv) Start(addr string) error { @@ -75,6 +145,16 @@ func (srv *clientSrv) Stop() { srv.srv.Stop() } +func (srv *clientSrv) PrintScopedCmdResult() { + if srv.pipeCount <= 1 { + return + } + srv.logger.Info("Command count per pipe results:") + for pipe, count := range srv.cmdsSentToPipe { + srv.logger.Infof("\tP%d=(%d)", pipe, count) + } +} + func (srv *clientSrv) ExecCommand(ctx gorums.ServerCtx, cmd *clientpb.Command) (*emptypb.Empty, error) { id := cmdID{cmd.ClientID, cmd.SequenceNumber} @@ -83,13 +163,13 @@ func (srv *clientSrv) ExecCommand(ctx gorums.ServerCtx, cmd *clientpb.Command) ( srv.awaitingCmds[id] = c srv.mut.Unlock() - srv.cmdCache.addCommand(cmd) + srv.commandAddingMethod(cmd) ctx.Release() err := <-c return &emptypb.Empty{}, err } -func (srv *clientSrv) Exec(cmd hotstuff.Command) { +func (srv *clientSrv) Exec(onPipe hotstuff.Pipe, cmd hotstuff.Command) { batch := new(clientpb.Batch) err := proto.UnmarshalOptions{AllowPartial: true}.Unmarshal([]byte(cmd), batch) if err != nil { @@ -97,7 +177,8 @@ func (srv *clientSrv) Exec(cmd hotstuff.Command) { return } - srv.eventLoop.AddEvent(hotstuff.CommitEvent{Commands: len(batch.GetCommands())}) + srv.eventLoop.AddEvent(hotstuff.CommitEvent{Pipe: onPipe, Commands: len(batch.GetCommands())}) + srv.logger.Debugf("Executed %d commands", len(batch.GetCommands())) for _, cmd := range batch.GetCommands() { _, _ = srv.hash.Write(cmd.Data) diff --git a/replica/cmdcache.go b/replica/cmdcache.go index f13ad6bf..4580b1f9 100644 --- a/replica/cmdcache.go +++ b/replica/cmdcache.go @@ -36,15 +36,20 @@ func newCmdCache(batchSize int) *cmdCache { } // InitModule gives the module access to the other modules. -func (c *cmdCache) InitModule(mods *modules.Core) { +func (c *cmdCache) InitModule(mods *modules.Core, _ modules.ScopeInfo) { mods.Get(&c.logger) } +func (c *cmdCache) commandCount() int { + return c.cache.Len() +} + func (c *cmdCache) addCommand(cmd *clientpb.Command) { c.mut.Lock() defer c.mut.Unlock() if serialNo := c.serialNumbers[cmd.GetClientID()]; serialNo >= cmd.GetSequenceNumber() { // command is too old + c.logger.Debugf("addCommand: command too old: %.8x", cmd.Data) return } c.cache.PushBack(cmd) @@ -151,3 +156,4 @@ func (c *cmdCache) Proposed(cmd hotstuff.Command) { } var _ modules.Acceptor = (*cmdCache)(nil) +var _ modules.CommandQueue = (*cmdCache)(nil) diff --git a/replica/replica.go b/replica/replica.go index 2a963e9a..862982a0 100644 --- a/replica/replica.go +++ b/replica/replica.go @@ -15,6 +15,7 @@ import ( "github.com/relab/hotstuff/backend" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/emptypb" ) @@ -46,6 +47,10 @@ type Config struct { ManagerOptions []gorums.ManagerOption // Location information of all replicas LocationInfo map[hotstuff.ID]string + // Number of pipes in pipelining mode + PipeCount uint32 + // Latency induced by all replicas. + HackyLatency durationpb.Duration } // Replica is a participant in the consensus protocol. @@ -70,7 +75,9 @@ func New(conf Config, builder modules.Builder) (replica *Replica) { )) } - clientSrv := newClientServer(conf, clientSrvOpts) + cmdCaches := builder.CreateScope(newCmdCache, int(conf.BatchSize)) + + clientSrv := newClientServer("hashed", clientSrvOpts) srv := &Replica{ clientSrv: clientSrv, @@ -95,6 +102,8 @@ func New(conf Config, builder modules.Builder) (replica *Replica) { backend.WithGorumsServerOptions(replicaSrvOpts...), ) + srv.hsSrv.SetHackyLatency(conf.HackyLatency.AsDuration()) + var creds credentials.TransportCredentials managerOpts := conf.ManagerOptions if conf.TLS { @@ -111,7 +120,9 @@ func New(conf Config, builder modules.Builder) (replica *Replica) { modules.ExtendedExecutor(srv.clientSrv), modules.ExtendedForkHandler(srv.clientSrv), - srv.clientSrv.cmdCache, + ) + builder.AddScoped( + cmdCaches, ) srv.hs = builder.Build() @@ -147,19 +158,29 @@ func (srv *Replica) Start() { // Stop stops the replica and closes connections. func (srv *Replica) Stop() { srv.cancel() + srv.clientSrv.logger.Info("Server stopping...") <-srv.done srv.Close() + srv.clientSrv.PrintScopedCmdResult() } // Run runs the replica until the context is canceled. func (srv *Replica) Run(ctx context.Context) { - var ( - synchronizer modules.Synchronizer - eventLoop *eventloop.EventLoop - ) - srv.hs.Get(&synchronizer, &eventLoop) + var eventLoop *eventloop.ScopedEventLoop + srv.hs.Get(&eventLoop) + + if srv.hs.ScopeCount() > 0 { + for pipe := hotstuff.Pipe(1); pipe <= hotstuff.Pipe(srv.hs.ScopeCount()); pipe++ { + var synchronizer modules.Synchronizer + srv.hs.MatchForScope(pipe, &synchronizer) + synchronizer.Start(ctx) + } + } else { + var synchronizer modules.Synchronizer + srv.hs.Get(&synchronizer) + synchronizer.Start(ctx) + } - synchronizer.Start(ctx) eventLoop.Run(ctx) } diff --git a/synchronizer/context.go b/synchronizer/context.go index 40cba173..93be8833 100644 --- a/synchronizer/context.go +++ b/synchronizer/context.go @@ -2,6 +2,7 @@ package synchronizer import ( "context" + "fmt" "github.com/relab/hotstuff" "github.com/relab/hotstuff/eventloop" @@ -11,7 +12,7 @@ import ( // ViewContext returns a context that is canceled at the end of view. // If view is nil or less than or equal to the current view, the context will be canceled at the next view change. -func ViewContext(parent context.Context, eventLoop *eventloop.EventLoop, view *hotstuff.View) (context.Context, context.CancelFunc) { +func ViewContext(parent context.Context, eventLoop *eventloop.ScopedEventLoop, view *hotstuff.View) (context.Context, context.CancelFunc) { ctx, cancel := context.WithCancel(parent) id := eventLoop.RegisterHandler(ViewChangeEvent{}, func(event any) { @@ -21,13 +22,13 @@ func ViewContext(parent context.Context, eventLoop *eventloop.EventLoop, view *h }, eventloop.Prioritize(), eventloop.UnsafeRunInAddEvent()) return ctx, func() { - eventLoop.UnregisterHandler(ViewChangeEvent{}, id) + eventLoop.UnregisterHandler(ViewChangeEvent{}, hotstuff.NullPipe, id) cancel() } } // TimeoutContext returns a context that is canceled either when a timeout occurs, or when the view changes. -func TimeoutContext(parent context.Context, eventLoop *eventloop.EventLoop) (context.Context, context.CancelFunc) { +func TimeoutContext(parent context.Context, eventLoop *eventloop.ScopedEventLoop) (context.Context, context.CancelFunc) { // ViewContext handles view-change case. ctx, cancel := ViewContext(parent, eventLoop, nil) @@ -36,7 +37,59 @@ func TimeoutContext(parent context.Context, eventLoop *eventloop.EventLoop) (con }, eventloop.Prioritize(), eventloop.UnsafeRunInAddEvent()) return ctx, func() { - eventLoop.UnregisterHandler(TimeoutEvent{}, id) + eventLoop.UnregisterHandler(TimeoutEvent{}, hotstuff.NullPipe, id) + cancel() + } +} + +// ScopedViewContext returns a context that is canceled at the end of view. +// If view is nil or less than or equal to the current view, the context will be canceled at the next view change. +// If pipe is null pipe, returns regular ScopedViewContext. +func ScopedViewContext(parent context.Context, eventLoop *eventloop.ScopedEventLoop, pipe hotstuff.Pipe, view *hotstuff.View) (context.Context, context.CancelFunc) { + if pipe == hotstuff.NullPipe { + return ViewContext(parent, eventLoop, view) + } + + ctx, cancel := context.WithCancel(parent) + + id := eventLoop.RegisterHandler(ViewChangeEvent{}, func(event any) { + myScope := pipe + viewChangeEvent := event.(ViewChangeEvent) + if viewChangeEvent.Pipe != myScope { + panic(fmt.Sprintf("incorrect pipe: want=%d, got=%d", myScope, viewChangeEvent.Pipe)) + } + if view == nil || viewChangeEvent.View >= *view { + cancel() + } + }, eventloop.Prioritize(), eventloop.UnsafeRunInAddEvent(), eventloop.RespondToScope(pipe)) + + return ctx, func() { + eventLoop.UnregisterHandler(ViewChangeEvent{}, pipe, id) + cancel() + } +} + +// ScopedTimeoutContext returns a context that is canceled either when a timeout occurs, or when the view changes. +// If pipe is NullPipe, returns regular TimeoutContext. +func ScopedTimeoutContext(parent context.Context, eventLoop *eventloop.ScopedEventLoop, pipe hotstuff.Pipe) (context.Context, context.CancelFunc) { + if pipe == hotstuff.NullPipe { + return TimeoutContext(parent, eventLoop) + } + + // ViewContext handles view-change case. + ctx, cancel := ScopedViewContext(parent, eventLoop, pipe, nil) + + id := eventLoop.RegisterHandler(TimeoutEvent{}, func(event any) { + myScope := pipe + timeoutEvent := event.(TimeoutEvent) + if timeoutEvent.Pipe != myScope { + panic(fmt.Sprintf("incorrect pipe: want=%d, got=%d", myScope, timeoutEvent.Pipe)) + } + cancel() + }, eventloop.Prioritize(), eventloop.UnsafeRunInAddEvent(), eventloop.RespondToScope(pipe)) + + return ctx, func() { + eventLoop.UnregisterHandler(TimeoutEvent{}, pipe, id) cancel() } } diff --git a/synchronizer/context_test.go b/synchronizer/context_test.go index 8370badb..8bec51e1 100644 --- a/synchronizer/context_test.go +++ b/synchronizer/context_test.go @@ -11,7 +11,7 @@ import ( // TestTimeoutContext tests that a timeout context is canceled after receiving a timeout event. func TestTimeoutContext(t *testing.T) { - eventloop := eventloop.New(10) + eventloop := eventloop.NewScoped(10, 0) ctx, cancel := synchronizer.TimeoutContext(context.Background(), eventloop) defer cancel() @@ -24,7 +24,7 @@ func TestTimeoutContext(t *testing.T) { // TestTimeoutContextView tests that a timeout context is canceled after receiving a view change event. func TestTimeoutContextView(t *testing.T) { - eventloop := eventloop.New(10) + eventloop := eventloop.NewScoped(10, 0) ctx, cancel := synchronizer.TimeoutContext(context.Background(), eventloop) defer cancel() @@ -37,7 +37,7 @@ func TestTimeoutContextView(t *testing.T) { // TestViewContext tests that a view context is canceled after receiving a view change event. func TestViewContext(t *testing.T) { - eventloop := eventloop.New(10) + eventloop := eventloop.NewScoped(10, 0) ctx, cancel := synchronizer.ViewContext(context.Background(), eventloop, nil) defer cancel() @@ -50,7 +50,7 @@ func TestViewContext(t *testing.T) { // TestViewContextEarlierView tests that a view context is not canceled when receiving a view change event for an earlier view. func TestViewContextEarlierView(t *testing.T) { - eventloop := eventloop.New(10) + eventloop := eventloop.NewScoped(10, 0) view := hotstuff.View(1) ctx, cancel := synchronizer.ViewContext(context.Background(), eventloop, &view) defer cancel() @@ -61,3 +61,56 @@ func TestViewContextEarlierView(t *testing.T) { t.Error("Context canceled") } } + +// TestTimeoutContext tests that a timeout context is canceled after receiving a timeout event. +func TestTimeoutContextScoped(t *testing.T) { + eventloop := eventloop.NewScoped(10, 1) + ctx, cancel := synchronizer.ScopedTimeoutContext(context.Background(), eventloop, hotstuff.Pipe(1)) + defer cancel() + + eventloop.AddScopedEvent(hotstuff.Pipe(1), synchronizer.TimeoutEvent{Pipe: 1}) + + if ctx.Err() != context.Canceled { + t.Error("Context not canceled") + } +} + +// TestTimeoutContextView tests that a timeout context is canceled after receiving a view change event. +func TestTimeoutContextViewScoped(t *testing.T) { + eventloop := eventloop.NewScoped(10, 1) + ctx, cancel := synchronizer.ScopedTimeoutContext(context.Background(), eventloop, hotstuff.Pipe(1)) + defer cancel() + + eventloop.AddScopedEvent(hotstuff.Pipe(1), synchronizer.ViewChangeEvent{View: 1, Pipe: 1}) + + if ctx.Err() != context.Canceled { + t.Error("Context not canceled") + } +} + +// TestViewContext tests that a view context is canceled after receiving a view change event. +func TestViewContextScoped(t *testing.T) { + eventloop := eventloop.NewScoped(10, 1) + ctx, cancel := synchronizer.ScopedViewContext(context.Background(), eventloop, hotstuff.Pipe(1), nil) + defer cancel() + + eventloop.AddScopedEvent(hotstuff.Pipe(1), synchronizer.ViewChangeEvent{View: 1, Pipe: 1}) + + if ctx.Err() != context.Canceled { + t.Error("Context not canceled") + } +} + +// TestViewContextEarlierView tests that a view context is not canceled when receiving a view change event for an earlier view. +func TestViewContextEarlierViewScoped(t *testing.T) { + eventloop := eventloop.NewScoped(10, 1) + view := hotstuff.View(1) + ctx, cancel := synchronizer.ScopedViewContext(context.Background(), eventloop, hotstuff.Pipe(1), &view) + defer cancel() + + eventloop.AddScopedEvent(hotstuff.Pipe(1), synchronizer.ViewChangeEvent{View: 0, Pipe: 1}) + + if ctx.Err() != nil { + t.Error("Context canceled") + } +} diff --git a/synchronizer/fixedduration.go b/synchronizer/fixedduration.go new file mode 100644 index 00000000..cbeed4e4 --- /dev/null +++ b/synchronizer/fixedduration.go @@ -0,0 +1,31 @@ +package synchronizer + +import "time" + +type fixedViewDuration struct { + duration time.Duration +} + +func NewFixedDuration(duration time.Duration) ViewDuration { + return &fixedViewDuration{duration: duration} +} + +// Duration returns the duration that the next view should last. +func (f *fixedViewDuration) Duration() time.Duration { + return f.duration +} + +// ViewStarted is called by the synchronizer when starting a new view. +func (_ *fixedViewDuration) ViewStarted() { + +} + +// ViewSucceeded is called by the synchronizer when a view ended successfully. +func (_ *fixedViewDuration) ViewSucceeded() { + +} + +// ViewTimeout is called by the synchronizer when a view timed out. +func (_ *fixedViewDuration) ViewTimeout() { + +} diff --git a/synchronizer/synchronizer.go b/synchronizer/synchronizer.go index 32aaa6ad..8053cf29 100644 --- a/synchronizer/synchronizer.go +++ b/synchronizer/synchronizer.go @@ -20,7 +20,7 @@ type Synchronizer struct { consensus modules.Consensus crypto modules.Crypto configuration modules.Configuration - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop leaderRotation modules.LeaderRotation logger logging.Logger opts *modules.Options @@ -38,58 +38,62 @@ type Synchronizer struct { duration ViewDuration timer oneShotTimer + pipe hotstuff.Pipe + // map of collected timeout messages per view timeouts map[hotstuff.View]map[hotstuff.ID]hotstuff.TimeoutMsg } // InitModule initializes the synchronizer. -func (s *Synchronizer) InitModule(mods *modules.Core) { - mods.Get( +func (s *Synchronizer) InitModule(mods *modules.Core, info modules.ScopeInfo) { + mods.GetScoped(s, &s.blockChain, - &s.consensus, &s.crypto, &s.configuration, + &s.consensus, + &s.duration, &s.eventLoop, &s.leaderRotation, &s.logger, &s.opts, ) + s.pipe = info.ModuleScope + s.eventLoop.RegisterHandler(TimeoutEvent{}, func(event any) { - timeoutView := event.(TimeoutEvent).View - if s.View() == timeoutView { + timeout := event.(TimeoutEvent) + if s.View() == timeout.View { s.OnLocalTimeout() } - }) + }, eventloop.RespondToScope(info.ModuleScope)) s.eventLoop.RegisterHandler(hotstuff.NewViewMsg{}, func(event any) { newViewMsg := event.(hotstuff.NewViewMsg) s.OnNewView(newViewMsg) - }) + }, eventloop.RespondToScope(info.ModuleScope)) s.eventLoop.RegisterHandler(hotstuff.TimeoutMsg{}, func(event any) { timeoutMsg := event.(hotstuff.TimeoutMsg) s.OnRemoteTimeout(timeoutMsg) - }) + }, eventloop.RespondToScope(info.ModuleScope)) var err error s.highQC, err = s.crypto.CreateQuorumCert(hotstuff.GetGenesis(), []hotstuff.PartialCert{}) if err != nil { - panic(fmt.Errorf("unable to create empty quorum cert for genesis block: %v", err)) + panic(fmt.Errorf("unable to create empty quorum cert for genesis block [p=%d, view=%d]: %v", s.pipe, s.View(), err)) } s.highTC, err = s.crypto.CreateTimeoutCert(hotstuff.View(0), []hotstuff.TimeoutMsg{}) if err != nil { - panic(fmt.Errorf("unable to create empty timeout cert for view 0: %v", err)) + panic(fmt.Errorf("unable to create empty timeout cert for view 0 [p=%d, view=%d]: %v", s.pipe, s.View(), err)) } } // New creates a new Synchronizer. -func New(viewDuration ViewDuration) modules.Synchronizer { +func New() modules.Synchronizer { return &Synchronizer{ currentView: 1, - duration: viewDuration, - timer: oneShotTimer{time.AfterFunc(0, func() {})}, // dummy timer that will be replaced after start() is called + timer: oneShotTimer{time.AfterFunc(0, func() {})}, // dummy timer that will be replaced after start() is called timeouts: make(map[hotstuff.View]map[hotstuff.ID]hotstuff.TimeoutMsg), } @@ -111,7 +115,7 @@ func (s *Synchronizer) startTimeoutTimer() { // It is important that the timer is NOT reused because then the view would be wrong. s.timer = oneShotTimer{time.AfterFunc(s.duration.Duration(), func() { // The event loop will execute onLocalTimeout for us. - s.eventLoop.AddEvent(TimeoutEvent{view}) + s.eventLoop.AddScopedEvent(s.pipe, TimeoutEvent{view, s.pipe}) })} } @@ -150,7 +154,10 @@ func (s *Synchronizer) View() hotstuff.View { func (s *Synchronizer) SyncInfo() hotstuff.SyncInfo { s.mut.RLock() defer s.mut.RUnlock() - return hotstuff.NewSyncInfo().WithQC(s.highQC).WithTC(s.highTC) + si := hotstuff.NewSyncInfo(s.pipe) + siWithQC := si.WithQC(s.highQC) + siWithTC := siWithQC.WithTC(s.highTC) + return siWithTC } // OnLocalTimeout is called when a local timeout happens. @@ -165,11 +172,11 @@ func (s *Synchronizer) OnLocalTimeout() { } s.duration.ViewTimeout() // increase the duration of the next view - s.logger.Debugf("OnLocalTimeout: %v", view) + s.logger.Debugf("OnLocalTimeout[p=%d, view=%d]", s.pipe, view) sig, err := s.crypto.Sign(view.ToBytes()) if err != nil { - s.logger.Warnf("Failed to sign view: %v", err) + s.logger.Warnf("Failed to sign view [p=%d, view=%d]: %v", s.pipe, s.View(), err) return } timeoutMsg := hotstuff.TimeoutMsg{ @@ -177,13 +184,14 @@ func (s *Synchronizer) OnLocalTimeout() { View: view, SyncInfo: s.SyncInfo(), ViewSignature: sig, + Pipe: s.pipe, } if s.opts.ShouldUseAggQC() { // generate a second signature that will become part of the aggregateQC sig, err := s.crypto.Sign(timeoutMsg.ToBytes()) if err != nil { - s.logger.Warnf("Failed to sign timeout message: %v", err) + s.logger.Warnf("Failed to sign timeout message [p=%d, view=%d]: %v", s.pipe, s.View(), err) return } timeoutMsg.MsgSignature = sig @@ -198,6 +206,10 @@ func (s *Synchronizer) OnLocalTimeout() { // OnRemoteTimeout handles an incoming timeout from a remote replica. func (s *Synchronizer) OnRemoteTimeout(timeout hotstuff.TimeoutMsg) { + if s.pipe != timeout.Pipe { + panic("incorrect pipe") + } + currView := s.View() defer func() { @@ -213,7 +225,7 @@ func (s *Synchronizer) OnRemoteTimeout(timeout hotstuff.TimeoutMsg) { if !verifier.Verify(timeout.ViewSignature, timeout.View.ToBytes()) { return } - s.logger.Debug("OnRemoteTimeout: ", timeout) + s.logger.Debugf("OnRemoteTimeout [p=%d, view=%d]: ", s.pipe, s.View(), timeout) s.AdvanceView(timeout.SyncInfo) @@ -240,7 +252,7 @@ func (s *Synchronizer) OnRemoteTimeout(timeout hotstuff.TimeoutMsg) { tc, err := s.crypto.CreateTimeoutCert(timeout.View, timeoutList) if err != nil { - s.logger.Debugf("Failed to create timeout certificate: %v", err) + s.logger.Debugf("Failed to create timeout certificate [p=%d, view=%d]: %v", s.pipe, s.View(), err) return } @@ -249,7 +261,7 @@ func (s *Synchronizer) OnRemoteTimeout(timeout hotstuff.TimeoutMsg) { if s.opts.ShouldUseAggQC() { aggQC, err := s.crypto.CreateAggregateQC(currView, timeoutList) if err != nil { - s.logger.Debugf("Failed to create aggregateQC: %v", err) + s.logger.Debugf("Failed to create aggregateQC [p=%d, view=%d]: %v", s.pipe, s.View(), err) } else { si = si.WithAggQC(aggQC) } @@ -268,13 +280,17 @@ func (s *Synchronizer) OnNewView(newView hotstuff.NewViewMsg) { // AdvanceView attempts to advance to the next view using the given QC. // qc must be either a regular quorum certificate, or a timeout certificate. func (s *Synchronizer) AdvanceView(syncInfo hotstuff.SyncInfo) { + if s.pipe != syncInfo.Pipe() { + panic("incorrect pipe") + } + v := hotstuff.View(0) timeout := false // check for a TC if tc, ok := syncInfo.TC(); ok { if !s.crypto.VerifyTimeoutCert(tc) { - s.logger.Info("Timeout Certificate could not be verified!") + s.logger.Infof("Timeout Certificate could not be verified! [p=%d, view=%d]", s.pipe, s.View()) return } s.updateHighTC(tc) @@ -292,7 +308,7 @@ func (s *Synchronizer) AdvanceView(syncInfo hotstuff.SyncInfo) { if aggQC, haveQC = syncInfo.AggQC(); haveQC && s.opts.ShouldUseAggQC() { highQC, ok := s.crypto.VerifyAggregateQC(aggQC) if !ok { - s.logger.Info("Aggregated Quorum Certificate could not be verified") + s.logger.Infof("Aggregated Quorum Certificate could not be verified [p=%d, view=%d]", s.pipe, s.View()) return } if aggQC.View() >= v { @@ -304,7 +320,7 @@ func (s *Synchronizer) AdvanceView(syncInfo hotstuff.SyncInfo) { qc = highQC } else if qc, haveQC = syncInfo.QC(); haveQC { if !s.crypto.VerifyQuorumCert(qc) { - s.logger.Info("Quorum Certificate could not be verified!") + s.logger.Infof("Quorum Certificate could not be verified!", s.pipe) return } } @@ -337,8 +353,8 @@ func (s *Synchronizer) AdvanceView(syncInfo hotstuff.SyncInfo) { s.startTimeoutTimer() - s.logger.Debugf("advanced to view %d", newView) - s.eventLoop.AddEvent(ViewChangeEvent{View: newView, Timeout: timeout}) + s.logger.Debugf("advanced to view %d [p=%d]", newView, s.pipe) + s.eventLoop.AddScopedEvent(s.pipe, ViewChangeEvent{View: newView, Timeout: timeout, Pipe: s.pipe}) leader := s.leaderRotation.GetLeader(newView) if leader == s.opts.ID() { @@ -352,7 +368,7 @@ func (s *Synchronizer) AdvanceView(syncInfo hotstuff.SyncInfo) { // This method is meant to be used instead of the exported UpdateHighQC internally // in this package when the qc has already been verified. func (s *Synchronizer) updateHighQC(qc hotstuff.QuorumCert) { - newBlock, ok := s.blockChain.Get(qc.BlockHash()) + newBlock, ok := s.blockChain.Get(qc.BlockHash(), qc.Pipe()) if !ok { s.logger.Info("updateHighQC: Could not find block referenced by new QC!") return @@ -360,7 +376,7 @@ func (s *Synchronizer) updateHighQC(qc hotstuff.QuorumCert) { if newBlock.View() > s.highQC.View() { s.highQC = qc - s.logger.Debug("HighQC updated") + s.logger.Debugf("[p=%d, view=%d] HighQC updated", s.pipe, s.View()) } } @@ -377,10 +393,12 @@ var _ modules.Synchronizer = (*Synchronizer)(nil) // ViewChangeEvent is sent on the eventloop whenever a view change occurs. type ViewChangeEvent struct { View hotstuff.View + Pipe hotstuff.Pipe Timeout bool } // TimeoutEvent is sent on the eventloop when a local timeout occurs. type TimeoutEvent struct { View hotstuff.View + Pipe hotstuff.Pipe } diff --git a/synchronizer/synchronizer_test.go b/synchronizer/synchronizer_test.go index fcb603d8..dd26f7c6 100644 --- a/synchronizer/synchronizer_test.go +++ b/synchronizer/synchronizer_test.go @@ -16,19 +16,25 @@ func TestAdvanceViewQC(t *testing.T) { const n = 4 ctrl := gomock.NewController(t) builders := testutil.CreateBuilders(t, ctrl, n) - s := synchronizer.New(testutil.FixedTimeout(1000)) + s := synchronizer.New() + d := testutil.FixedTimeout(1000) hs := mocks.NewMockConsensus(ctrl) - builders[0].Add(s, hs) + builders[0].Add(s, d, hs) hl := builders.Build() signers := hl.Signers() block := hotstuff.NewBlock( hotstuff.GetGenesis().Hash(), - hotstuff.NewQuorumCert(nil, 0, hotstuff.GetGenesis().Hash()), + hotstuff.NewQuorumCert( + nil, + 0, + hotstuff.NullPipe, // TODO: Verify if this code conflicts with pipelining + hotstuff.GetGenesis().Hash()), "foo", 1, 2, + 0, ) var blockChain modules.BlockChain @@ -37,9 +43,9 @@ func TestAdvanceViewQC(t *testing.T) { blockChain.Store(block) qc := testutil.CreateQC(t, block, signers) // synchronizer should tell hotstuff to propose - hs.EXPECT().Propose(gomock.AssignableToTypeOf(hotstuff.NewSyncInfo())) + hs.EXPECT().Propose(gomock.AssignableToTypeOf(hotstuff.NewSyncInfo(hotstuff.NullPipe))) - s.AdvanceView(hotstuff.NewSyncInfo().WithQC(qc)) + s.AdvanceView(hotstuff.NewSyncInfo(hotstuff.NullPipe).WithQC(qc)) if s.View() != 2 { t.Errorf("wrong view: expected: %v, got: %v", 2, s.View()) @@ -50,9 +56,10 @@ func TestAdvanceViewTC(t *testing.T) { const n = 4 ctrl := gomock.NewController(t) builders := testutil.CreateBuilders(t, ctrl, n) - s := synchronizer.New(testutil.FixedTimeout(100)) + s := synchronizer.New() + d := testutil.FixedTimeout(100) hs := mocks.NewMockConsensus(ctrl) - builders[0].Add(s, hs) + builders[0].Add(s, d, hs) hl := builders.Build() signers := hl.Signers() @@ -60,9 +67,9 @@ func TestAdvanceViewTC(t *testing.T) { tc := testutil.CreateTC(t, 1, signers) // synchronizer should tell hotstuff to propose - hs.EXPECT().Propose(gomock.AssignableToTypeOf(hotstuff.NewSyncInfo())) + hs.EXPECT().Propose(gomock.AssignableToTypeOf(hotstuff.NewSyncInfo(hotstuff.NullPipe))) - s.AdvanceView(hotstuff.NewSyncInfo().WithTC(tc)) + s.AdvanceView(hotstuff.NewSyncInfo(hotstuff.NullPipe).WithTC(tc)) if s.View() != 2 { t.Errorf("wrong view: expected: %v, got: %v", 2, s.View()) diff --git a/twins/fhsbug_test.go b/twins/fhsbug_test.go index f0cdc3ba..f39a55c8 100644 --- a/twins/fhsbug_test.go +++ b/twins/fhsbug_test.go @@ -141,13 +141,13 @@ type vulnerableFHS struct { inner fasthotstuff.FastHotStuff } -func (fhs *vulnerableFHS) InitModule(mods *modules.Core) { +func (fhs *vulnerableFHS) InitModule(mods *modules.Core, info modules.ScopeInfo) { mods.Get( &fhs.logger, &fhs.blockChain, ) - fhs.inner.InitModule(mods) + fhs.inner.InitModule(mods, info) } // VoteRule decides whether to vote for the block. @@ -159,7 +159,7 @@ func (fhs *vulnerableFHS) qcRef(qc hotstuff.QuorumCert) (*hotstuff.Block, bool) if (hotstuff.Hash{}) == qc.BlockHash() { return nil, false } - return fhs.blockChain.Get(qc.BlockHash()) + return fhs.blockChain.Get(qc.BlockHash(), qc.Pipe()) } // CommitRule decides whether an ancestor of the block can be committed. diff --git a/twins/network.go b/twins/network.go index 82fe2049..e2ccef87 100644 --- a/twins/network.go +++ b/twins/network.go @@ -11,6 +11,7 @@ import ( "github.com/relab/hotstuff" "github.com/relab/hotstuff/blockchain" + "github.com/relab/hotstuff/committer" "github.com/relab/hotstuff/consensus" "github.com/relab/hotstuff/crypto" "github.com/relab/hotstuff/crypto/ecdsa" @@ -37,10 +38,10 @@ func (id NodeID) String() string { type node struct { blockChain modules.BlockChain consensus modules.Consensus - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop leaderRotation modules.LeaderRotation - synchronizer modules.Synchronizer opts *modules.Options + synchronizer modules.Synchronizer id NodeID executedBlocks []*hotstuff.Block @@ -48,7 +49,11 @@ type node struct { log strings.Builder } -func (n *node) InitModule(mods *modules.Core) { +func (n *node) InitModule(mods *modules.Core, info modules.ScopeInfo) { + if info.IsPipeliningEnabled { + panic("pipelining not supported for this module") + } + mods.Get( &n.blockChain, &n.consensus, @@ -137,13 +142,17 @@ func (n *Network) createTwinsNodes(nodes []NodeID, _ Scenario, consensusName str if !ok { return fmt.Errorf("unknown consensus module: '%s'", consensusName) } + builder.Add( - eventloop.New(100), + eventloop.NewScoped(100, 0), blockchain.New(), - consensus.New(consensusModule), + committer.New(), + consensus.New(), + consensusModule, consensus.NewVotingMachine(), crypto.NewCache(ecdsa.New(), 100), - synchronizer.New(FixedTimeout(1*time.Millisecond)), + synchronizer.New(), + FixedTimeout(1*time.Millisecond), logging.NewWithDest(&node.log, fmt.Sprintf("r%dn%d", nodeID.ReplicaID, nodeID.NetworkID)), // twins-specific: &configuration{network: n, node: node}, @@ -234,7 +243,7 @@ type configuration struct { } // alternative way to get a pointer to the node. -func (c *configuration) InitModule(mods *modules.Core) { +func (c *configuration) InitModule(mods *modules.Core, _ modules.ScopeInfo) { if c.node == nil { mods.TryGet(&c.node) } @@ -428,7 +437,7 @@ type tick struct{} type timeoutManager struct { synchronizer modules.Synchronizer - eventLoop *eventloop.EventLoop + eventLoop *eventloop.ScopedEventLoop node *node network *Network @@ -460,7 +469,7 @@ func (tm *timeoutManager) viewChange(event synchronizer.ViewChangeEvent) { // InitModule gives the module a reference to the Modules object. // It also allows the module to set module options using the OptionsBuilder. -func (tm *timeoutManager) InitModule(mods *modules.Core) { +func (tm *timeoutManager) InitModule(mods *modules.Core, info modules.ScopeInfo) { mods.Get( &tm.synchronizer, &tm.eventLoop, diff --git a/types.go b/types.go index 51cbfe8c..8d50c063 100644 --- a/types.go +++ b/types.go @@ -134,18 +134,19 @@ type ThresholdSignature = QuorumSignature type PartialCert struct { // shortcut to the signer of the signature signer ID + pipe Pipe signature QuorumSignature blockHash Hash } // NewPartialCert returns a new partial certificate. -func NewPartialCert(signature QuorumSignature, blockHash Hash) PartialCert { +func NewPartialCert(pipe Pipe, signature QuorumSignature, blockHash Hash) PartialCert { var signer ID signature.Participants().RangeWhile(func(i ID) bool { signer = i return false }) - return PartialCert{signer, signature, blockHash} + return PartialCert{signer, pipe, signature, blockHash} } // Signer returns the ID of the replica that created the certificate. @@ -163,6 +164,11 @@ func (pc PartialCert) BlockHash() Hash { return pc.blockHash } +// Pipe returns which pipe the PartialCert belongs to. +func (pc PartialCert) Pipe() Pipe { + return pc.pipe +} + // ToBytes returns a byte representation of the partial certificate. func (pc PartialCert) ToBytes() []byte { return append(pc.blockHash[:], pc.signature.ToBytes()...) @@ -176,11 +182,12 @@ type SyncInfo struct { qc *QuorumCert tc *TimeoutCert aggQC *AggregateQC + pipe Pipe } // NewSyncInfo returns a new SyncInfo struct. -func NewSyncInfo() SyncInfo { - return SyncInfo{} +func NewSyncInfo(pipe Pipe) SyncInfo { + return SyncInfo{pipe: pipe} } // WithQC returns a copy of the SyncInfo struct with the given QC. @@ -228,6 +235,10 @@ func (si SyncInfo) AggQC() (_ AggregateQC, _ bool) { return } +func (si SyncInfo) Pipe() Pipe { + return si.pipe +} + func (si SyncInfo) String() string { var sb strings.Builder sb.WriteString("{ ") @@ -248,12 +259,13 @@ func (si SyncInfo) String() string { type QuorumCert struct { signature QuorumSignature view View + pipe Pipe hash Hash } // NewQuorumCert creates a new quorum cert from the given values. -func NewQuorumCert(signature QuorumSignature, view View, hash Hash) QuorumCert { - return QuorumCert{signature, view, hash} +func NewQuorumCert(signature QuorumSignature, view View, pipe Pipe, hash Hash) QuorumCert { + return QuorumCert{signature, view, pipe, hash} } // ToBytes returns a byte representation of the quorum certificate. @@ -281,6 +293,10 @@ func (qc QuorumCert) View() View { return qc.view } +func (qc QuorumCert) Pipe() Pipe { + return qc.pipe +} + // Equals returns true if the other QC equals this QC. func (qc QuorumCert) Equals(other QuorumCert) bool { if qc.view != other.view { @@ -383,3 +399,21 @@ func writeParticipants(wr io.Writer, participants IDSet) (err error) { }) return err } + +// Pipe is a number indicating the pipe. +type Pipe uint32 + +// Flag for no piplining and identifier for unscoped modules and events. +const NullPipe = Pipe(0) + +// ToBytes returns the pipe id as bytes. +func (p Pipe) ToBytes() []byte { + var viewBytes [4]byte + binary.LittleEndian.PutUint32(viewBytes[:], uint32(p)) + return viewBytes[:] +} + +// If the pipe ID is not NullPipe, then return true +func (i Pipe) IsNull() bool { + return i != NullPipe +}