Skip to content

Commit

Permalink
Purge left nodes along with dead ones (#254)
Browse files Browse the repository at this point in the history
Keep the state from infinitely expanding
  • Loading branch information
bwaters authored Jan 12, 2022
1 parent 5f7e384 commit 0bff309
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 4 deletions.
4 changes: 2 additions & 2 deletions util.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,13 +96,13 @@ func pushPullScale(interval time.Duration, n int) time.Duration {
return time.Duration(multiplier) * interval
}

// moveDeadNodes moves nodes that are dead and beyond the gossip to the dead interval
// moveDeadNodes moves dead and left nodes that that have not changed during the gossipToTheDeadTime interval
// to the end of the slice and returns the index of the first moved node.
func moveDeadNodes(nodes []*nodeState, gossipToTheDeadTime time.Duration) int {
numDead := 0
n := len(nodes)
for i := 0; i < n-numDead; i++ {
if nodes[i].State != StateDead {
if !nodes[i].DeadOrLeft() {
continue
}

Expand Down
23 changes: 21 additions & 2 deletions util_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,16 @@ func TestMoveDeadNodes(t *testing.T) {
State: StateDead,
StateChange: time.Now().Add(-10 * time.Second),
},
// This left node should not be moved, as its state changed
// less than the specified GossipToTheDead time ago
&nodeState{
State: StateLeft,
StateChange: time.Now().Add(-10 * time.Second),
},
&nodeState{
State: StateLeft,
StateChange: time.Now().Add(-20 * time.Second),
},
&nodeState{
State: StateAlive,
StateChange: time.Now().Add(-20 * time.Second),
Expand All @@ -190,10 +200,14 @@ func TestMoveDeadNodes(t *testing.T) {
State: StateAlive,
StateChange: time.Now().Add(-20 * time.Second),
},
&nodeState{
State: StateLeft,
StateChange: time.Now().Add(-20 * time.Second),
},
}

idx := moveDeadNodes(nodes, (15 * time.Second))
if idx != 4 {
if idx != 5 {
t.Fatalf("bad index")
}
for i := 0; i < idx; i++ {
Expand All @@ -204,14 +218,19 @@ func TestMoveDeadNodes(t *testing.T) {
if nodes[i].State != StateDead {
t.Fatalf("Bad state %d", i)
}
case 3:
//Recently left node should remain at 3
if nodes[i].State != StateLeft {
t.Fatalf("Bad State %d", i)
}
default:
if nodes[i].State != StateAlive {
t.Fatalf("Bad state %d", i)
}
}
}
for i := idx; i < len(nodes); i++ {
if nodes[i].State != StateDead {
if !nodes[i].DeadOrLeft() {
t.Fatalf("Bad state %d", i)
}
}
Expand Down

0 comments on commit 0bff309

Please sign in to comment.