change backoff algorithm to exponential backoff (#726)

change backoff algorithm to exponential backoff

op/reboot.go

@@ -16,7 +16,8 @@ import (
 	"k8s.io/client-go/kubernetes"
 )
 
-const drainBackOffBaseSeconds = 60
+const drainBackOffBaseSeconds = 300
+const drainBackOffMaxSeconds = 1200
 
 type rebootDrainStartOp struct {
 	finished bool
@@ -571,7 +572,12 @@ func drainBackOff(ctx context.Context, inf cke.Infrastructure, entry *cke.Reboot
 	entry.Status = cke.RebootStatusQueued
 	entry.LastTransitionTime = time.Now().Truncate(time.Second).UTC()
 	entry.DrainBackOffCount++
-	entry.DrainBackOffExpire = entry.LastTransitionTime.Add(time.Second * time.Duration(drainBackOffBaseSeconds+rand.Int63n(int64(drainBackOffBaseSeconds*entry.DrainBackOffCount))))
+	var backoffSeconds int64 = (1 << (entry.DrainBackOffCount - 1)) * drainBackOffBaseSeconds
+	if backoffSeconds > drainBackOffMaxSeconds {
+		backoffSeconds = drainBackOffMaxSeconds
+	}
+	entry.DrainBackOffExpire = entry.LastTransitionTime.Add(time.Second * time.Duration(backoffSeconds+rand.Int63n(drainBackOffBaseSeconds)))
+
 	err = inf.Storage().UpdateRebootsEntry(ctx, entry)
 	if err != nil {
 		return err