Skip to content

Commit f21df6e

Browse files
Merge pull request #118132 from marseel/improve_reflector_retries
Improve backoff policy in reflector. Kubernetes-commit: 7f2a1e8cd3363b04d970c619daf9f9d5ee5d50c0
2 parents c670796 + bae1024 commit f21df6e

File tree

4 files changed

+30
-33
lines changed

4 files changed

+30
-33
lines changed

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ require (
2323
golang.org/x/term v0.7.0
2424
golang.org/x/time v0.3.0
2525
google.golang.org/protobuf v1.30.0
26-
k8s.io/api v0.0.0-20230515170019-2f9553831ec2
26+
k8s.io/api v0.0.0-20230527221710-63505c4e796f
2727
k8s.io/apimachinery v0.0.0-20230526141509-403fb36f6a53
2828
k8s.io/klog/v2 v2.100.1
2929
k8s.io/kube-openapi v0.0.0-20230524182850-78281498afbb
@@ -60,6 +60,6 @@ require (
6060
)
6161

6262
replace (
63-
k8s.io/api => k8s.io/api v0.0.0-20230515170019-2f9553831ec2
63+
k8s.io/api => k8s.io/api v0.0.0-20230527221710-63505c4e796f
6464
k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20230526141509-403fb36f6a53
6565
)

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -210,8 +210,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
210210
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
211211
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
212212
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
213-
k8s.io/api v0.0.0-20230515170019-2f9553831ec2 h1:ZHyoWIXElhnLI03GkOsebYT1uqYowGjuVCbxBqcjjGw=
214-
k8s.io/api v0.0.0-20230515170019-2f9553831ec2/go.mod h1:qZNz3vKcyZb5wFaBlHqUjHk7D5XhIaXu/d/df6PjNKk=
213+
k8s.io/api v0.0.0-20230527221710-63505c4e796f h1:5n05Z8OLrrVvkQF6z0OgUK3yGzRm7GY1bUwhUBp3g5g=
214+
k8s.io/api v0.0.0-20230527221710-63505c4e796f/go.mod h1:iclI1239YG5Z5YyOH/tKJi2vGhQ0k0dl+1xSvFaR6Dk=
215215
k8s.io/apimachinery v0.0.0-20230526141509-403fb36f6a53 h1:8/61R28R74ZzZ1WiUVu4zqCtyQSdVCBa4qCNjPLjOsU=
216216
k8s.io/apimachinery v0.0.0-20230526141509-403fb36f6a53/go.mod h1:qCTDst7QeP2n3JDxbpuJSTTaAxalvFKMTN0Lga1+0zU=
217217
k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=

tools/cache/reflector.go

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,7 @@ type Reflector struct {
7070
listerWatcher ListerWatcher
7171
// backoff manages backoff of ListWatch
7272
backoffManager wait.BackoffManager
73-
// initConnBackoffManager manages backoff the initial connection with the Watch call of ListAndWatch.
74-
initConnBackoffManager wait.BackoffManager
75-
resyncPeriod time.Duration
73+
resyncPeriod time.Duration
7674
// clock allows tests to manipulate time
7775
clock clock.Clock
7876
// paginatedResult defines whether pagination should be forced for list calls.
@@ -221,11 +219,10 @@ func NewReflectorWithOptions(lw ListerWatcher, expectedType interface{}, store S
221219
// We used to make the call every 1sec (1 QPS), the goal here is to achieve ~98% traffic reduction when
222220
// API server is not healthy. With these parameters, backoff will stop at [30,60) sec interval which is
223221
// 0.22 QPS. If we don't backoff for 2min, assume API server is healthy and we reset the backoff.
224-
backoffManager: wait.NewExponentialBackoffManager(800*time.Millisecond, 30*time.Second, 2*time.Minute, 2.0, 1.0, reflectorClock),
225-
initConnBackoffManager: wait.NewExponentialBackoffManager(800*time.Millisecond, 30*time.Second, 2*time.Minute, 2.0, 1.0, reflectorClock),
226-
clock: reflectorClock,
227-
watchErrorHandler: WatchErrorHandler(DefaultWatchErrorHandler),
228-
expectedType: reflect.TypeOf(expectedType),
222+
backoffManager: wait.NewExponentialBackoffManager(800*time.Millisecond, 30*time.Second, 2*time.Minute, 2.0, 1.0, reflectorClock),
223+
clock: reflectorClock,
224+
watchErrorHandler: WatchErrorHandler(DefaultWatchErrorHandler),
225+
expectedType: reflect.TypeOf(expectedType),
229226
}
230227

231228
if r.name == "" {
@@ -425,7 +422,7 @@ func (r *Reflector) watch(w watch.Interface, stopCh <-chan struct{}, resyncerrc
425422
select {
426423
case <-stopCh:
427424
return nil
428-
case <-r.initConnBackoffManager.Backoff().C():
425+
case <-r.backoffManager.Backoff().C():
429426
continue
430427
}
431428
}
@@ -451,7 +448,7 @@ func (r *Reflector) watch(w watch.Interface, stopCh <-chan struct{}, resyncerrc
451448
select {
452449
case <-stopCh:
453450
return nil
454-
case <-r.initConnBackoffManager.Backoff().C():
451+
case <-r.backoffManager.Backoff().C():
455452
continue
456453
}
457454
case apierrors.IsInternalError(err) && retry.ShouldRetry():
@@ -604,7 +601,7 @@ func (r *Reflector) watchList(stopCh <-chan struct{}) (watch.Interface, error) {
604601
isErrorRetriableWithSideEffectsFn := func(err error) bool {
605602
if canRetry := isWatchErrorRetriable(err); canRetry {
606603
klog.V(2).Infof("%s: watch-list of %v returned %v - backing off", r.name, r.typeDescription, err)
607-
<-r.initConnBackoffManager.Backoff().C()
604+
<-r.backoffManager.Backoff().C()
608605
return true
609606
}
610607
if isExpiredError(err) || isTooLargeResourceVersionError(err) {

tools/cache/reflector_test.go

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -411,12 +411,12 @@ func TestReflectorListAndWatchInitConnBackoff(t *testing.T) {
411411
},
412412
}
413413
r := &Reflector{
414-
name: "test-reflector",
415-
listerWatcher: lw,
416-
store: NewFIFO(MetaNamespaceKeyFunc),
417-
initConnBackoffManager: bm,
418-
clock: fakeClock,
419-
watchErrorHandler: WatchErrorHandler(DefaultWatchErrorHandler),
414+
name: "test-reflector",
415+
listerWatcher: lw,
416+
store: NewFIFO(MetaNamespaceKeyFunc),
417+
backoffManager: bm,
418+
clock: fakeClock,
419+
watchErrorHandler: WatchErrorHandler(DefaultWatchErrorHandler),
420420
}
421421
start := fakeClock.Now()
422422
err := r.ListAndWatch(stopCh)
@@ -471,12 +471,12 @@ func TestBackoffOnTooManyRequests(t *testing.T) {
471471
}
472472

473473
r := &Reflector{
474-
name: "test-reflector",
475-
listerWatcher: lw,
476-
store: NewFIFO(MetaNamespaceKeyFunc),
477-
initConnBackoffManager: bm,
478-
clock: clock,
479-
watchErrorHandler: WatchErrorHandler(DefaultWatchErrorHandler),
474+
name: "test-reflector",
475+
listerWatcher: lw,
476+
store: NewFIFO(MetaNamespaceKeyFunc),
477+
backoffManager: bm,
478+
clock: clock,
479+
watchErrorHandler: WatchErrorHandler(DefaultWatchErrorHandler),
480480
}
481481

482482
stopCh := make(chan struct{})
@@ -540,12 +540,12 @@ func TestRetryInternalError(t *testing.T) {
540540
}
541541

542542
r := &Reflector{
543-
name: "test-reflector",
544-
listerWatcher: lw,
545-
store: NewFIFO(MetaNamespaceKeyFunc),
546-
initConnBackoffManager: bm,
547-
clock: fakeClock,
548-
watchErrorHandler: WatchErrorHandler(DefaultWatchErrorHandler),
543+
name: "test-reflector",
544+
listerWatcher: lw,
545+
store: NewFIFO(MetaNamespaceKeyFunc),
546+
backoffManager: bm,
547+
clock: fakeClock,
548+
watchErrorHandler: WatchErrorHandler(DefaultWatchErrorHandler),
549549
}
550550

551551
r.MaxInternalErrorRetryDuration = tc.maxInternalDuration

0 commit comments

Comments
 (0)