diff --git a/charts/gha-runner-scale-set-controller/templates/deployment.yaml b/charts/gha-runner-scale-set-controller/templates/deployment.yaml index 10428912cb..2f7602ae2a 100644 --- a/charts/gha-runner-scale-set-controller/templates/deployment.yaml +++ b/charts/gha-runner-scale-set-controller/templates/deployment.yaml @@ -103,6 +103,9 @@ spec: {{- with .Values.flags.clientGoRateLimiterBurst }} - "--client-go-rate-limiter-burst={{ . }}" {{- end }} + {{- with .Values.flags.disableWorkqueueBucketRateLimiter }} + - "--disable-workqueue-bucket-rate-limiter={{ . }}" + {{- end }} command: - "/manager" {{- if or .Values.metrics .Values.pprof }} diff --git a/charts/gha-runner-scale-set-controller/values.yaml b/charts/gha-runner-scale-set-controller/values.yaml index 621001ea16..75e5941b11 100644 --- a/charts/gha-runner-scale-set-controller/values.yaml +++ b/charts/gha-runner-scale-set-controller/values.yaml @@ -144,3 +144,4 @@ flags: ## Defines the client-go rate limiter parameters. # clientGoRateLimiterQPS: 20 # clientGoRateLimiterBurst: 30 + # disableWorkqueueBucketRateLimiter: false diff --git a/controllers/actions.github.com/autoscalinglistener_controller.go b/controllers/actions.github.com/autoscalinglistener_controller.go index bcffd9228e..04d79492f0 100644 --- a/controllers/actions.github.com/autoscalinglistener_controller.go +++ b/controllers/actions.github.com/autoscalinglistener_controller.go @@ -24,6 +24,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -57,6 +58,7 @@ type AutoscalingListenerReconciler struct { ListenerMetricsEndpoint string MaxConcurrentReconciles int + WorkqueueRateLimiter workqueue.RateLimiter ResourceBuilder } @@ -733,7 +735,10 @@ func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error Watches(&rbacv1.Role{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). Watches(&rbacv1.RoleBinding{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)). WithEventFilter(predicate.ResourceVersionChangedPredicate{}). - WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: r.MaxConcurrentReconciles, + RateLimiter: r.WorkqueueRateLimiter, + }). Complete(r) } diff --git a/controllers/actions.github.com/autoscalingrunnerset_controller.go b/controllers/actions.github.com/autoscalingrunnerset_controller.go index 84bd059d8a..46d065428f 100644 --- a/controllers/actions.github.com/autoscalingrunnerset_controller.go +++ b/controllers/actions.github.com/autoscalingrunnerset_controller.go @@ -32,6 +32,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -81,6 +82,7 @@ type AutoscalingRunnerSetReconciler struct { UpdateStrategy UpdateStrategy ActionsClient actions.MultiClient MaxConcurrentReconciles int + WorkqueueRateLimiter workqueue.RateLimiter ResourceBuilder } @@ -765,7 +767,10 @@ func (r *AutoscalingRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) erro }, )). WithEventFilter(predicate.ResourceVersionChangedPredicate{}). - WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: r.MaxConcurrentReconciles, + RateLimiter: r.WorkqueueRateLimiter, + }). Complete(r) } diff --git a/controllers/actions.github.com/ephemeralrunner_controller.go b/controllers/actions.github.com/ephemeralrunner_controller.go index 00393adf1c..3339c7f212 100644 --- a/controllers/actions.github.com/ephemeralrunner_controller.go +++ b/controllers/actions.github.com/ephemeralrunner_controller.go @@ -31,6 +31,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -54,6 +55,7 @@ type EphemeralRunnerReconciler struct { Scheme *runtime.Scheme ActionsClient actions.MultiClient MaxConcurrentReconciles int + WorkqueueRateLimiter workqueue.RateLimiter ResourceBuilder } @@ -837,7 +839,10 @@ func (r *EphemeralRunnerReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&v1alpha1.EphemeralRunner{}). Owns(&corev1.Pod{}). WithEventFilter(predicate.ResourceVersionChangedPredicate{}). - WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: r.MaxConcurrentReconciles, + RateLimiter: r.WorkqueueRateLimiter, + }). Complete(r) } diff --git a/controllers/actions.github.com/ephemeralrunnerset_controller.go b/controllers/actions.github.com/ephemeralrunnerset_controller.go index 8fdb9c2304..479b95730c 100644 --- a/controllers/actions.github.com/ephemeralrunnerset_controller.go +++ b/controllers/actions.github.com/ephemeralrunnerset_controller.go @@ -34,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -55,6 +56,7 @@ type EphemeralRunnerSetReconciler struct { PublishMetrics bool MaxConcurrentReconciles int + WorkqueueRateLimiter workqueue.RateLimiter ResourceBuilder } @@ -578,7 +580,10 @@ func (r *EphemeralRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error For(&v1alpha1.EphemeralRunnerSet{}). Owns(&v1alpha1.EphemeralRunner{}). WithEventFilter(predicate.ResourceVersionChangedPredicate{}). - WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: r.MaxConcurrentReconciles, + RateLimiter: r.WorkqueueRateLimiter, + }). Complete(r) } diff --git a/main.go b/main.go index cbff64c91b..a67b6a4659 100644 --- a/main.go +++ b/main.go @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" @@ -112,6 +113,8 @@ func main() { rateLimiterQPS int rateLimiterBurst int + + disableWorkqueueBucketRateLimiter bool ) var c github.Config err = envconfig.Process("github", &c) @@ -161,6 +164,7 @@ func main() { flag.IntVar(&maxConcurrentReconcilesForAutoscalingListener, "max-concurrent-reconciles-for-autoscaling-listener", 1, "The maximum number of concurrent reconciles for AutoscalingListener.") flag.IntVar(&rateLimiterQPS, "client-go-rate-limiter-qps", 20, "The QPS value of client-go rate limiter.") flag.IntVar(&rateLimiterBurst, "client-go-rate-limiter-burst", 30, "The burst value of client-go rate limiter.") + flag.BoolVar(&disableWorkqueueBucketRateLimiter, "disable-workqueue-bucket-rate-limiter", false, "Disable workqueue BucketRateLimiter.") flag.Parse() runnerPodDefaults.RunnerImagePullSecrets = runnerImagePullSecrets @@ -265,6 +269,13 @@ func main() { } if autoScalingRunnerSetOnly { + var newWorkqueueRateLimiter func() workqueue.RateLimiter + if disableWorkqueueBucketRateLimiter { + newWorkqueueRateLimiter = workqueue.DefaultItemBasedRateLimiter + } else { + newWorkqueueRateLimiter = workqueue.DefaultControllerRateLimiter + } + if err := actionsgithubcom.SetupIndexers(mgr); err != nil { log.Error(err, "unable to setup indexers") os.Exit(1) @@ -298,6 +309,7 @@ func main() { UpdateStrategy: actionsgithubcom.UpdateStrategy(updateStrategy), DefaultRunnerScaleSetListenerImagePullSecrets: autoScalerImagePullSecrets, MaxConcurrentReconciles: maxConcurrentReconcilesForAutoscalingRunnerSet, + WorkqueueRateLimiter: newWorkqueueRateLimiter(), ResourceBuilder: rb, }).SetupWithManager(mgr); err != nil { log.Error(err, "unable to create controller", "controller", "AutoscalingRunnerSet") @@ -310,6 +322,7 @@ func main() { Scheme: mgr.GetScheme(), ActionsClient: actionsMultiClient, MaxConcurrentReconciles: maxConcurrentReconcilesForEphemeralRunner, + WorkqueueRateLimiter: newWorkqueueRateLimiter(), ResourceBuilder: rb, }).SetupWithManager(mgr); err != nil { log.Error(err, "unable to create controller", "controller", "EphemeralRunner") @@ -323,6 +336,7 @@ func main() { ActionsClient: actionsMultiClient, PublishMetrics: metricsAddr != "0", MaxConcurrentReconciles: maxConcurrentReconcilesForEphemeralRunnerSet, + WorkqueueRateLimiter: newWorkqueueRateLimiter(), ResourceBuilder: rb, }).SetupWithManager(mgr); err != nil { log.Error(err, "unable to create controller", "controller", "EphemeralRunnerSet") @@ -336,6 +350,7 @@ func main() { ListenerMetricsAddr: listenerMetricsAddr, ListenerMetricsEndpoint: listenerMetricsEndpoint, MaxConcurrentReconciles: maxConcurrentReconcilesForAutoscalingListener, + WorkqueueRateLimiter: newWorkqueueRateLimiter(), ResourceBuilder: rb, }).SetupWithManager(mgr); err != nil { log.Error(err, "unable to create controller", "controller", "AutoscalingListener")