gha: make MaxConcurrentReconciles for each reconciler configurable

Author: tfujiwar

.github/workflows/gha-e2e-tests.yaml

@@ -4,9 +4,11 @@ on:
   push:
     branches:
       - master
+      - mercari-master
   pull_request:
     branches:
       - master
+      - mercari-master
   workflow_dispatch:
 
 permissions:

.github/workflows/gha-validate-chart.yaml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - master
+      - mercari-master
     paths:
       - 'charts/**'
       - '.github/workflows/gha-validate-chart.yaml'

charts/gha-runner-scale-set-controller/templates/deployment.yaml

@@ -82,6 +82,18 @@ spec:
         {{- range .Values.flags.excludeLabelPropagationPrefixes }}
         - "--exclude-label-propagation-prefix={{ . }}"
         {{- end }}
+        {{- with .Values.flags.maxConcurrentReconcilesForAutoscalingRunnerSet }}
+        - "--max-concurrent-reconciles-for-autoscaling-runner-set={{ . }}"
+        {{- end }}
+        {{- with .Values.flags.maxConcurrentReconcilesForEphemeralRunnerSet }}
+        - "--max-concurrent-reconciles-for-ephemeral-runner-set={{ . }}"
+        {{- end }}
+        {{- with .Values.flags.maxConcurrentReconcilesForEphemeralRunner }}
+        - "--max-concurrent-reconciles-for-ephemeral-runner={{ . }}"
+        {{- end }}
+        {{- with .Values.flags.maxConcurrentReconcilesForAutoscalingListener }}
+        - "--max-concurrent-reconciles-for-autoscaling-listener={{ . }}"
+        {{- end }}
         command:
         - "/manager"
         {{- with .Values.metrics }}

charts/gha-runner-scale-set-controller/values.yaml

@@ -130,3 +130,9 @@ flags:
   ## Labels that match prefix specified in the list are excluded from propagation.
   # excludeLabelPropagationPrefixes:
   #   - "argocd.argoproj.io/instance"
+
+  ## Defines the maximum number of concurrent reconciles for each reconciler.
+  # maxConcurrentReconcilesForAutoscalingRunnerSet: 1
+  # maxConcurrentReconcilesForEphemeralRunnerSet: 1
+  # maxConcurrentReconcilesForEphemeralRunner: 1
+  # maxConcurrentReconcilesForAutoscalingListener: 1

controllers/actions.github.com/autoscalinglistener_controller.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -55,6 +56,8 @@ type AutoscalingListenerReconciler struct {
 	ListenerMetricsAddr     string
 	ListenerMetricsEndpoint string
 
+	MaxConcurrentReconciles int
+
 	ResourceBuilder
 }
 
@@ -730,6 +733,7 @@ func (r *AutoscalingListenerReconciler) SetupWithManager(mgr ctrl.Manager) error
 		Watches(&rbacv1.Role{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)).
 		Watches(&rbacv1.RoleBinding{}, handler.EnqueueRequestsFromMapFunc(labelBasedWatchFunc)).
 		WithEventFilter(predicate.ResourceVersionChangedPredicate{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}).
 		Complete(r)
 }
 

controllers/actions.github.com/autoscalingrunnerset_controller.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -79,6 +80,7 @@ type AutoscalingRunnerSetReconciler struct {
 	DefaultRunnerScaleSetListenerImagePullSecrets []string
 	UpdateStrategy                                UpdateStrategy
 	ActionsClient                                 actions.MultiClient
+	MaxConcurrentReconciles                       int
 	ResourceBuilder
 }
 
@@ -763,6 +765,7 @@ func (r *AutoscalingRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) erro
 			},
 		)).
 		WithEventFilter(predicate.ResourceVersionChangedPredicate{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}).
 		Complete(r)
 }
 

controllers/actions.github.com/ephemeralrunner_controller.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
@@ -49,9 +50,10 @@ const (
 // EphemeralRunnerReconciler reconciles a EphemeralRunner object
 type EphemeralRunnerReconciler struct {
 	client.Client
-	Log           logr.Logger
-	Scheme        *runtime.Scheme
-	ActionsClient actions.MultiClient
+	Log                     logr.Logger
+	Scheme                  *runtime.Scheme
+	ActionsClient           actions.MultiClient
+	MaxConcurrentReconciles int
 	ResourceBuilder
 }
 
@@ -828,6 +830,7 @@ func (r *EphemeralRunnerReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		For(&v1alpha1.EphemeralRunner{}).
 		Owns(&corev1.Pod{}).
 		WithEventFilter(predicate.ResourceVersionChangedPredicate{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}).
 		Complete(r)
 }
 

controllers/actions.github.com/ephemeralrunnerset_controller.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 )
@@ -53,6 +54,8 @@ type EphemeralRunnerSetReconciler struct {
 
 	PublishMetrics bool
 
+	MaxConcurrentReconciles int
+
 	ResourceBuilder
 }
 
@@ -575,6 +578,7 @@ func (r *EphemeralRunnerSetReconciler) SetupWithManager(mgr ctrl.Manager) error
 		For(&v1alpha1.EphemeralRunnerSet{}).
 		Owns(&v1alpha1.EphemeralRunner{}).
 		WithEventFilter(predicate.ResourceVersionChangedPredicate{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxConcurrentReconciles}).
 		Complete(r)
 }
 

main.go

@@ -103,6 +103,11 @@ func main() {
 		autoScalerImagePullSecrets stringSlice
 
 		commonRunnerLabels commaSeparatedStringSlice
+
+		maxConcurrentReconcilesForAutoscalingRunnerSet int
+		maxConcurrentReconcilesForEphemeralRunnerSet   int
+		maxConcurrentReconcilesForEphemeralRunner      int
+		maxConcurrentReconcilesForAutoscalingListener  int
 	)
 	var c github.Config
 	err = envconfig.Process("github", &c)
@@ -145,6 +150,10 @@ func main() {
 	flag.BoolVar(&autoScalingRunnerSetOnly, "auto-scaling-runner-set-only", false, "Make controller only reconcile AutoRunnerScaleSet object.")
 	flag.StringVar(&updateStrategy, "update-strategy", "immediate", `Resources reconciliation strategy on upgrade with running/pending jobs. Valid values are: "immediate", "eventual". Defaults to "immediate".`)
 	flag.Var(&autoScalerImagePullSecrets, "auto-scaler-image-pull-secrets", "The default image-pull secret name for auto-scaler listener container.")
+	flag.IntVar(&maxConcurrentReconcilesForAutoscalingRunnerSet, "max-concurrent-reconciles-for-autoscaling-runner-set", 1, "The maximum number of concurrent reconciles for AutoscalingRunnerSet.")
+	flag.IntVar(&maxConcurrentReconcilesForEphemeralRunnerSet, "max-concurrent-reconciles-for-ephemeral-runner-set", 1, "The maximum number of concurrent reconciles for EphemeralRunnerSet.")
+	flag.IntVar(&maxConcurrentReconcilesForEphemeralRunner, "max-concurrent-reconciles-for-ephemeral-runner", 1, "The maximum number of concurrent reconciles for EphemeralRunner.")
+	flag.IntVar(&maxConcurrentReconcilesForAutoscalingListener, "max-concurrent-reconciles-for-autoscaling-listener", 1, "The maximum number of concurrent reconciles for AutoscalingListener.")
 	flag.Parse()
 
 	runnerPodDefaults.RunnerImagePullSecrets = runnerImagePullSecrets
@@ -273,30 +282,33 @@ func main() {
 			ActionsClient:                      actionsMultiClient,
 			UpdateStrategy:                     actionsgithubcom.UpdateStrategy(updateStrategy),
 			DefaultRunnerScaleSetListenerImagePullSecrets: autoScalerImagePullSecrets,
-			ResourceBuilder: rb,
+			MaxConcurrentReconciles:                       maxConcurrentReconcilesForAutoscalingRunnerSet,
+			ResourceBuilder:                               rb,
 		}).SetupWithManager(mgr); err != nil {
 			log.Error(err, "unable to create controller", "controller", "AutoscalingRunnerSet")
 			os.Exit(1)
 		}
 
 		if err = (&actionsgithubcom.EphemeralRunnerReconciler{
-			Client:          mgr.GetClient(),
-			Log:             log.WithName("EphemeralRunner").WithValues("version", build.Version),
-			Scheme:          mgr.GetScheme(),
-			ActionsClient:   actionsMultiClient,
-			ResourceBuilder: rb,
+			Client:                  mgr.GetClient(),
+			Log:                     log.WithName("EphemeralRunner").WithValues("version", build.Version),
+			Scheme:                  mgr.GetScheme(),
+			ActionsClient:           actionsMultiClient,
+			MaxConcurrentReconciles: maxConcurrentReconcilesForEphemeralRunner,
+			ResourceBuilder:         rb,
 		}).SetupWithManager(mgr); err != nil {
 			log.Error(err, "unable to create controller", "controller", "EphemeralRunner")
 			os.Exit(1)
 		}
 
 		if err = (&actionsgithubcom.EphemeralRunnerSetReconciler{
-			Client:          mgr.GetClient(),
-			Log:             log.WithName("EphemeralRunnerSet").WithValues("version", build.Version),
-			Scheme:          mgr.GetScheme(),
-			ActionsClient:   actionsMultiClient,
-			PublishMetrics:  metricsAddr != "0",
-			ResourceBuilder: rb,
+			Client:                  mgr.GetClient(),
+			Log:                     log.WithName("EphemeralRunnerSet").WithValues("version", build.Version),
+			Scheme:                  mgr.GetScheme(),
+			ActionsClient:           actionsMultiClient,
+			PublishMetrics:          metricsAddr != "0",
+			MaxConcurrentReconciles: maxConcurrentReconcilesForEphemeralRunnerSet,
+			ResourceBuilder:         rb,
 		}).SetupWithManager(mgr); err != nil {
 			log.Error(err, "unable to create controller", "controller", "EphemeralRunnerSet")
 			os.Exit(1)
@@ -308,6 +320,7 @@ func main() {
 			Scheme:                  mgr.GetScheme(),
 			ListenerMetricsAddr:     listenerMetricsAddr,
 			ListenerMetricsEndpoint: listenerMetricsEndpoint,
+			MaxConcurrentReconciles: maxConcurrentReconcilesForAutoscalingListener,
 			ResourceBuilder:         rb,
 		}).SetupWithManager(mgr); err != nil {
 			log.Error(err, "unable to create controller", "controller", "AutoscalingListener")