diff --git a/cocoonset/agents.go b/cocoonset/agents.go index 75c0b2d..542b014 100644 --- a/cocoonset/agents.go +++ b/cocoonset/agents.go @@ -32,40 +32,77 @@ const subAgentCreateConcurrency = 8 // (non-zero when a sub-agent is in rebuild backoff and the caller should // re-reconcile when backoff elapses). Missing slots are created concurrently // so batch scale-ups do not serialize N apiserver round trips. -func (r *Reconciler) ensureSubAgents(ctx context.Context, cs *cocoonv1.CocoonSet, classified classifiedPods, mainVMName, mainNodeName string) (bool, time.Duration, error) { +func (r *Reconciler) ensureSubAgents(ctx context.Context, cs *cocoonv1.CocoonSet, classified classifiedPods, mainVMName, mainNodeName string, intent restoreIntent) (bool, time.Duration, error) { logger := log.WithFunc("cocoonset.Reconciler.ensureSubAgents") changed := false var requeueAfter time.Duration - restorable, err := r.podsRestorableByCR(ctx, cs.Namespace) + var missing []int32 + for slot := int32(1); slot <= cs.Spec.Agent.Replicas; slot++ { + pod, exists := classified.sub[slot] + if !exists { + missing = append(missing, slot) + continue + } + deleted, wait, err := r.triageSubAgent(ctx, logger, pod, cs, slot) + if err != nil { + return changed, requeueAfter, err + } + if deleted { + changed = true + } + if wait > 0 && (requeueAfter == 0 || wait < requeueAfter) { + requeueAfter = wait + } + } + + created, err := r.createSubAgents(ctx, logger, cs, missing, mainVMName, mainNodeName, intent) + if created { + changed = true + } if err != nil { return changed, requeueAfter, err } + for _, slot := range slices.Sorted(maps.Keys(classified.sub)) { + if slot <= cs.Spec.Agent.Replicas { + continue + } + pod := classified.sub[slot] + if err := r.Delete(ctx, pod); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return changed, requeueAfter, fmt.Errorf("delete extra sub-agent slot %d: %w", slot, err) + } + logger.Infof(ctx, "deleted extra sub-agent %s/%s", pod.Namespace, pod.Name) + changed = true + } + return changed, requeueAfter, nil +} + +// createSubAgents builds the missing slots concurrently so a batch scale-up does +// not serialize N apiserver round trips. +func (r *Reconciler) createSubAgents(ctx context.Context, logger *log.Fields, cs *cocoonv1.CocoonSet, missing []int32, mainVMName, mainNodeName string, intent restoreIntent) (bool, error) { + if len(missing) == 0 { + return false, nil + } + // Resolved before the fan-out so the goroutines only ever read it. + restorable, err := intent() + if err != nil { + return false, err + } g, gctx := errgroup.WithContext(ctx) g.SetLimit(subAgentCreateConcurrency) var created atomic.Bool - for slot := int32(1); slot <= cs.Spec.Agent.Replicas; slot++ { - if pod, exists := classified.sub[slot]; exists { - deleted, wait, err := r.triageSubAgent(ctx, logger, pod, cs, slot) - if err != nil { - return changed, requeueAfter, err - } - if deleted { - changed = true - } - if wait > 0 && (requeueAfter == 0 || wait < requeueAfter) { - requeueAfter = wait - } - continue - } + for _, slot := range missing { g.Go(func() error { subPod, err := buildAgentPod(cs, slot, mainVMName, mainNodeName, r.Scheme) if err != nil { return fmt.Errorf("build sub-agent slot %d: %w", slot, err) } - _, intent := restorable[subPod.Name] - if err := r.markRestoreIfHibernated(gctx, subPod, intent); err != nil { + _, wantRestore := restorable[subPod.Name] + if err := r.markRestoreIfHibernated(gctx, subPod, wantRestore); err != nil { return fmt.Errorf("mark restore sub-agent slot %d: %w", slot, err) } if err := r.Create(gctx, subPod); err != nil { @@ -79,27 +116,8 @@ func (r *Reconciler) ensureSubAgents(ctx context.Context, cs *cocoonv1.CocoonSet return nil }) } - if err := g.Wait(); err != nil { - return changed || created.Load(), requeueAfter, err - } - if created.Load() { - changed = true - } - for _, slot := range slices.Sorted(maps.Keys(classified.sub)) { - if slot <= cs.Spec.Agent.Replicas { - continue - } - pod := classified.sub[slot] - if err := r.Delete(ctx, pod); err != nil { - if apierrors.IsNotFound(err) { - continue - } - return changed, requeueAfter, fmt.Errorf("delete extra sub-agent slot %d: %w", slot, err) - } - logger.Infof(ctx, "deleted extra sub-agent %s/%s", pod.Namespace, pod.Name) - changed = true - } - return changed, requeueAfter, nil + waitErr := g.Wait() + return created.Load(), waitErr } // triageSubAgent deletes pod when it is terminal or has drifted from spec. diff --git a/cocoonset/pods_test.go b/cocoonset/pods_test.go index 1ada991..0adcb83 100644 --- a/cocoonset/pods_test.go +++ b/cocoonset/pods_test.go @@ -559,7 +559,7 @@ func TestPodSpecMatchesAgentIgnoresAffinity(t *testing.T) { } } -func testScheme(t *testing.T) *runtime.Scheme { +func testScheme(t testing.TB) *runtime.Scheme { t.Helper() scheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) diff --git a/cocoonset/reconcile_bench_test.go b/cocoonset/reconcile_bench_test.go new file mode 100644 index 0000000..07e553c --- /dev/null +++ b/cocoonset/reconcile_bench_test.go @@ -0,0 +1,112 @@ +package cocoonset + +import ( + "context" + "fmt" + "slices" + "sync" + "sync/atomic" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cocoonv1 "github.com/cocoonstack/cocoon-common/apis/v1" +) + +const ( + benchCocoonSets = 24 + benchRegistryDelay = 5 * time.Millisecond +) + +// BenchmarkReconcileThroughput drives independent CocoonSets that each block on +// one fixed-latency registry probe; concurrency 1 is the pre-change behavior. +func BenchmarkReconcileThroughput(b *testing.B) { + for _, concurrency := range []int{1, 4, 8} { + b.Run(fmt.Sprintf("concurrency=%d", concurrency), func(b *testing.B) { + var wall time.Duration + var latencies []time.Duration + for b.Loop() { + b.StopTimer() + r, sets := newBenchFixture(b) + b.StartTimer() + + start := time.Now() + latencies = append(latencies, runReconciles(b, r, sets, concurrency)...) + wall += time.Since(start) + } + slices.Sort(latencies) + b.ReportMetric(float64(wall.Milliseconds())/float64(b.N), "wall-ms/op") + b.ReportMetric(float64(percentile(latencies, 95).Milliseconds()), "p95-ms") + }) + } +} + +// runReconciles drains sets through a worker pool, the shape controller-runtime +// gives MaxConcurrentReconciles. +func runReconciles(b *testing.B, r *Reconciler, sets []*cocoonv1.CocoonSet, concurrency int) []time.Duration { + b.Helper() + queue := make(chan *cocoonv1.CocoonSet, len(sets)) + for _, cs := range sets { + queue <- cs + } + close(queue) + + latencies := make([]time.Duration, len(sets)) + var idx atomic.Int32 + var wg sync.WaitGroup + for range concurrency { + wg.Go(func() { + for cs := range queue { + start := time.Now() + if _, err := r.Reconcile(context.Background(), reqFor(cs)); err != nil { + b.Errorf("Reconcile %s: %v", cs.Name, err) + } + latencies[idx.Add(1)-1] = time.Since(start) + } + }) + } + wg.Wait() + return latencies +} + +// newBenchFixture builds independent CocoonSets each missing its main agent and +// carrying a hibernated CR, so every reconcile pays exactly one registry probe. +func newBenchFixture(b *testing.B) (*Reconciler, []*cocoonv1.CocoonSet) { + b.Helper() + scheme := testScheme(b) + objs := make([]client.Object, 0, benchCocoonSets*2) + sets := make([]*cocoonv1.CocoonSet, 0, benchCocoonSets) + for i := range benchCocoonSets { + cs := &cocoonv1.CocoonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("bench-%d", i), + Namespace: "ns", + UID: types.UID(fmt.Sprintf("uid-%d", i)), + Finalizers: []string{finalizerName}, + }, + Spec: cocoonv1.CocoonSetSpec{ + Agent: cocoonv1.AgentSpec{Image: "ghcr.io/cocoonstack/cocoon/ubuntu:24.04"}, + }, + } + sets = append(sets, cs) + objs = append(objs, cs, hibernatedFor(cs)) + } + cli := ctrlfake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objs...). + WithStatusSubresource(&cocoonv1.CocoonSet{}). + Build() + return &Reconciler{Client: cli, Scheme: scheme, Registry: &fakeRegistry{delay: benchRegistryDelay}}, sets +} + +func percentile(sorted []time.Duration, p int) time.Duration { + if len(sorted) == 0 { + return 0 + } + i := min(len(sorted)*p/100, len(sorted)-1) + return sorted[i] +} diff --git a/cocoonset/reconciler.go b/cocoonset/reconciler.go index 6964c7b..a72e5e6 100644 --- a/cocoonset/reconciler.go +++ b/cocoonset/reconciler.go @@ -13,6 +13,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -36,15 +37,22 @@ type Reconciler struct { Scheme *runtime.Scheme Registry snapshot.Registry Recorder record.EventRecorder + // Concurrency caps in-flight reconciles; at 1 one slow registry probe + // stalls every other CocoonSet. + Concurrency int } // SetupWithManager registers the reconciler. `For` uses GenerationChangedPredicate // to avoid status-update loops; Owns filters pod events to creation, deletion, // and readiness transitions to prevent reconcile storms from VK status churn. func (r *Reconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager) error { + if r.Concurrency < 1 { + return fmt.Errorf("cocoonset concurrency must be at least 1, got %d", r.Concurrency) + } return ctrl.NewControllerManagedBy(mgr). For(&cocoonv1.CocoonSet{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Owns(&corev1.Pod{}, builder.WithPredicates(podRelevantChange{})). + WithOptions(controller.Options{MaxConcurrentReconciles: r.Concurrency}). Complete(r) } @@ -122,8 +130,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } return ctrl.Result{Requeue: true}, nil } + intent := r.newRestoreIntent(ctx, cs.Namespace) if classified.main == nil { - return r.createMainAgent(ctx, &cs) + return r.createMainAgent(ctx, &cs, intent) } // Sub-agents fork from main and need it live before creation. @@ -135,11 +144,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu mainVMName := meta.ParseVMSpec(classified.main).VMName mainNodeName := classified.main.Spec.NodeName - subChanged, subRequeue, err := r.ensureSubAgents(ctx, &cs, classified, mainVMName, mainNodeName) + subChanged, subRequeue, err := r.ensureSubAgents(ctx, &cs, classified, mainVMName, mainNodeName, intent) if err != nil { return ctrl.Result{}, err } - tbChanged, err := r.ensureToolboxes(ctx, &cs, classified) + tbChanged, err := r.ensureToolboxes(ctx, &cs, classified, intent) if err != nil { return ctrl.Result{}, err } @@ -157,18 +166,18 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // restore-from-hibernate when the agent is hibernated so a cross-node recreate // restores from the :hibernate snapshot instead of booting fresh. It always // requeues so sub-agents fork off the now-created main. -func (r *Reconciler) createMainAgent(ctx context.Context, cs *cocoonv1.CocoonSet) (ctrl.Result, error) { +func (r *Reconciler) createMainAgent(ctx context.Context, cs *cocoonv1.CocoonSet, intent restoreIntent) (ctrl.Result, error) { logger := log.WithFunc("cocoonset.Reconciler.createMainAgent") mainPod, err := buildAgentPod(cs, 0, "", "", r.Scheme) if err != nil { return ctrl.Result{}, fmt.Errorf("build main agent: %w", err) } - restorable, err := r.podsRestorableByCR(ctx, cs.Namespace) + restorable, err := intent() if err != nil { return ctrl.Result{}, err } - _, intent := restorable[mainPod.Name] - if err := r.markRestoreIfHibernated(ctx, mainPod, intent); err != nil { + _, wantRestore := restorable[mainPod.Name] + if err := r.markRestoreIfHibernated(ctx, mainPod, wantRestore); err != nil { return ctrl.Result{}, err } if err := r.Create(ctx, mainPod); err != nil { diff --git a/cocoonset/reconciler_test.go b/cocoonset/reconciler_test.go index 4aa40e3..269999e 100644 --- a/cocoonset/reconciler_test.go +++ b/cocoonset/reconciler_test.go @@ -6,6 +6,7 @@ import ( "slices" "sync" "testing" + "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -120,7 +121,7 @@ func TestEnsureToolboxesCollisionReturnsError(t *testing.T) { allByName: map[string]*corev1.Pod{agentPod.Name: agentPod}, } - _, err := r.ensureToolboxes(t.Context(), cs, classified) + _, err := r.ensureToolboxes(t.Context(), cs, classified, r.newRestoreIntent(t.Context(), cs.Namespace)) if err == nil { t.Fatal("ensureToolboxes should return error on name collision with agent pod") } @@ -143,7 +144,7 @@ func TestEnsureToolboxesRejectsDuplicateNames(t *testing.T) { allByName: map[string]*corev1.Pod{}, } - _, err := r.ensureToolboxes(t.Context(), cs, classified) + _, err := r.ensureToolboxes(t.Context(), cs, classified, r.newRestoreIntent(t.Context(), cs.Namespace)) if err == nil { t.Fatal("ensureToolboxes must reject a spec with duplicate toolbox names") } @@ -169,7 +170,7 @@ func TestEnsureToolboxesIdempotentOnExistingToolbox(t *testing.T) { allByName: map[string]*corev1.Pod{}, } - changed, err := r.ensureToolboxes(t.Context(), cs, classified) + changed, err := r.ensureToolboxes(t.Context(), cs, classified, r.newRestoreIntent(t.Context(), cs.Namespace)) if err != nil { t.Fatalf("ensureToolboxes: %v", err) } @@ -280,7 +281,7 @@ func TestEnsureSubAgentsReplacesTerminalPod(t *testing.T) { allByName: map[string]*corev1.Pod{subPod.Name: subPod}, } - changed, _, err := r.ensureSubAgents(t.Context(), cs, classified, "vk-ns-demo-0", "") + changed, _, err := r.ensureSubAgents(t.Context(), cs, classified, "vk-ns-demo-0", "", r.newRestoreIntent(t.Context(), cs.Namespace)) if err != nil { t.Fatalf("ensureSubAgents: %v", err) } @@ -378,7 +379,7 @@ func TestEnsureSubAgentsTreatsLifecycleFailedAsTerminal(t *testing.T) { allByName: map[string]*corev1.Pod{subPod.Name: subPod}, } - changed, _, err := r.ensureSubAgents(t.Context(), cs, classified, "vk-ns-demo-0", "") + changed, _, err := r.ensureSubAgents(t.Context(), cs, classified, "vk-ns-demo-0", "", r.newRestoreIntent(t.Context(), cs.Namespace)) if err != nil { t.Fatalf("ensureSubAgents: %v", err) } @@ -412,7 +413,7 @@ func TestEnsureToolboxesReplacesTerminalPod(t *testing.T) { allByName: map[string]*corev1.Pod{tbPod.Name: tbPod}, } - changed, err := r.ensureToolboxes(t.Context(), cs, classified) + changed, err := r.ensureToolboxes(t.Context(), cs, classified, r.newRestoreIntent(t.Context(), cs.Namespace)) if err != nil { t.Fatalf("ensureToolboxes: %v", err) } @@ -653,9 +654,23 @@ func TestApplyUnsuspendSkipsPodHibernatedByCR(t *testing.T) { } } +// A nil manager suffices: the guard rejects before mgr is touched. +func TestSetupWithManagerRejectsInvalidConcurrency(t *testing.T) { + for _, n := range []int{0, -1} { + if err := (&Reconciler{Concurrency: n}).SetupWithManager(t.Context(), nil); err == nil { + t.Errorf("concurrency %d must be rejected", n) + } + } +} + type fakeRegistry struct { - present map[string]bool - probeErr error + present map[string]bool + probeErr error + // delay models a remote round trip; block wedges the named VM's probe and + // entered reports that the probe is actually wedged. + delay time.Duration + block map[string]chan struct{} + entered chan string deletedMu sync.Mutex deleted []string } @@ -664,6 +679,13 @@ func (f *fakeRegistry) HasManifest(_ context.Context, name, tag string) (bool, e if f.probeErr != nil { return false, f.probeErr } + if ch, ok := f.block[name]; ok { + if f.entered != nil { + f.entered <- name + } + <-ch + } + time.Sleep(f.delay) return f.present[name+":"+tag], nil } diff --git a/cocoonset/restore.go b/cocoonset/restore.go index 7e870f6..01d25ea 100644 --- a/cocoonset/restore.go +++ b/cocoonset/restore.go @@ -3,6 +3,7 @@ package cocoonset import ( "context" "fmt" + "sync" "github.com/projecteru2/core/log" corev1 "k8s.io/api/core/v1" @@ -12,6 +13,17 @@ import ( "github.com/cocoonstack/cocoon-common/meta" ) +// restoreIntent returns the namespace's restore-intent set, loaded at most once. +type restoreIntent func() (map[string]struct{}, error) + +// newRestoreIntent defers the List until a pod actually has to be built: it is +// O(CocoonHibernations in the namespace) and the steady path must not pay it. +func (r *Reconciler) newRestoreIntent(ctx context.Context, namespace string) restoreIntent { + return sync.OnceValues(func() (map[string]struct{}, error) { + return r.podsRestorableByCR(ctx, namespace) + }) +} + // hibernationPodNames lists the namespace's CocoonHibernations and returns the // set of PodRef names whose CR satisfies accept. func (r *Reconciler) hibernationPodNames(ctx context.Context, namespace string, accept func(*cocoonv1.CocoonHibernation) bool) (map[string]struct{}, error) { diff --git a/cocoonset/restore_intent_test.go b/cocoonset/restore_intent_test.go new file mode 100644 index 0000000..081bf68 --- /dev/null +++ b/cocoonset/restore_intent_test.go @@ -0,0 +1,161 @@ +package cocoonset + +import ( + "context" + "sync/atomic" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + cocoonv1 "github.com/cocoonstack/cocoon-common/apis/v1" + "github.com/cocoonstack/cocoon-common/meta" +) + +// TestReconcileSteadyStateSkipsHibernationList pins the lazy load: with every +// desired pod present, the reconcile must not pay the CocoonHibernation List. +func TestReconcileSteadyStateSkipsHibernationList(t *testing.T) { + scheme := testScheme(t) + cs := newCocoonSet("demo", func(cs *cocoonv1.CocoonSet) { + cs.Finalizers = []string{finalizerName} + cs.Spec.Agent.Replicas = 1 + cs.Spec.Toolboxes = []cocoonv1.ToolboxSpec{ + {Name: "tb", Image: "img", Mode: cocoonv1.ToolboxModeRun}, + } + }) + mainPod := readyPod(mustBuildAgentPod(t, cs, 0, "", "", scheme)) + subPod := readyPod(mustBuildAgentPod(t, cs, 1, "vk-ns-demo-0", "", scheme)) + tbPod := readyPod(mustBuildToolboxPod(t, cs, cs.Spec.Toolboxes[0], scheme)) + + var lists atomic.Int32 + cli := ctrlfake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(cs, mainPod, subPod, tbPod). + WithStatusSubresource(&cocoonv1.CocoonSet{}). + WithInterceptorFuncs(countHibernationLists(&lists)). + Build() + r := &Reconciler{Client: cli, Scheme: scheme, Registry: &fakeRegistry{}} + + if _, err := r.Reconcile(t.Context(), reqFor(cs)); err != nil { + t.Fatalf("Reconcile: %v", err) + } + // Guards the assertion below against a vacuous pass: 0 Lists would also + // follow from an early return or from triage deleting a drifted pod. + for _, p := range []*corev1.Pod{mainPod, subPod, tbPod} { + if err := cli.Get(t.Context(), client.ObjectKeyFromObject(p), &corev1.Pod{}); err != nil { + t.Fatalf("pod %s must survive a steady reconcile: %v", p.Name, err) + } + } + if got := lists.Load(); got != 0 { + t.Errorf("steady reconcile listed CocoonHibernations %d times, want 0", got) + } +} + +// TestReconcileMissingPodsListsHibernationsOnce pins the sharing: a reconcile +// missing both a sub-agent and a toolbox loads restore intent exactly once. +func TestReconcileMissingPodsListsHibernationsOnce(t *testing.T) { + scheme := testScheme(t) + cs := newCocoonSet("demo", func(cs *cocoonv1.CocoonSet) { + cs.Finalizers = []string{finalizerName} + cs.Spec.Agent.Replicas = 2 + cs.Spec.Toolboxes = []cocoonv1.ToolboxSpec{ + {Name: "tb", Image: "img", Mode: cocoonv1.ToolboxModeRun}, + } + }) + mainPod := readyPod(mustBuildAgentPod(t, cs, 0, "", "", scheme)) + + var lists atomic.Int32 + cli := ctrlfake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(cs, mainPod). + WithStatusSubresource(&cocoonv1.CocoonSet{}). + WithInterceptorFuncs(countHibernationLists(&lists)). + Build() + r := &Reconciler{Client: cli, Scheme: scheme, Registry: &fakeRegistry{}} + + if _, err := r.Reconcile(t.Context(), reqFor(cs)); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if got := lists.Load(); got != 1 { + t.Errorf("reconcile listed CocoonHibernations %d times, want exactly 1", got) + } + for _, name := range []string{"demo-1", "demo-2", toolboxPodName(cs.Name, "tb")} { + if err := cli.Get(t.Context(), types.NamespacedName{Namespace: "ns", Name: name}, &corev1.Pod{}); err != nil { + t.Errorf("pod %s should have been created: %v", name, err) + } + } +} + +// TestReconcileUnrelatedKeyProgressesWhileProbeBlocks pins what concurrency +// buys: with one CocoonSet's probe wedged, a second must still finish. +func TestReconcileUnrelatedKeyProgressesWhileProbeBlocks(t *testing.T) { + scheme := testScheme(t) + blocked, wedged := newCocoonSet("blocked", withMainRestore), newCocoonSet("free", withMainRestore) + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + + cli := ctrlfake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(blocked, wedged, hibernatedFor(blocked), hibernatedFor(wedged)). + WithStatusSubresource(&cocoonv1.CocoonSet{}). + Build() + entered := make(chan string, 1) + r := &Reconciler{Client: cli, Scheme: scheme, Registry: &fakeRegistry{ + block: map[string]chan struct{}{meta.VMNameForPod("ns", "blocked-0"): release}, + entered: entered, + }} + + go func() { _, _ = r.Reconcile(context.Background(), reqFor(blocked)) }() + // Wait until the probe is genuinely wedged; otherwise the second reconcile + // could finish first and pass even under full serialization. + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("blocked reconcile never reached the registry probe") + } + + done := make(chan error, 1) + go func() { _, err := r.Reconcile(context.Background(), reqFor(wedged)); done <- err }() + select { + case err := <-done: + if err != nil { + t.Fatalf("unrelated reconcile: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("unrelated CocoonSet was blocked by another key's registry probe") + } +} + +func withMainRestore(cs *cocoonv1.CocoonSet) { cs.Finalizers = []string{finalizerName} } + +func hibernatedFor(cs *cocoonv1.CocoonSet) *cocoonv1.CocoonHibernation { + return &cocoonv1.CocoonHibernation{ + ObjectMeta: metav1.ObjectMeta{Name: "h-" + cs.Name, Namespace: cs.Namespace}, + Spec: cocoonv1.CocoonHibernationSpec{ + PodRef: cocoonv1.HibernationPodRef{Name: cs.Name + "-0"}, + Desire: cocoonv1.HibernationDesireHibernate, + }, + Status: cocoonv1.CocoonHibernationStatus{Phase: cocoonv1.CocoonHibernationPhaseHibernated}, + } +} + +func countHibernationLists(counter *atomic.Int32) interceptor.Funcs { + return interceptor.Funcs{ + List: func(ctx context.Context, cli client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*cocoonv1.CocoonHibernationList); ok { + counter.Add(1) + } + return cli.List(ctx, list, opts...) + }, + } +} + +func reqFor(cs *cocoonv1.CocoonSet) ctrl.Request { + return ctrl.Request{NamespacedName: types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}} +} diff --git a/cocoonset/restore_test.go b/cocoonset/restore_test.go index 17816a7..d246a70 100644 --- a/cocoonset/restore_test.go +++ b/cocoonset/restore_test.go @@ -122,7 +122,7 @@ func TestEnsureToolboxesRestoresHibernated(t *testing.T) { Registry: &fakeRegistry{present: map[string]bool{tbVMName + ":hibernate": true}}, } - changed, err := r.ensureToolboxes(t.Context(), cs, classifyPods(nil)) + changed, err := r.ensureToolboxes(t.Context(), cs, classifyPods(nil), r.newRestoreIntent(t.Context(), cs.Namespace)) if err != nil { t.Fatalf("ensureToolboxes: %v", err) } diff --git a/cocoonset/toolboxes.go b/cocoonset/toolboxes.go index 52152f1..c0ddf3c 100644 --- a/cocoonset/toolboxes.go +++ b/cocoonset/toolboxes.go @@ -16,7 +16,7 @@ import ( "github.com/cocoonstack/cocoon-common/meta" ) -func (r *Reconciler) ensureToolboxes(ctx context.Context, cs *cocoonv1.CocoonSet, classified classifiedPods) (bool, error) { +func (r *Reconciler) ensureToolboxes(ctx context.Context, cs *cocoonv1.CocoonSet, classified classifiedPods, intent restoreIntent) (bool, error) { logger := log.WithFunc("cocoonset.Reconciler.ensureToolboxes") // Defense in depth: webhook should already reject duplicates. Validate // upfront before any Create/Delete so a bypass can't leave partial state. @@ -27,10 +27,6 @@ func (r *Reconciler) ensureToolboxes(ctx context.Context, cs *cocoonv1.CocoonSet } desired[tb.Name] = true } - restorable, err := r.podsRestorableByCR(ctx, cs.Namespace) - if err != nil { - return false, err - } changed := false for _, tb := range cs.Spec.Toolboxes { podName := toolboxPodName(cs.Name, tb.Name) @@ -51,8 +47,12 @@ func (r *Reconciler) ensureToolboxes(ctx context.Context, cs *cocoonv1.CocoonSet if err != nil { return changed, fmt.Errorf("build toolbox %s: %w", tb.Name, err) } - _, intent := restorable[tbPod.Name] - if err := r.markRestoreIfHibernated(ctx, tbPod, intent); err != nil { + restorable, err := intent() + if err != nil { + return changed, err + } + _, wantRestore := restorable[tbPod.Name] + if err := r.markRestoreIfHibernated(ctx, tbPod, wantRestore); err != nil { return changed, fmt.Errorf("mark restore toolbox %s: %w", tb.Name, err) } if err := r.Create(ctx, tbPod); err != nil { diff --git a/docs/configuration.md b/docs/configuration.md index a75bf6e..8410146 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -8,5 +8,9 @@ | `METRICS_ADDR` | `:8080` | Prometheus listener | | `PROBE_ADDR` | `:8081` | healthz / readyz listener | | `LEADER_ELECT` | `true` | Enable leader election so only one replica reconciles | +| `COCOONSET_CONCURRENCY` | `4` | Maximum concurrent CocoonSet reconciles. Must be at least 1. | +| `HIBERNATION_CONCURRENCY` | `4` | Maximum concurrent CocoonHibernation reconciles. Must be at least 1. | -CLI flags (`--metrics-bind-address`, `--health-probe-bind-address`, `--leader-elect`) override the corresponding env var. +CLI flags (`--metrics-bind-address`, `--health-probe-bind-address`, `--leader-elect`, `--cocoonset-concurrency`, `--hibernation-concurrency`) override the corresponding env var. + +Reconciles block on registry round trips, so concurrency above 1 overlaps those waits across unrelated resources. Reconciles of one resource are never concurrent, and CocoonHibernation CRs that target the same pod are serialized against each other. diff --git a/hibernation/reconciler.go b/hibernation/reconciler.go index 54d2c3f..6558cd3 100644 --- a/hibernation/reconciler.go +++ b/hibernation/reconciler.go @@ -4,6 +4,7 @@ package hibernation import ( "context" "fmt" + "hash/fnv" "sync" "time" @@ -18,6 +19,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -44,6 +46,10 @@ const ( // finalizerName keeps the CR alive long enough to clear its :hibernate tag from the registry. finalizerName = "cocoonhibernation.cocoonset.cocoonstack.io/finalizer" + // vmLockStripes bounds the per-VM lock table; 64 keeps collisions rare + // against realistic per-namespace VM counts. + vmLockStripes = 64 + conditionReasonPending = "Pending" conditionReasonDone = "Done" conditionReasonFailed = "Failed" @@ -56,16 +62,27 @@ type Reconciler struct { Scheme *runtime.Scheme Registry snapshot.Registry Recorder record.EventRecorder + // Concurrency caps in-flight reconciles; at 1 one slow registry request + // stalls every other CR. + Concurrency int // observed[UID] = last recorded Ready.LastTransitionTime, dedups // phase-exit observations against controller-runtime cache lag. observed sync.Map + + // vmLocks serializes the distinct CRs that may reach one VM: above one worker + // their opposing desires would race that VM's :hibernate tag and its pod's + // hibernate annotation. + vmLocks [vmLockStripes]sync.Mutex } // SetupWithManager registers the reconciler with the controller manager. // An index on spec.podRef.name lets the pod watcher fan out events to every // CR targeting a given pod, so late-arriving pods self-heal without user edits. func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + if r.Concurrency < 1 { + return fmt.Errorf("hibernation concurrency must be at least 1, got %d", r.Concurrency) + } if err := mgr.GetFieldIndexer().IndexField( ctx, &cocoonv1.CocoonHibernation{}, indexPodRefName, func(o client.Object) []string { @@ -89,6 +106,7 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) err }, )), ). + WithOptions(controller.Options{MaxConcurrentReconciles: r.Concurrency}). Complete(r) } @@ -105,7 +123,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } if !hib.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, &hib) + return ctrl.Result{}, r.reconcileDelete(ctx, &hib) } if !controllerutil.ContainsFinalizer(&hib, finalizerName) { controllerutil.AddFinalizer(&hib, finalizerName) @@ -118,6 +136,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if hib.Spec.PodRef.Name == "" { return ctrl.Result{}, r.markFailed(ctx, &hib, "spec.podRef.name is required") } + var pod corev1.Pod err := r.Get(ctx, types.NamespacedName{Namespace: hib.Namespace, Name: hib.Spec.PodRef.Name}, &pod) if err != nil { @@ -134,6 +153,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // VMName is filled by vk-cocoon once the VM is provisioned; wait. return ctrl.Result{RequeueAfter: requeueInterval}, r.markPending(ctx, &hib, fmt.Sprintf("pod %s/%s has no %s annotation yet", pod.Namespace, pod.Name, meta.AnnotationVMName)) } + defer r.lockVM(vmName)() logger.Debugf(ctx, "reconcile hibernation %s/%s desire=%s vm=%s", hib.Namespace, hib.Name, hib.Spec.Desire, vmName) @@ -147,9 +167,26 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } } +// lockVM locks the VM whose :hibernate tag and pod annotation are about to be +// touched, and returns its release. Striping bounds the table because VM names +// come and go with CocoonSets; a collision only costs two unrelated VMs an +// occasional serialization. +func (r *Reconciler) lockVM(vmName string) func() { + h := fnv.New32a() + _, _ = h.Write([]byte(vmName)) + mu := &r.vmLocks[h.Sum32()%vmLockStripes] + mu.Lock() + return mu.Unlock +} + // reconcileDelete clears the :hibernate tag (if Status.VMName is set) and removes the finalizer. -func (r *Reconciler) reconcileDelete(ctx context.Context, hib *cocoonv1.CocoonHibernation) (ctrl.Result, error) { +func (r *Reconciler) reconcileDelete(ctx context.Context, hib *cocoonv1.CocoonHibernation) error { logger := log.WithFunc("hibernation.Reconciler.reconcileDelete") + // spec.podRef is mutable, so a retargeted CR still owns the tag its status + // names: lock that VM, not whichever pod spec points at now. + if hib.Status.VMName != "" { + defer r.lockVM(hib.Status.VMName)() + } if r.Registry != nil && hib.Status.VMName != "" { if err := r.Registry.DeleteManifest(ctx, hib.Status.VMName, meta.HibernateSnapshotTag); err != nil { logger.Errorf(ctx, err, "delete hibernate snapshot %s", hib.Status.VMName) @@ -159,10 +196,10 @@ func (r *Reconciler) reconcileDelete(ctx context.Context, hib *cocoonv1.CocoonHi if controllerutil.ContainsFinalizer(hib, finalizerName) { controllerutil.RemoveFinalizer(hib, finalizerName) if err := r.Update(ctx, hib); err != nil { - return ctrl.Result{}, fmt.Errorf("remove finalizer: %w", err) + return fmt.Errorf("remove finalizer: %w", err) } } - return ctrl.Result{}, nil + return nil } // hibernationsTargetingPod returns reconcile requests for every CocoonHibernation diff --git a/hibernation/reconciler_test.go b/hibernation/reconciler_test.go index 898dbc8..c36d893 100644 --- a/hibernation/reconciler_test.go +++ b/hibernation/reconciler_test.go @@ -3,6 +3,8 @@ package hibernation import ( "context" "errors" + "sync" + "sync/atomic" "testing" "time" @@ -143,7 +145,7 @@ func TestReconcileDeleteClearsHibernateTagAndFinalizer(t *testing.T) { reg := &fakeRegistry{} r := &Reconciler{Client: cli, Scheme: scheme, Registry: reg} - if _, err := r.reconcileDelete(t.Context(), hib); err != nil { + if err := r.reconcileDelete(t.Context(), hib); err != nil { t.Fatalf("reconcileDelete: %v", err) } if !reg.deleteCalled { @@ -179,7 +181,7 @@ func TestReconcileDeleteSkipsTagWhenVMNameMissing(t *testing.T) { reg := &fakeRegistry{} r := &Reconciler{Client: cli, Scheme: scheme, Registry: reg} - if _, err := r.reconcileDelete(t.Context(), hib); err != nil { + if err := r.reconcileDelete(t.Context(), hib); err != nil { t.Fatalf("reconcileDelete: %v", err) } if reg.deleteCalled { @@ -761,6 +763,200 @@ func TestReconcilePendingWhenPodMissingVMName(t *testing.T) { } } +// TestReconcileSerializesCRsTargetingOnePod pins the pod lock: nothing stops two +// CRs from naming one pod with opposing desires, and above one worker they would +// otherwise interleave the shared hibernate annotation and :hibernate tag. +func TestReconcileSerializesCRsTargetingOnePod(t *testing.T) { + hib := func(name string, desire cocoonv1.HibernationDesire) *cocoonv1.CocoonHibernation { + return &cocoonv1.CocoonHibernation{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ns", Finalizers: []string{finalizerName}}, + Spec: cocoonv1.CocoonHibernationSpec{ + Desire: desire, + PodRef: cocoonv1.HibernationPodRef{Name: "demo-0"}, + }, + } + } + // Running + a VMID is what drives the wake path all the way to DeleteManifest, + // so both desires actually reach the registry and can be observed racing. + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo-0", Namespace: "ns", + Annotations: map[string]string{ + meta.AnnotationVMName: "vk-ns-demo-0", + meta.AnnotationVMID: "vmid-1", + }, + }, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{ + {State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}, + }}, + } + scheme := testScheme(t) + cli := ctrlfake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(hib("a", cocoonv1.HibernationDesireHibernate), hib("b", cocoonv1.HibernationDesireWake), pod). + WithStatusSubresource(&cocoonv1.CocoonHibernation{}). + Build() + reg := &concurrencyProbe{} + r := &Reconciler{Client: cli, Scheme: scheme, Registry: reg} + + var wg sync.WaitGroup + for _, name := range []string{"a", "b"} { + wg.Go(func() { + _, _ = r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "ns", Name: name}, + }) + }) + } + wg.Wait() + if reg.maxInFlight.Load() > 1 { + t.Errorf("CRs targeting one pod ran %d registry calls in flight, want serialized", reg.maxInFlight.Load()) + } +} + +// TestReconcileSerializesDeletingCRAgainstLiveCR covers the delete fast path: +// reconcileDelete drops the same :hibernate tag a live CR on that pod probes, so +// it must take the pod lock too rather than return ahead of it. +func TestReconcileSerializesDeletingCRAgainstLiveCR(t *testing.T) { + deleting := &cocoonv1.CocoonHibernation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "a", Namespace: "ns", + Finalizers: []string{finalizerName}, + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + }, + Spec: cocoonv1.CocoonHibernationSpec{ + Desire: cocoonv1.HibernationDesireHibernate, + PodRef: cocoonv1.HibernationPodRef{Name: "demo-0"}, + }, + Status: cocoonv1.CocoonHibernationStatus{VMName: "vk-ns-demo-0"}, + } + live := &cocoonv1.CocoonHibernation{ + ObjectMeta: metav1.ObjectMeta{Name: "b", Namespace: "ns", Finalizers: []string{finalizerName}}, + Spec: cocoonv1.CocoonHibernationSpec{ + Desire: cocoonv1.HibernationDesireHibernate, + PodRef: cocoonv1.HibernationPodRef{Name: "demo-0"}, + }, + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "demo-0", Namespace: "ns", + Annotations: map[string]string{meta.AnnotationVMName: "vk-ns-demo-0"}, + }} + scheme := testScheme(t) + cli := ctrlfake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(deleting, live, pod). + WithStatusSubresource(&cocoonv1.CocoonHibernation{}). + Build() + reg := &concurrencyProbe{} + r := &Reconciler{Client: cli, Scheme: scheme, Registry: reg} + + var wg sync.WaitGroup + for _, name := range []string{"a", "b"} { + wg.Go(func() { + _, _ = r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "ns", Name: name}, + }) + }) + } + wg.Wait() + if reg.maxInFlight.Load() > 1 { + t.Errorf("deleting CR ran %d registry calls in flight with a live CR on the same pod, want serialized", reg.maxInFlight.Load()) + } +} + +// TestReconcileSerializesRetargetedCRAgainstItsStaleVM covers the podRef/status +// split: spec.podRef is mutable and markPending never rewrites Status.VMName, so +// a CR retargeted to demo-1 still deletes demo-0's tag on the way out. Keying the +// lock on spec.podRef would lock demo-1 while touching demo-0's tag, leaving a +// live CR on demo-0 free to probe the tag being deleted. +func TestReconcileSerializesRetargetedCRAgainstItsStaleVM(t *testing.T) { + retargeted := &cocoonv1.CocoonHibernation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "a", Namespace: "ns", + Finalizers: []string{finalizerName}, + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + }, + Spec: cocoonv1.CocoonHibernationSpec{ + Desire: cocoonv1.HibernationDesireHibernate, + PodRef: cocoonv1.HibernationPodRef{Name: "demo-1"}, + }, + // Status still names demo-0's VM: the tag this CR will delete. + Status: cocoonv1.CocoonHibernationStatus{VMName: "vk-ns-demo-0"}, + } + live := &cocoonv1.CocoonHibernation{ + ObjectMeta: metav1.ObjectMeta{Name: "b", Namespace: "ns", Finalizers: []string{finalizerName}}, + Spec: cocoonv1.CocoonHibernationSpec{ + Desire: cocoonv1.HibernationDesireHibernate, + PodRef: cocoonv1.HibernationPodRef{Name: "demo-0"}, + }, + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "demo-0", Namespace: "ns", + Annotations: map[string]string{meta.AnnotationVMName: "vk-ns-demo-0"}, + }} + scheme := testScheme(t) + cli := ctrlfake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(retargeted, live, pod). + WithStatusSubresource(&cocoonv1.CocoonHibernation{}). + Build() + reg := &concurrencyProbe{} + r := &Reconciler{Client: cli, Scheme: scheme, Registry: reg} + + var wg sync.WaitGroup + for _, name := range []string{"a", "b"} { + wg.Go(func() { + _, _ = r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "ns", Name: name}, + }) + }) + } + wg.Wait() + if reg.maxInFlight.Load() > 1 { + t.Errorf("retargeted CR ran %d registry calls in flight against its stale VM, want serialized", reg.maxInFlight.Load()) + } +} + +// concurrencyProbe records the peak number of registry calls in flight. +type concurrencyProbe struct { + inFlight atomic.Int32 + maxInFlight atomic.Int32 +} + +func (c *concurrencyProbe) HasManifest(context.Context, string, string) (bool, error) { + c.enter() + defer c.inFlight.Add(-1) + time.Sleep(20 * time.Millisecond) + return false, nil +} + +func (c *concurrencyProbe) DeleteManifest(context.Context, string, string) error { + c.enter() + defer c.inFlight.Add(-1) + time.Sleep(20 * time.Millisecond) + return nil +} + +// enter raises the peak by CAS: a plain load-then-store lets a later, smaller +// caller overwrite a peak of 2 with 1 and pass the test falsely. +func (c *concurrencyProbe) enter() { + n := c.inFlight.Add(1) + for { + peak := c.maxInFlight.Load() + if n <= peak || c.maxInFlight.CompareAndSwap(peak, n) { + return + } + } +} + +// A nil manager suffices: the guard rejects before mgr is touched. +func TestSetupWithManagerRejectsInvalidConcurrency(t *testing.T) { + for _, n := range []int{0, -1} { + if err := (&Reconciler{Concurrency: n}).SetupWithManager(t.Context(), nil); err == nil { + t.Errorf("concurrency %d must be rejected", n) + } + } +} + func testScheme(t *testing.T) *runtime.Scheme { t.Helper() sch := runtime.NewScheme() diff --git a/main.go b/main.go index 288fe26..5392374 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "os/signal" + "strconv" "syscall" "github.com/google/go-containerregistry/pkg/authn" @@ -42,19 +43,26 @@ const ( defaultMetricsAddr = ":8080" defaultProbeAddr = ":8081" leaderElectionID = "cocoon-operator.cocoonset.cocoonstack.io" + + // defaultConcurrency overlaps registry waits without straining client-go QPS. + defaultConcurrency = 4 ) func main() { leaderDefault := commonk8s.EnvBool("LEADER_ELECT", true) var ( - metricsAddr string - probeAddr string - enableLeaderElection bool + metricsAddr string + probeAddr string + enableLeaderElection bool + cocoonSetConcurrency int + hibernationConcurrency int ) flag.StringVar(&metricsAddr, "metrics-bind-address", commonk8s.EnvOrDefault("METRICS_ADDR", defaultMetricsAddr), "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", commonk8s.EnvOrDefault("PROBE_ADDR", defaultProbeAddr), "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", leaderDefault, "Enable leader election so only one operator instance reconciles at a time.") + flag.IntVar(&cocoonSetConcurrency, "cocoonset-concurrency", envInt("COCOONSET_CONCURRENCY", defaultConcurrency), "Maximum concurrent CocoonSet reconciles.") + flag.IntVar(&hibernationConcurrency, "hibernation-concurrency", envInt("HIBERNATION_CONCURRENCY", defaultConcurrency), "Maximum concurrent CocoonHibernation reconciles.") flag.Parse() ctx := context.Background() @@ -109,18 +117,20 @@ func main() { recorder := broadcaster.NewRecorder(mgr.GetScheme(), corev1.EventSource{Component: "cocoon-operator"}) if err = (&cocoonset.Reconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Registry: registry, - Recorder: recorder, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Registry: registry, + Recorder: recorder, + Concurrency: cocoonSetConcurrency, }).SetupWithManager(ctx, mgr); err != nil { logger.Fatalf(ctx, err, "register cocoonset.Reconciler") } if err = (&hibernation.Reconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Registry: registry, - Recorder: recorder, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Registry: registry, + Recorder: recorder, + Concurrency: hibernationConcurrency, }).SetupWithManager(ctx, mgr); err != nil { logger.Fatalf(ctx, err, "register hibernation.Reconciler") } @@ -128,8 +138,8 @@ func main() { signalCtx, cancel := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) defer cancel() - logger.Infof(signalCtx, "starting controller manager (metrics=%s probe=%s leader=%t)", - metricsAddr, probeAddr, enableLeaderElection) + logger.Infof(signalCtx, "starting controller manager (metrics=%s probe=%s leader=%t cocoonset-concurrency=%d hibernation-concurrency=%d)", + metricsAddr, probeAddr, enableLeaderElection, cocoonSetConcurrency, hibernationConcurrency) if err = mgr.Start(signalCtx); err != nil { logger.Fatalf(signalCtx, err, "run manager") } @@ -152,3 +162,12 @@ func buildScheme() *runtime.Scheme { utilruntime.Must(cocoonv1.AddToScheme(scheme)) return scheme } + +// envInt parses an int env var, falling back when unset or invalid. +func envInt(key string, fallback int) int { + n, err := strconv.Atoi(os.Getenv(key)) + if err != nil { + return fallback + } + return n +} diff --git a/main_test.go b/main_test.go index 6c82f8a..80916d9 100644 --- a/main_test.go +++ b/main_test.go @@ -21,3 +21,23 @@ func TestBuildRegistry(t *testing.T) { t.Fatal("buildRegistry with no OCI_REGISTRY: want error, got nil") } } + +func TestEnvInt(t *testing.T) { + cases := []struct { + name string + set string + want int + }{ + {"unset falls back", "", 4}, + {"valid value wins", "8", 8}, + {"invalid falls back", "many", 4}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + t.Setenv("TEST_CONCURRENCY", c.set) + if got := envInt("TEST_CONCURRENCY", 4); got != c.want { + t.Errorf("envInt = %d, want %d", got, c.want) + } + }) + } +}