From c8259f38d9297cf1e07959f2b70db0abcba3a9f4 Mon Sep 17 00:00:00 2001 From: payall4u Date: Mon, 27 Nov 2023 19:34:36 +0800 Subject: [PATCH 1/3] Optimize qos initializer. --- pkg/ensurance/analyzer/analyzer.go | 4 +- .../podqos-fetcher.go => util/match_qos.go} | 39 ++++++++- pkg/providers/grpc/pb/provider.pb.go | 5 +- pkg/providers/grpc/pb/provider_grpc.pb.go | 1 + pkg/webhooks/pod/mutating.go | 67 ++++++++++++-- pkg/webhooks/pod/mutating_test.go | 87 ++++++++++++++++--- pkg/webhooks/webhook.go | 26 ++++-- tools/initializer/resource.yaml | 24 +++-- 8 files changed, 211 insertions(+), 42 deletions(-) rename pkg/ensurance/{analyzer/podqos-fetcher.go => util/match_qos.go} (85%) diff --git a/pkg/ensurance/analyzer/analyzer.go b/pkg/ensurance/analyzer/analyzer.go index 64fed0781..561e18a46 100644 --- a/pkg/ensurance/analyzer/analyzer.go +++ b/pkg/ensurance/analyzer/analyzer.go @@ -6,6 +6,8 @@ import ( "strings" "time" + "github.com/gocrane/crane/pkg/ensurance/util" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -490,7 +492,7 @@ func (s *AnomalyAnalyzer) filterPodQOSMatches(pods []*v1.Pod, actionName string) } for _, qos := range podQOSList { for _, pod := range pods { - if !match(pod, qos) { + if !util.MatchPodAndPodQOS(pod, qos) { klog.V(4).Infof("Pod %s/%s does not match PodQOS %s", pod.Namespace, pod.Name, qos.Name) continue diff --git a/pkg/ensurance/analyzer/podqos-fetcher.go b/pkg/ensurance/util/match_qos.go similarity index 85% rename from pkg/ensurance/analyzer/podqos-fetcher.go rename to pkg/ensurance/util/match_qos.go index 3d95f0d82..6a2ef83bf 100644 --- a/pkg/ensurance/analyzer/podqos-fetcher.go +++ b/pkg/ensurance/util/match_qos.go @@ -1,8 +1,9 @@ -package analyzer +package util import ( "fmt" "reflect" + "sort" "strconv" "strings" @@ -69,7 +70,41 @@ func labelMatch(labelSelector metav1.LabelSelector, matchLabels map[string]strin return true } -func match(pod *v1.Pod, podQOS *ensuranceapi.PodQOS) bool { +func sortQOSSlice(qosSlice []*ensuranceapi.PodQOS) { + sort.Slice(qosSlice, func(i, j int) bool { + if len(qosSlice[i].Spec.LabelSelector.MatchLabels) != len(qosSlice[j].Spec.LabelSelector.MatchLabels) { + return len(qosSlice[i].Spec.LabelSelector.MatchLabels) > len(qosSlice[j].Spec.LabelSelector.MatchLabels) + } + + if qosSlice[i].Spec.ScopeSelector == nil && qosSlice[j].Spec.ScopeSelector == nil { + return true + } + + if qosSlice[i].Spec.ScopeSelector == nil { + return false + } + + if qosSlice[j].Spec.ScopeSelector == nil { + return true + } + + return len(qosSlice[i].Spec.ScopeSelector.MatchExpressions) > len(qosSlice[j].Spec.ScopeSelector.MatchExpressions) + }) +} + +func MatchPodAndPodQOSSlice(pod *v1.Pod, qosSlice []*ensuranceapi.PodQOS) (res *ensuranceapi.PodQOS) { + newSlice := make([]*ensuranceapi.PodQOS, len(qosSlice), len(qosSlice)) + copy(newSlice, qosSlice) + sortQOSSlice(newSlice) + for _, qos := range newSlice { + if MatchPodAndPodQOS(pod, qos) { + return qos + } + } + return nil +} + +func MatchPodAndPodQOS(pod *v1.Pod, podQOS *ensuranceapi.PodQOS) bool { if podQOS.Spec.ScopeSelector == nil && podQOS.Spec.LabelSelector.MatchLabels == nil && diff --git a/pkg/providers/grpc/pb/provider.pb.go b/pkg/providers/grpc/pb/provider.pb.go index 3b39c4ac1..684a39d71 100644 --- a/pkg/providers/grpc/pb/provider.pb.go +++ b/pkg/providers/grpc/pb/provider.pb.go @@ -7,10 +7,11 @@ package pb import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" ) const ( diff --git a/pkg/providers/grpc/pb/provider_grpc.pb.go b/pkg/providers/grpc/pb/provider_grpc.pb.go index a23383cf6..e30766d6e 100644 --- a/pkg/providers/grpc/pb/provider_grpc.pb.go +++ b/pkg/providers/grpc/pb/provider_grpc.pb.go @@ -8,6 +8,7 @@ package pb import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/pkg/webhooks/pod/mutating.go b/pkg/webhooks/pod/mutating.go index 9ed650f18..e3224a46e 100644 --- a/pkg/webhooks/pod/mutating.go +++ b/pkg/webhooks/pod/mutating.go @@ -4,12 +4,16 @@ import ( "context" "fmt" - corev1 "k8s.io/api/core/v1" + "github.com/gocrane/crane/pkg/ensurance/util" + "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" + "github.com/gocrane/api/ensurance/v1alpha1" "github.com/gocrane/crane/pkg/ensurance/config" ) @@ -18,7 +22,15 @@ var ( ) type MutatingAdmission struct { - Config *config.QOSConfig + Config *config.QOSConfig + listPodQOS func() ([]*v1alpha1.PodQOS, error) +} + +func NewMutatingAdmission(config *config.QOSConfig, listPodQOS func() ([]*v1alpha1.PodQOS, error)) *MutatingAdmission { + return &MutatingAdmission{ + Config: config, + listPodQOS: listPodQOS, + } } // Default implements webhook.Defaulter so a webhook will be registered for the type @@ -28,7 +40,7 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err return fmt.Errorf("expected a Pod but got a %T", obj) } - klog.Infof("Into Pod injection %s/%s", pod.Namespace, pod.Name) + klog.Infof("mutating started for pod %s/%s", pod.Namespace, pod.Name) if _, exist := SystemNamespaces[pod.Namespace]; exist { return nil @@ -47,17 +59,54 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err return err } - if ls.Matches(labels.Set(pod.Labels)) { - if m.Config.QOSInitializer.InitContainerTemplate != nil { - pod.Spec.InitContainers = append(pod.Spec.InitContainers, *m.Config.QOSInitializer.InitContainerTemplate) + if !ls.Matches(labels.Set(pod.Labels)) { + klog.Infof("injection skipped: webhook is not interested in the pod") + return nil + } + + qosSlice, err := m.listPodQOS() + if err != nil { + return errors.WithMessage(err, "list PodQOS failed") + } + + /**************************************************************** + * Check whether the pod has a low CPUPriority (CPUPriority > 0) + ****************************************************************/ + qos := util.MatchPodAndPodQOSSlice(pod, qosSlice) + if qos == nil { + klog.Infof("injection skipped: no podqos matched") + return nil + } + + if qos.Spec.ResourceQOS.CPUQOS == nil || + qos.Spec.ResourceQOS.CPUQOS.CPUPriority == nil || + *qos.Spec.ResourceQOS.CPUQOS.CPUPriority == 0 { + klog.Infof("injection skipped: not a low CPUPriority pod, qos %s", qos.Name) + return nil + } + for _, container := range pod.Spec.InitContainers { + if container.Name == m.Config.QOSInitializer.InitContainerTemplate.Name { + klog.Infof("injection skipped: pod has initializerContainer already") + return nil } + } - if m.Config.QOSInitializer.VolumeTemplate != nil { - pod.Spec.Volumes = append(pod.Spec.Volumes, *m.Config.QOSInitializer.VolumeTemplate) + for _, volume := range pod.Spec.Volumes { + if volume.Name == m.Config.QOSInitializer.VolumeTemplate.Name { + klog.Infof("injection skipped: pod has initializerVolume already") + return nil } + } - klog.Infof("Injected QOSInitializer for Pod %s/%s", pod.Namespace, pod.Name) + if m.Config.QOSInitializer.InitContainerTemplate != nil { + pod.Spec.InitContainers = append(pod.Spec.InitContainers, *m.Config.QOSInitializer.InitContainerTemplate) } + if m.Config.QOSInitializer.VolumeTemplate != nil { + pod.Spec.Volumes = append(pod.Spec.Volumes, *m.Config.QOSInitializer.VolumeTemplate) + } + + klog.Infof("mutating completed for pod %s/%s", pod.Namespace, pod.Name) + return nil } diff --git a/pkg/webhooks/pod/mutating_test.go b/pkg/webhooks/pod/mutating_test.go index 6ab632392..8d2d2c4cd 100644 --- a/pkg/webhooks/pod/mutating_test.go +++ b/pkg/webhooks/pod/mutating_test.go @@ -4,6 +4,10 @@ import ( "context" "testing" + "github.com/gocrane/api/ensurance/v1alpha1" + "github.com/stretchr/testify/assert" + "k8s.io/utils/pointer" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" @@ -20,23 +24,80 @@ func TestDefaultingPodQOSInitializer(t *testing.T) { t.Errorf("unmarshal config failed:%v", err) } m := MutatingAdmission{ - Config: config, + Config: config, + listPodQOS: MockListPodQOSFunc, } - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1", - Labels: map[string]string{ - "app": "nginx", - "type": "offline", + type Case struct { + Pod *v1.Pod + Inject bool + } + + for _, tc := range []Case{ + {Pod: MockPod("offline", "offline", "enable", "app", "nginx"), Inject: true}, + {Pod: MockPod("offline-not-interested", "offline", "enable"), Inject: false}, + {Pod: MockPod("online", "offline", "disable", "app", "nginx"), Inject: false}, + {Pod: MockPod("online-not-interested", "offline", "disable"), Inject: false}, + {Pod: MockPod("default"), Inject: false}, + } { + assert.NoError(t, m.Default(context.Background(), tc.Pod)) + t.Log(tc.Pod.Name) + assert.Equal(t, len(tc.Pod.Spec.InitContainers) == 1, tc.Inject) + assert.Equal(t, len(tc.Pod.Spec.Volumes) == 1, tc.Inject) + } +} + +func MockListPodQOSFunc() ([]*v1alpha1.PodQOS, error) { + return []*v1alpha1.PodQOS{ + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: v1alpha1.PodQOSSpec{ + LabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"offline": "enable"}, + }, + ResourceQOS: v1alpha1.ResourceQOS{ + CPUQOS: &v1alpha1.CPUQOS{ + CPUPriority: pointer.Int32(7), + }, + }, + }, + }, { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: v1alpha1.PodQOSSpec{ + LabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"offline": "disable"}, + }, + ResourceQOS: v1alpha1.ResourceQOS{ + CPUQOS: &v1alpha1.CPUQOS{ + CPUPriority: pointer.Int32(0), + }, + }, + }, + }, { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: v1alpha1.PodQOSSpec{ + ResourceQOS: v1alpha1.ResourceQOS{ + CPUQOS: &v1alpha1.CPUQOS{ + CPUPriority: pointer.Int32(7), + }, + }, }, }, + }, nil +} + +func MockPod(name string, labels ...string) *v1.Pod { + labelmap := map[string]string{} + for i := 0; i < len(labels)-1; i += 2 { + labelmap[labels[i]] = labels[i+1] } - err = m.Default(context.TODO(), pod) - if err != nil { - t.Fatalf("inject pod failed: %v", err) - } - if len(pod.Spec.InitContainers) == 0 { - t.Fatalf("should inject containers") + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: labelmap, + }, } } diff --git a/pkg/webhooks/webhook.go b/pkg/webhooks/webhook.go index fdea80860..977f9f738 100644 --- a/pkg/webhooks/webhook.go +++ b/pkg/webhooks/webhook.go @@ -17,14 +17,15 @@ limitations under the License. package webhooks import ( - corev1 "k8s.io/api/core/v1" - "k8s.io/klog/v2" - ctrl "sigs.k8s.io/controller-runtime" + "context" analysisapi "github.com/gocrane/api/analysis/v1alpha1" autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1" ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1" predictionapi "github.com/gocrane/api/prediction/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" "github.com/gocrane/crane/pkg/ensurance/config" "github.com/gocrane/crane/pkg/webhooks/autoscaling" @@ -109,12 +110,10 @@ func SetupWebhookWithManager(mgr ctrl.Manager, autoscalingEnabled, nodeResourceE klog.Errorf("Failed to load qos initializer config: %v", err) } - podMutatingAdmission := pod.MutatingAdmission{ - Config: qosConfig, - } + podMutatingAdmission := pod.NewMutatingAdmission(qosConfig, BuildPodQosListFunction(mgr)) err = ctrl.NewWebhookManagedBy(mgr). For(&corev1.Pod{}). - WithDefaulter(&podMutatingAdmission). + WithDefaulter(podMutatingAdmission). Complete() if err != nil { klog.Errorf("Failed to setup qos initializer webhook: %v", err) @@ -124,3 +123,16 @@ func SetupWebhookWithManager(mgr ctrl.Manager, autoscalingEnabled, nodeResourceE return nil } + +func BuildPodQosListFunction(mgr ctrl.Manager) func() ([]*ensuranceapi.PodQOS, error) { + return func() (qosSlice []*ensuranceapi.PodQOS, err error) { + podQOSList := ensuranceapi.PodQOSList{} + if err := mgr.GetCache().List(context.Background(), &podQOSList); err != nil { + return nil, err + } + for _, qos := range podQOSList.Items { + qosSlice = append(qosSlice, qos.DeepCopy()) + } + return qosSlice, err + } +} diff --git a/tools/initializer/resource.yaml b/tools/initializer/resource.yaml index 9a5b07017..81bf6b509 100644 --- a/tools/initializer/resource.yaml +++ b/tools/initializer/resource.yaml @@ -100,28 +100,36 @@ data: kind: QOSConfig qosInitializer: enable: true - selector: + selector: matchLabels: app: nginx initContainerTemplate: - name: crane-qos-initializer + name: qos-initializer-container image: docker.io/gocrane/qos-init:v0.1.6 imagePullPolicy: IfNotPresent + args: + - "while ! grep -q gocrane.io/cpu-qos /etc/podinfo/annotations; do sleep 1; done; echo cpu qos setting competed;" command: - - sh - - -x - - /qos-checking.sh + - /bin/bash + - -c + resources: + requests: + cpu: 10m + memory: 10Mi + limits: + cpu: 10m + memory: 10Mi volumeMounts: - - name: podinfo + - name: qos-initializer-volume mountPath: /etc/podinfo volumeTemplate: - name: podinfo + name: qos-initializer-volume downwardAPI: items: - path: "annotations" fieldRef: fieldPath: metadata.annotations - + --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration From 3e8bbbec6823a8e618060cdbc40c1a7967eca516 Mon Sep 17 00:00:00 2001 From: payall4u Date: Wed, 29 Nov 2023 18:41:10 +0800 Subject: [PATCH 2/3] Fix log level. --- pkg/webhooks/pod/mutating.go | 14 +++++++------- tools/initializer/resource.yaml | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/webhooks/pod/mutating.go b/pkg/webhooks/pod/mutating.go index e3224a46e..215630308 100644 --- a/pkg/webhooks/pod/mutating.go +++ b/pkg/webhooks/pod/mutating.go @@ -40,7 +40,7 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err return fmt.Errorf("expected a Pod but got a %T", obj) } - klog.Infof("mutating started for pod %s/%s", pod.Namespace, pod.Name) + klog.V(2).Infof("Mutating started for pod %s/%s", pod.Namespace, pod.Name) if _, exist := SystemNamespaces[pod.Namespace]; exist { return nil @@ -60,7 +60,7 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err } if !ls.Matches(labels.Set(pod.Labels)) { - klog.Infof("injection skipped: webhook is not interested in the pod") + klog.V(2).Infof("Injection skipped: webhook is not interested in the pod") return nil } @@ -74,26 +74,26 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err ****************************************************************/ qos := util.MatchPodAndPodQOSSlice(pod, qosSlice) if qos == nil { - klog.Infof("injection skipped: no podqos matched") + klog.V(2).Infof("Injection skipped: no podqos matched") return nil } if qos.Spec.ResourceQOS.CPUQOS == nil || qos.Spec.ResourceQOS.CPUQOS.CPUPriority == nil || *qos.Spec.ResourceQOS.CPUQOS.CPUPriority == 0 { - klog.Infof("injection skipped: not a low CPUPriority pod, qos %s", qos.Name) + klog.V(2).Infof("Injection skipped: not a low CPUPriority pod, qos %s", qos.Name) return nil } for _, container := range pod.Spec.InitContainers { if container.Name == m.Config.QOSInitializer.InitContainerTemplate.Name { - klog.Infof("injection skipped: pod has initializerContainer already") + klog.V(2).Infof("Injection skipped: pod has initializerContainer already") return nil } } for _, volume := range pod.Spec.Volumes { if volume.Name == m.Config.QOSInitializer.VolumeTemplate.Name { - klog.Infof("injection skipped: pod has initializerVolume already") + klog.V(2).Infof("Injection skipped: pod has initializerVolume already") return nil } } @@ -106,7 +106,7 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err pod.Spec.Volumes = append(pod.Spec.Volumes, *m.Config.QOSInitializer.VolumeTemplate) } - klog.Infof("mutating completed for pod %s/%s", pod.Namespace, pod.Name) + klog.V(2).Infof("Mutating completed for pod %s/%s", pod.Namespace, pod.Name) return nil } diff --git a/tools/initializer/resource.yaml b/tools/initializer/resource.yaml index 81bf6b509..1d47a4779 100644 --- a/tools/initializer/resource.yaml +++ b/tools/initializer/resource.yaml @@ -104,7 +104,7 @@ data: matchLabels: app: nginx initContainerTemplate: - name: qos-initializer-container + name: crane-qos-initializer image: docker.io/gocrane/qos-init:v0.1.6 imagePullPolicy: IfNotPresent args: @@ -120,10 +120,10 @@ data: cpu: 10m memory: 10Mi volumeMounts: - - name: qos-initializer-volume + - name: crane-qos-initializer-volume mountPath: /etc/podinfo volumeTemplate: - name: qos-initializer-volume + name: crane-qos-initializer-volume downwardAPI: items: - path: "annotations" From 74132afea8308a26c1d896cb95befd00e1cdeaa6 Mon Sep 17 00:00:00 2001 From: payall4u Date: Wed, 29 Nov 2023 20:23:44 +0800 Subject: [PATCH 3/3] Add nil check and add unit test --- pkg/ensurance/analyzer/analyzer.go | 3 +- .../collector/cadvisor/cadvisor_linux.go | 2 +- pkg/webhooks/pod/mutating.go | 20 ++++++---- pkg/webhooks/pod/mutating_test.go | 40 ++++++++++++++----- 4 files changed, 45 insertions(+), 20 deletions(-) diff --git a/pkg/ensurance/analyzer/analyzer.go b/pkg/ensurance/analyzer/analyzer.go index 561e18a46..55aad6ffe 100644 --- a/pkg/ensurance/analyzer/analyzer.go +++ b/pkg/ensurance/analyzer/analyzer.go @@ -6,8 +6,6 @@ import ( "strings" "time" - "github.com/gocrane/crane/pkg/ensurance/util" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -29,6 +27,7 @@ import ( ecache "github.com/gocrane/crane/pkg/ensurance/cache" "github.com/gocrane/crane/pkg/ensurance/executor" "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" + "github.com/gocrane/crane/pkg/ensurance/util" "github.com/gocrane/crane/pkg/known" "github.com/gocrane/crane/pkg/metrics" "github.com/gocrane/crane/pkg/utils" diff --git a/pkg/ensurance/collector/cadvisor/cadvisor_linux.go b/pkg/ensurance/collector/cadvisor/cadvisor_linux.go index f0850606d..ce0416a57 100644 --- a/pkg/ensurance/collector/cadvisor/cadvisor_linux.go +++ b/pkg/ensurance/collector/cadvisor/cadvisor_linux.go @@ -78,7 +78,7 @@ func NewCadvisorManager(cgroupDriver string) Manager { sysfs := csysfs.NewRealSysFs() maxHousekeepingConfig := cmanager.HouskeepingConfig{Interval: &maxHousekeepingInterval, AllowDynamic: &allowDynamic} - m, err := cmanager.New(memCache, sysfs, maxHousekeepingConfig, includedMetrics, http.DefaultClient, []string{"/" + utils.CgroupKubePods}, nil /* containerEnvMetadataWhiteList */, "" /* perfEventsFile */, time.Duration(0) /*resctrlInterval*/) + m, err := cmanager.New(memCache, sysfs, maxHousekeepingConfig, includedMetrics, http.DefaultClient, []string{"/" + utils.CgroupKubePods}, nil /* containerEnvMetadataWhiteList */, "" /* perfEventsFile */, time.Duration(0) /*resctrlInterval*/) if err != nil { klog.Errorf("Failed to create cadvisor manager start: %v", err) return nil diff --git a/pkg/webhooks/pod/mutating.go b/pkg/webhooks/pod/mutating.go index 215630308..c0112f67f 100644 --- a/pkg/webhooks/pod/mutating.go +++ b/pkg/webhooks/pod/mutating.go @@ -4,17 +4,17 @@ import ( "context" "fmt" - "github.com/gocrane/crane/pkg/ensurance/util" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" "github.com/gocrane/api/ensurance/v1alpha1" + "github.com/gocrane/crane/pkg/ensurance/config" + "github.com/gocrane/crane/pkg/ensurance/util" ) var ( @@ -46,11 +46,7 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err return nil } - if m.Config == nil || !m.Config.QOSInitializer.Enable { - return nil - } - - if pod.Labels == nil { + if !m.available() { return nil } @@ -84,6 +80,7 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err klog.V(2).Infof("Injection skipped: not a low CPUPriority pod, qos %s", qos.Name) return nil } + for _, container := range pod.Spec.InitContainers { if container.Name == m.Config.QOSInitializer.InitContainerTemplate.Name { klog.V(2).Infof("Injection skipped: pod has initializerContainer already") @@ -110,3 +107,10 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err return nil } + +func (m *MutatingAdmission) available() bool { + return m.Config != nil && + m.Config.QOSInitializer.Enable && + m.Config.QOSInitializer.InitContainerTemplate != nil && + m.Config.QOSInitializer.VolumeTemplate != nil +} diff --git a/pkg/webhooks/pod/mutating_test.go b/pkg/webhooks/pod/mutating_test.go index 8d2d2c4cd..e060f1ce5 100644 --- a/pkg/webhooks/pod/mutating_test.go +++ b/pkg/webhooks/pod/mutating_test.go @@ -4,14 +4,14 @@ import ( "context" "testing" - "github.com/gocrane/api/ensurance/v1alpha1" "github.com/stretchr/testify/assert" - "k8s.io/utils/pointer" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" "sigs.k8s.io/yaml" + "github.com/gocrane/api/ensurance/v1alpha1" + "github.com/gocrane/crane/pkg/ensurance/config" ) @@ -47,6 +47,21 @@ func TestDefaultingPodQOSInitializer(t *testing.T) { } } +func TestPrecheck(t *testing.T) { + configYaml := "apiVersion: ensurance.crane.io/v1alpha1\nkind: QOSConfig\nqosInitializer:\n enable: true\n selector: \n matchLabels:\n app: nginx\n" + + config := &config.QOSConfig{} + err := yaml.Unmarshal([]byte(configYaml), config) + if err != nil { + t.Errorf("unmarshal config failed:%v", err) + } + m := MutatingAdmission{ + Config: config, + listPodQOS: MockListPodQOSFunc, + } + assert.False(t, m.available()) +} + func MockListPodQOSFunc() ([]*v1alpha1.PodQOS, error) { return []*v1alpha1.PodQOS{ { @@ -90,14 +105,21 @@ func MockListPodQOSFunc() ([]*v1alpha1.PodQOS, error) { } func MockPod(name string, labels ...string) *v1.Pod { - labelmap := map[string]string{} - for i := 0; i < len(labels)-1; i += 2 { - labelmap[labels[i]] = labels[i+1] - } - return &v1.Pod{ + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Labels: labelmap, + Labels: nil, }, } + + if len(labels) < 2 { + return pod + } + + labelmap := map[string]string{} + for i := 0; i < len(labels)-1; i += 2 { + labelmap[labels[i]] = labels[i+1] + } + pod.Labels = labelmap + return pod }