Optimize qos initializer.

pkg/ensurance/analyzer/analyzer.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gocrane/crane/pkg/ensurance/util"
+
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -490,7 +492,7 @@ func (s *AnomalyAnalyzer) filterPodQOSMatches(pods []*v1.Pod, actionName string)
 	}
 	for _, qos := range podQOSList {
 		for _, pod := range pods {
-			if !match(pod, qos) {
+			if !util.MatchPodAndPodQOS(pod, qos) {
 				klog.V(4).Infof("Pod %s/%s does not match PodQOS %s", pod.Namespace, pod.Name, qos.Name)
 				continue
 

pkg/ensurance/analyzer/podqos-fetcher.go → pkg/ensurance/util/match_qos.go

@@ -1,8 +1,9 @@
-package analyzer
+package util
 
 import (
 	"fmt"
 	"reflect"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -69,7 +70,41 @@ func labelMatch(labelSelector metav1.LabelSelector, matchLabels map[string]strin
 	return true
 }
 
-func match(pod *v1.Pod, podQOS *ensuranceapi.PodQOS) bool {
+func sortQOSSlice(qosSlice []*ensuranceapi.PodQOS) {
+	sort.Slice(qosSlice, func(i, j int) bool {
+		if len(qosSlice[i].Spec.LabelSelector.MatchLabels) != len(qosSlice[j].Spec.LabelSelector.MatchLabels) {
+			return len(qosSlice[i].Spec.LabelSelector.MatchLabels) > len(qosSlice[j].Spec.LabelSelector.MatchLabels)
+		}
+
+		if qosSlice[i].Spec.ScopeSelector == nil && qosSlice[j].Spec.ScopeSelector == nil {
+			return true
+		}
+
+		if qosSlice[i].Spec.ScopeSelector == nil {
+			return false
+		}
+
+		if qosSlice[j].Spec.ScopeSelector == nil {
+			return true
+		}
+
+		return len(qosSlice[i].Spec.ScopeSelector.MatchExpressions) > len(qosSlice[j].Spec.ScopeSelector.MatchExpressions)
+	})
+}
+
+func MatchPodAndPodQOSSlice(pod *v1.Pod, qosSlice []*ensuranceapi.PodQOS) (res *ensuranceapi.PodQOS) {
+	newSlice := make([]*ensuranceapi.PodQOS, len(qosSlice), len(qosSlice))
+	copy(newSlice, qosSlice)
+	sortQOSSlice(newSlice)
+	for _, qos := range newSlice {
+		if MatchPodAndPodQOS(pod, qos) {
+			return qos
+		}
+	}
+	return nil
+}
+
+func MatchPodAndPodQOS(pod *v1.Pod, podQOS *ensuranceapi.PodQOS) bool {
 
 	if podQOS.Spec.ScopeSelector == nil &&
 		podQOS.Spec.LabelSelector.MatchLabels == nil &&

pkg/webhooks/pod/mutating.go

@@ -4,12 +4,16 @@ import (
 	"context"
 	"fmt"
 
-	corev1 "k8s.io/api/core/v1"
+	"github.com/gocrane/crane/pkg/ensurance/util"
+	"github.com/pkg/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/klog/v2"
 
+	"github.com/gocrane/api/ensurance/v1alpha1"
 	"github.com/gocrane/crane/pkg/ensurance/config"
 )
 
@@ -18,7 +22,15 @@ var (
 )
 
 type MutatingAdmission struct {
-	Config *config.QOSConfig
+	Config     *config.QOSConfig
+	listPodQOS func() ([]*v1alpha1.PodQOS, error)
+}
+
+func NewMutatingAdmission(config *config.QOSConfig, listPodQOS func() ([]*v1alpha1.PodQOS, error)) *MutatingAdmission {
+	return &MutatingAdmission{
+		Config:     config,
+		listPodQOS: listPodQOS,
+	}
 }
 
 // Default implements webhook.Defaulter so a webhook will be registered for the type
@@ -28,7 +40,7 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err
 		return fmt.Errorf("expected a Pod but got a %T", obj)
 	}
 
-	klog.Infof("Into Pod injection %s/%s", pod.Namespace, pod.Name)
+	klog.V(2).Infof("Mutating started for pod %s/%s", pod.Namespace, pod.Name)
 
 	if _, exist := SystemNamespaces[pod.Namespace]; exist {
 		return nil
@@ -47,17 +59,54 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err
 		return err
 	}
 
-	if ls.Matches(labels.Set(pod.Labels)) {
-		if m.Config.QOSInitializer.InitContainerTemplate != nil {
-			pod.Spec.InitContainers = append(pod.Spec.InitContainers, *m.Config.QOSInitializer.InitContainerTemplate)
+	if !ls.Matches(labels.Set(pod.Labels)) {
+		klog.V(2).Infof("Injection skipped: webhook is not interested in the pod")
+		return nil
+	}
+
+	qosSlice, err := m.listPodQOS()
+	if err != nil {
+		return errors.WithMessage(err, "list PodQOS failed")
+	}
+
+	/****************************************************************
+	 *	Check whether the pod has a low CPUPriority (CPUPriority > 0)
+	 ****************************************************************/
+	qos := util.MatchPodAndPodQOSSlice(pod, qosSlice)
+	if qos == nil {
+		klog.V(2).Infof("Injection skipped: no podqos matched")
+		return nil
+	}
+
+	if qos.Spec.ResourceQOS.CPUQOS == nil ||
+		qos.Spec.ResourceQOS.CPUQOS.CPUPriority == nil ||
+		*qos.Spec.ResourceQOS.CPUQOS.CPUPriority == 0 {
+		klog.V(2).Infof("Injection skipped: not a low CPUPriority pod, qos %s", qos.Name)
+		return nil
+	}
+	for _, container := range pod.Spec.InitContainers {
+		if container.Name == m.Config.QOSInitializer.InitContainerTemplate.Name {
+			klog.V(2).Infof("Injection skipped: pod has initializerContainer already")
+			return nil
 		}
+	}
 
-		if m.Config.QOSInitializer.VolumeTemplate != nil {
-			pod.Spec.Volumes = append(pod.Spec.Volumes, *m.Config.QOSInitializer.VolumeTemplate)
+	for _, volume := range pod.Spec.Volumes {
+		if volume.Name == m.Config.QOSInitializer.VolumeTemplate.Name {
+			klog.V(2).Infof("Injection skipped: pod has initializerVolume already")
+			return nil
 		}
+	}
 
-		klog.Infof("Injected QOSInitializer for Pod %s/%s", pod.Namespace, pod.Name)
+	if m.Config.QOSInitializer.InitContainerTemplate != nil {
+		pod.Spec.InitContainers = append(pod.Spec.InitContainers, *m.Config.QOSInitializer.InitContainerTemplate)
 	}
 
+	if m.Config.QOSInitializer.VolumeTemplate != nil {
+		pod.Spec.Volumes = append(pod.Spec.Volumes, *m.Config.QOSInitializer.VolumeTemplate)
+	}
+
+	klog.V(2).Infof("Mutating completed for pod %s/%s", pod.Namespace, pod.Name)
+
 	return nil
 }

pkg/webhooks/pod/mutating_test.go

@@ -4,6 +4,10 @@ import (
 	"context"
 	"testing"
 
+	"github.com/gocrane/api/ensurance/v1alpha1"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/utils/pointer"
+
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
@@ -20,23 +24,80 @@ func TestDefaultingPodQOSInitializer(t *testing.T) {
 		t.Errorf("unmarshal config failed:%v", err)
 	}
 	m := MutatingAdmission{
-		Config: config,
+		Config:     config,
+		listPodQOS: MockListPodQOSFunc,
 	}
 
-	pod := &v1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "pod1",
-			Labels: map[string]string{
-				"app":  "nginx",
-				"type": "offline",
+	type Case struct {
+		Pod    *v1.Pod
+		Inject bool
+	}
+
+	for _, tc := range []Case{
+		{Pod: MockPod("offline", "offline", "enable", "app", "nginx"), Inject: true},
+		{Pod: MockPod("offline-not-interested", "offline", "enable"), Inject: false},
+		{Pod: MockPod("online", "offline", "disable", "app", "nginx"), Inject: false},
+		{Pod: MockPod("online-not-interested", "offline", "disable"), Inject: false},
+		{Pod: MockPod("default"), Inject: false},
+	} {
+		assert.NoError(t, m.Default(context.Background(), tc.Pod))
+		t.Log(tc.Pod.Name)
+		assert.Equal(t, len(tc.Pod.Spec.InitContainers) == 1, tc.Inject)
+		assert.Equal(t, len(tc.Pod.Spec.Volumes) == 1, tc.Inject)
+	}
+}
+
+func MockListPodQOSFunc() ([]*v1alpha1.PodQOS, error) {
+	return []*v1alpha1.PodQOS{
+		{
+			TypeMeta:   metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{},
+			Spec: v1alpha1.PodQOSSpec{
+				LabelSelector: metav1.LabelSelector{
+					MatchLabels: map[string]string{"offline": "enable"},
+				},
+				ResourceQOS: v1alpha1.ResourceQOS{
+					CPUQOS: &v1alpha1.CPUQOS{
+						CPUPriority: pointer.Int32(7),
+					},
+				},
+			},
+		}, {
+			TypeMeta:   metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{},
+			Spec: v1alpha1.PodQOSSpec{
+				LabelSelector: metav1.LabelSelector{
+					MatchLabels: map[string]string{"offline": "disable"},
+				},
+				ResourceQOS: v1alpha1.ResourceQOS{
+					CPUQOS: &v1alpha1.CPUQOS{
+						CPUPriority: pointer.Int32(0),
+					},
+				},
+			},
+		}, {
+			TypeMeta:   metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{},
+			Spec: v1alpha1.PodQOSSpec{
+				ResourceQOS: v1alpha1.ResourceQOS{
+					CPUQOS: &v1alpha1.CPUQOS{
+						CPUPriority: pointer.Int32(7),
+					},
+				},
 			},
 		},
+	}, nil
+}
+
+func MockPod(name string, labels ...string) *v1.Pod {
+	labelmap := map[string]string{}
+	for i := 0; i < len(labels)-1; i += 2 {
+		labelmap[labels[i]] = labels[i+1]
 	}
-	err = m.Default(context.TODO(), pod)
-	if err != nil {
-		t.Fatalf("inject pod failed: %v", err)
-	}
-	if len(pod.Spec.InitContainers) == 0 {
-		t.Fatalf("should inject containers")
+	return &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   name,
+			Labels: labelmap,
+		},
 	}
 }

pkg/webhooks/webhook.go

@@ -17,14 +17,15 @@ limitations under the License.
 package webhooks
 
 import (
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/klog/v2"
-	ctrl "sigs.k8s.io/controller-runtime"
+	"context"
 
 	analysisapi "github.com/gocrane/api/analysis/v1alpha1"
 	autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1"
 	ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1"
 	predictionapi "github.com/gocrane/api/prediction/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/gocrane/crane/pkg/ensurance/config"
 	"github.com/gocrane/crane/pkg/webhooks/autoscaling"
@@ -109,12 +110,10 @@ func SetupWebhookWithManager(mgr ctrl.Manager, autoscalingEnabled, nodeResourceE
 			klog.Errorf("Failed to load qos initializer config: %v", err)
 		}
 
-		podMutatingAdmission := pod.MutatingAdmission{
-			Config: qosConfig,
-		}
+		podMutatingAdmission := pod.NewMutatingAdmission(qosConfig, BuildPodQosListFunction(mgr))
 		err = ctrl.NewWebhookManagedBy(mgr).
 			For(&corev1.Pod{}).
-			WithDefaulter(&podMutatingAdmission).
+			WithDefaulter(podMutatingAdmission).
 			Complete()
 		if err != nil {
 			klog.Errorf("Failed to setup qos initializer webhook: %v", err)
@@ -124,3 +123,16 @@ func SetupWebhookWithManager(mgr ctrl.Manager, autoscalingEnabled, nodeResourceE
 
 	return nil
 }
+
+func BuildPodQosListFunction(mgr ctrl.Manager) func() ([]*ensuranceapi.PodQOS, error) {
+	return func() (qosSlice []*ensuranceapi.PodQOS, err error) {
+		podQOSList := ensuranceapi.PodQOSList{}
+		if err := mgr.GetCache().List(context.Background(), &podQOSList); err != nil {
+			return nil, err
+		}
+		for _, qos := range podQOSList.Items {
+			qosSlice = append(qosSlice, qos.DeepCopy())
+		}
+		return qosSlice, err
+	}
+}

tools/initializer/resource.yaml

@@ -100,28 +100,36 @@ data:
     kind: QOSConfig
     qosInitializer:
       enable: true
-      selector: 
+      selector:
         matchLabels:
           app: nginx
       initContainerTemplate:
         name: crane-qos-initializer
         image: docker.io/gocrane/qos-init:v0.1.6
         imagePullPolicy: IfNotPresent
+        args:
+          - "while ! grep -q gocrane.io/cpu-qos /etc/podinfo/annotations; do sleep 1; done; echo cpu qos setting competed;"
         command:
-          - sh
-          - -x
-          - /qos-checking.sh
+          - /bin/bash
+          - -c
+        resources:
+          requests:
+            cpu: 10m
+            memory: 10Mi
+          limits:
+            cpu: 10m
+            memory: 10Mi
         volumeMounts:
-          - name: podinfo
+          - name: crane-qos-initializer-volume
             mountPath: /etc/podinfo
       volumeTemplate:
-        name: podinfo
+        name: crane-qos-initializer-volume
         downwardAPI:
           items:
           - path: "annotations"
             fieldRef:
               fieldPath: metadata.annotations
-
+              
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration