From bde23d3d95b642cff2c7bab2014c0cacaed871e3 Mon Sep 17 00:00:00 2001 From: qmhu Date: Fri, 24 Mar 2023 14:59:45 +0800 Subject: [PATCH 1/6] update deploy to v0.10.0 --- deploy/crane-agent/daemonset.yaml | 2 +- deploy/craned/deployment.yaml | 4 ++-- deploy/craned/new.yaml | 0 deploy/manifests/analysis.crane.io_recommendationrules.yaml | 4 ++++ deploy/metric-adapter/deployment.yaml | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) delete mode 100644 deploy/craned/new.yaml diff --git a/deploy/crane-agent/daemonset.yaml b/deploy/crane-agent/daemonset.yaml index c3d38744d..0c47ac0da 100644 --- a/deploy/crane-agent/daemonset.yaml +++ b/deploy/crane-agent/daemonset.yaml @@ -28,7 +28,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - image: docker.io/gocrane/crane-agent:v0.9.0 + image: docker.io/gocrane/crane-agent:v0.10.0 imagePullPolicy: Always command: - /crane-agent diff --git a/deploy/craned/deployment.yaml b/deploy/craned/deployment.yaml index 4f4793435..be013e17e 100644 --- a/deploy/craned/deployment.yaml +++ b/deploy/craned/deployment.yaml @@ -35,7 +35,7 @@ spec: operator: Exists containers: - name: craned - image: docker.io/gocrane/craned:v0.9.0 + image: docker.io/gocrane/craned:v0.10.0 imagePullPolicy: IfNotPresent env: - name: TZ @@ -61,7 +61,7 @@ spec: path: /api/healthz port: 8082 scheme: HTTP - - image: docker.io/gocrane/dashboard:v0.8.0 + - image: docker.io/gocrane/dashboard:v0.10.0 imagePullPolicy: IfNotPresent name: dashboard volumeMounts: diff --git a/deploy/craned/new.yaml b/deploy/craned/new.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/deploy/manifests/analysis.crane.io_recommendationrules.yaml b/deploy/manifests/analysis.crane.io_recommendationrules.yaml index 3892b7c32..f8a250052 100644 --- a/deploy/manifests/analysis.crane.io_recommendationrules.yaml +++ b/deploy/manifests/analysis.crane.io_recommendationrules.yaml @@ -287,6 +287,10 @@ spec: type: object type: array x-kubernetes-list-type: atomic + runNumber: + description: RunNumber is the numbers of runs + format: int32 + type: integer type: object type: object served: true diff --git a/deploy/metric-adapter/deployment.yaml b/deploy/metric-adapter/deployment.yaml index 0bc7c89d6..ca6716b54 100644 --- a/deploy/metric-adapter/deployment.yaml +++ b/deploy/metric-adapter/deployment.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: metric-adapter containers: - name: metric-adapter - image: docker.io/gocrane/metric-adapter:v0.9.0 + image: docker.io/gocrane/metric-adapter:v0.10.0 imagePullPolicy: IfNotPresent env: - name: TZ From f7032698849652f1ff010054ffa243f4aaa064e7 Mon Sep 17 00:00:00 2001 From: qmhu Date: Fri, 24 Mar 2023 15:02:05 +0800 Subject: [PATCH 2/6] add go.sum --- go.sum | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.sum b/go.sum index 4eb8c7cf4..db35d60e9 100644 --- a/go.sum +++ b/go.sum @@ -356,6 +356,8 @@ github.com/gocrane/api v0.0.0-20230117025609-8b3a495647d9 h1:3R06LMq6zl+LyYBlmoV github.com/gocrane/api v0.0.0-20230117025609-8b3a495647d9/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= github.com/gocrane/api v0.9.1-0.20230307114903-ce6fcd9a2eaf h1:HwjISCuAIPI/ZqzLDmw5WL1Yz0vjshzXt/3vNXEsCJs= github.com/gocrane/api v0.9.1-0.20230307114903-ce6fcd9a2eaf/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= +github.com/gocrane/api v0.10.0 h1:qzYFg9I2NNuvkZ7PZKCOmz4oEnRa+hvl68rEW12ZGdI= +github.com/gocrane/api v0.10.0/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= From 04da0402dd3163a1ab421881a5b25a625f93f4e9 Mon Sep 17 00:00:00 2001 From: qmhu Date: Tue, 18 Apr 2023 14:39:58 +0800 Subject: [PATCH 3/6] analytics conversion for recommendation namespace --- cmd/craned/app/manager.go | 7 +++++- .../analytics/analytics_controller.go | 14 ++++++++++-- .../recommendation_rule_controller.go | 22 +++++++++++++++++++ .../recommendation_trigger_controller.go | 2 +- pkg/known/annotation.go | 1 + 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/cmd/craned/app/manager.go b/cmd/craned/app/manager.go index a5fd00f97..e526755b8 100644 --- a/cmd/craned/app/manager.go +++ b/cmd/craned/app/manager.go @@ -106,7 +106,12 @@ func Run(ctx context.Context, opts *options.Options) error { return err } - if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil { + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + klog.ErrorS(err, "failed to add health check endpoint") + return err + } + + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { klog.ErrorS(err, "failed to add health check endpoint") return err } diff --git a/pkg/controller/analytics/analytics_controller.go b/pkg/controller/analytics/analytics_controller.go index 928c39be5..53b1a8d55 100644 --- a/pkg/controller/analytics/analytics_controller.go +++ b/pkg/controller/analytics/analytics_controller.go @@ -546,8 +546,11 @@ func setReadyCondition(status *analysisv1alph1.AnalyticsStatus, conditionStatus func ConvertToRecommendationRule(analytics *analysisv1alph1.Analytics) *analysisv1alph1.RecommendationRule { recommendationRule := &analysisv1alph1.RecommendationRule{} recommendationRule.Name = analytics.Name + if recommendationRule.Annotations == nil { + recommendationRule.Annotations = map[string]string{} + } + recommendationRule.Annotations[known.AnalyticsConversionAnnotation] = string(analytics.UID) recommendationRule.Spec.ResourceSelectors = analytics.Spec.ResourceSelectors - // todo: make sure the conversion is right after recommendation refactor recommendationRule.Spec.Recommenders = []analysisv1alph1.Recommender{{Name: string(analytics.Spec.Type)}} if analytics.Namespace == known.CraneSystemNamespace { recommendationRule.Spec.NamespaceSelector.Any = true @@ -560,6 +563,11 @@ func ConvertToRecommendationRule(analytics *analysisv1alph1.Analytics) *analysis recommendationRule.Spec.RunInterval = (time.Duration(*analytics.Spec.CompletionStrategy.PeriodSeconds) * time.Second).String() } + recommendationRule.Status = analysisv1alph1.RecommendationRuleStatus{ + LastUpdateTime: analytics.Status.LastUpdateTime, + Recommendations: analytics.Status.Recommendations, + } + return recommendationRule } @@ -572,8 +580,10 @@ func UpsertRecommendationRule(recommendationRule *analysisv1alph1.Recommendation return err } - if !reflect.DeepEqual(recommendationRule.Spec, recommendationRuleExist.Spec) { + if !reflect.DeepEqual(recommendationRule.Spec, recommendationRuleExist.Spec) || + reflect.DeepEqual(recommendationRule.Annotations, recommendationRuleExist.Annotations) { recommendationRuleExist.Spec = recommendationRule.Spec + recommendationRuleExist.Annotations = recommendationRule.Annotations return client.Update(context.TODO(), recommendationRuleExist) } diff --git a/pkg/controller/recommendation/recommendation_rule_controller.go b/pkg/controller/recommendation/recommendation_rule_controller.go index 1323ccc61..6e145a7c1 100644 --- a/pkg/controller/recommendation/recommendation_rule_controller.go +++ b/pkg/controller/recommendation/recommendation_rule_controller.go @@ -157,6 +157,12 @@ func (c *RecommendationRuleController) doReconcile(ctx context.Context, recommen opts := []client.ListOption{ client.MatchingLabels(map[string]string{known.RecommendationRuleUidLabel: string(recommendationRule.UID)}), } + if convert, uid := IsConvertFromAnalytics(recommendationRule); convert { + opts = []client.ListOption{ + client.MatchingLabels(map[string]string{known.AnalyticsUidLabel: uid}), + } + } + err = c.Client.List(ctx, &currRecommendations, opts...) if err != nil { c.Recorder.Event(recommendationRule, corev1.EventTypeWarning, "FailedSelectResource", err.Error()) @@ -378,6 +384,9 @@ func CreateRecommendationObject(recommendationRule *analysisv1alph1.Recommendati if id.Namespace != "" { namespace = id.Namespace } + if convert, _ := IsConvertFromAnalytics(recommendationRule); convert { + namespace = known.CraneSystemNamespace + } recommendation := &analysisv1alph1.Recommendation{ ObjectMeta: metav1.ObjectMeta{ @@ -396,6 +405,9 @@ func CreateRecommendationObject(recommendationRule *analysisv1alph1.Recommendati labels := map[string]string{} labels[known.RecommendationRuleNameLabel] = recommendationRule.Name labels[known.RecommendationRuleUidLabel] = string(recommendationRule.UID) + if convert, uid := IsConvertFromAnalytics(recommendationRule); convert { + labels[known.AnalyticsUidLabel] = uid + } labels[known.RecommendationRuleRecommenderLabel] = recommenderName labels[known.RecommendationRuleTargetKindLabel] = target.Kind labels[known.RecommendationRuleTargetVersionLabel] = target.GroupVersionKind().Version @@ -483,3 +495,13 @@ func executeMission(ctx context.Context, wg *sync.WaitGroup, recommenderMgr reco mission.APIVersion = recommendation.APIVersion } } + +func IsConvertFromAnalytics(recommendationRule *analysisv1alph1.RecommendationRule) (bool, string) { + if recommendationRule.Annotations != nil { + if uid, exist := recommendationRule.Annotations[known.AnalyticsConversionAnnotation]; exist { + return true, uid + } + } + + return false, "" +} diff --git a/pkg/controller/recommendation/recommendation_trigger_controller.go b/pkg/controller/recommendation/recommendation_trigger_controller.go index 354d58e1d..b10f12299 100644 --- a/pkg/controller/recommendation/recommendation_trigger_controller.go +++ b/pkg/controller/recommendation/recommendation_trigger_controller.go @@ -54,7 +54,7 @@ func (c *RecommendationTriggerController) Reconcile(ctx context.Context, req ctr recommendationRuleRef := utils.GetRecommendationRuleOwnerReference(recommendation) if recommendationRuleRef == nil { - klog.Warningf("cannot found referred recommendation rule %s/%s", recommendation.Namespace, recommendationRuleRef.Name) + klog.Warning("cannot found referred recommendation rule") return ctrl.Result{}, nil } diff --git a/pkg/known/annotation.go b/pkg/known/annotation.go index 6574d5b77..4775eae73 100644 --- a/pkg/known/annotation.go +++ b/pkg/known/annotation.go @@ -5,6 +5,7 @@ const ( ReplicasRecommendationValueAnnotation = "analysis.crane.io/replicas-recommendation" ResourceRecommendationValueAnnotation = "analysis.crane.io/resource-recommendation" RunNumberAnnotation = "analysis.crane.io/run-number" + AnalyticsConversionAnnotation = "analysis.crane.io/analytics-conversion" ) const ( From f91585f261611ae43fc20078f582b927544b7651 Mon Sep 17 00:00:00 2001 From: payall4u Date: Mon, 27 Nov 2023 19:34:36 +0800 Subject: [PATCH 4/6] Optimize qos initializer. --- go.mod | 7 +- go.sum | 13 --- initializer/Dockerfile | 4 - initializer/podinfo | 4 - initializer/qos-checking.sh | 22 ----- pkg/ensurance/analyzer/analyzer.go | 4 +- .../podqos-fetcher.go => util/match_qos.go} | 39 ++++++++- pkg/providers/grpc/pb/provider.pb.go | 5 +- pkg/providers/grpc/pb/provider_grpc.pb.go | 1 + pkg/webhooks/pod/mutating.go | 68 +++++++++++++-- pkg/webhooks/pod/mutating_test.go | 86 ++++++++++++++++--- pkg/webhooks/webhook.go | 26 ++++-- tools/initializer/resource.yaml | 24 ++++-- 13 files changed, 213 insertions(+), 90 deletions(-) delete mode 100644 initializer/Dockerfile delete mode 100644 initializer/podinfo delete mode 100644 initializer/qos-checking.sh rename pkg/ensurance/{analyzer/podqos-fetcher.go => util/match_qos.go} (85%) diff --git a/go.mod b/go.mod index 1681aaf04..a3eb6f23f 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/google/cadvisor v0.39.2 github.com/jaypipes/ghw v0.9.0 github.com/mjibson/go-dsp v0.0.0-20180508042940-11479a337f12 + github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.26.0 github.com/shirou/gopsutil v3.21.10+incompatible @@ -52,12 +53,10 @@ require ( ) require ( - github.com/BurntSushi/toml v0.3.1 // indirect github.com/Microsoft/go-winio v0.5.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/alessio/shellescape v1.4.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/blang/semver v3.5.1+incompatible // indirect @@ -76,7 +75,6 @@ require ( github.com/emicklei/go-restful v2.15.0+incompatible // indirect github.com/emicklei/go-restful-swagger12 v0.0.0-20201014110547-68ccff494617 // indirect github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.2.0 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -121,8 +119,6 @@ require ( github.com/opencontainers/runc v1.0.2 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect github.com/opencontainers/selinux v1.8.2 // indirect - github.com/pelletier/go-toml v1.9.3 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect @@ -169,7 +165,6 @@ require ( k8s.io/kube-scheduler v0.0.0 // indirect k8s.io/mount-utils v0.22.3 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 // indirect - sigs.k8s.io/kind v0.11.1 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect ) diff --git a/go.sum b/go.sum index db35d60e9..3454d2863 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= -github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -260,8 +258,6 @@ github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdR github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.2.0 h1:8ozOH5xxoMYDt5/u+yMTsVXydVCbTORFnOOoq2lumco= -github.com/evanphx/json-patch/v5 v5.2.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -352,10 +348,6 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.1.0-rc.5 h1:QOAag7FoBaBYYHRqzqkhhd8fq5RTubvI4v3Ft/gDVVQ= github.com/gobwas/ws v1.1.0-rc.5/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= -github.com/gocrane/api v0.0.0-20230117025609-8b3a495647d9 h1:3R06LMq6zl+LyYBlmoVPXjY2E3XtsMqZ7WPBA+BMa/M= -github.com/gocrane/api v0.0.0-20230117025609-8b3a495647d9/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= -github.com/gocrane/api v0.9.1-0.20230307114903-ce6fcd9a2eaf h1:HwjISCuAIPI/ZqzLDmw5WL1Yz0vjshzXt/3vNXEsCJs= -github.com/gocrane/api v0.9.1-0.20230307114903-ce6fcd9a2eaf/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= github.com/gocrane/api v0.10.0 h1:qzYFg9I2NNuvkZ7PZKCOmz4oEnRa+hvl68rEW12ZGdI= github.com/gocrane/api v0.10.0/go.mod h1:GxI+t9AW8+NsHkz2JkPBIJN//9eLUjTZl1ScYAbXMbk= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -724,8 +716,6 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -824,7 +814,6 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= @@ -1517,8 +1506,6 @@ sigs.k8s.io/controller-runtime v0.10.2 h1:jW8qiY+yMnnPx6O9hu63tgcwaKzd1yLYui+mpv sigs.k8s.io/controller-runtime v0.10.2/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/custom-metrics-apiserver v1.22.0 h1:nRrRRCq46m3y6lCp/6rfptPjX0eGsF88s66vt9TWgac= sigs.k8s.io/custom-metrics-apiserver v1.22.0/go.mod h1:QST5+Nu7RXcDVbg19K+0UT+2QMxnw+FSB6q7sv3yDUw= -sigs.k8s.io/kind v0.11.1 h1:pVzOkhUwMBrCB0Q/WllQDO3v14Y+o2V0tFgjTqIUjwA= -sigs.k8s.io/kind v0.11.1/go.mod h1:fRpgVhtqAWrtLB9ED7zQahUimpUXuG/iHT88xYqEGIA= sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g= sigs.k8s.io/kustomize/cmd/config v0.9.13/go.mod h1:7547FLF8W/lTaDf0BDqFTbZxM9zqwEJqCKN9sSR0xSs= sigs.k8s.io/kustomize/kustomize/v4 v4.2.0/go.mod h1:MOkR6fmhwG7hEDRXBYELTi5GSFcLwfqwzTRHW3kv5go= diff --git a/initializer/Dockerfile b/initializer/Dockerfile deleted file mode 100644 index 2720475c1..000000000 --- a/initializer/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM alpine:3.13.5 - -WORKDIR / -COPY qos-checking.sh / \ No newline at end of file diff --git a/initializer/podinfo b/initializer/podinfo deleted file mode 100644 index 1444295cd..000000000 --- a/initializer/podinfo +++ /dev/null @@ -1,4 +0,0 @@ -cluster="test-cluster1" -rack="rack-22" -zone="us-est-coast" -gocrane.io/cpu-qos="123" diff --git a/initializer/qos-checking.sh b/initializer/qos-checking.sh deleted file mode 100644 index 2ae9d28aa..000000000 --- a/initializer/qos-checking.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -ready=0 -while [ $ready -eq 0 ]; do - while read line; - do - echo $line; - if [[ $line =~ "gocrane.io/cpu-qos" ]] - then - echo "into if" - ready=1 - break - fi - done < ./podinfo - - if [[ $ready -eq 0 ]] - then - sleep 5 - fi -done - -echo "QOS Initialized" \ No newline at end of file diff --git a/pkg/ensurance/analyzer/analyzer.go b/pkg/ensurance/analyzer/analyzer.go index 64fed0781..561e18a46 100644 --- a/pkg/ensurance/analyzer/analyzer.go +++ b/pkg/ensurance/analyzer/analyzer.go @@ -6,6 +6,8 @@ import ( "strings" "time" + "github.com/gocrane/crane/pkg/ensurance/util" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -490,7 +492,7 @@ func (s *AnomalyAnalyzer) filterPodQOSMatches(pods []*v1.Pod, actionName string) } for _, qos := range podQOSList { for _, pod := range pods { - if !match(pod, qos) { + if !util.MatchPodAndPodQOS(pod, qos) { klog.V(4).Infof("Pod %s/%s does not match PodQOS %s", pod.Namespace, pod.Name, qos.Name) continue diff --git a/pkg/ensurance/analyzer/podqos-fetcher.go b/pkg/ensurance/util/match_qos.go similarity index 85% rename from pkg/ensurance/analyzer/podqos-fetcher.go rename to pkg/ensurance/util/match_qos.go index 3d95f0d82..6a2ef83bf 100644 --- a/pkg/ensurance/analyzer/podqos-fetcher.go +++ b/pkg/ensurance/util/match_qos.go @@ -1,8 +1,9 @@ -package analyzer +package util import ( "fmt" "reflect" + "sort" "strconv" "strings" @@ -69,7 +70,41 @@ func labelMatch(labelSelector metav1.LabelSelector, matchLabels map[string]strin return true } -func match(pod *v1.Pod, podQOS *ensuranceapi.PodQOS) bool { +func sortQOSSlice(qosSlice []*ensuranceapi.PodQOS) { + sort.Slice(qosSlice, func(i, j int) bool { + if len(qosSlice[i].Spec.LabelSelector.MatchLabels) != len(qosSlice[j].Spec.LabelSelector.MatchLabels) { + return len(qosSlice[i].Spec.LabelSelector.MatchLabels) > len(qosSlice[j].Spec.LabelSelector.MatchLabels) + } + + if qosSlice[i].Spec.ScopeSelector == nil && qosSlice[j].Spec.ScopeSelector == nil { + return true + } + + if qosSlice[i].Spec.ScopeSelector == nil { + return false + } + + if qosSlice[j].Spec.ScopeSelector == nil { + return true + } + + return len(qosSlice[i].Spec.ScopeSelector.MatchExpressions) > len(qosSlice[j].Spec.ScopeSelector.MatchExpressions) + }) +} + +func MatchPodAndPodQOSSlice(pod *v1.Pod, qosSlice []*ensuranceapi.PodQOS) (res *ensuranceapi.PodQOS) { + newSlice := make([]*ensuranceapi.PodQOS, len(qosSlice), len(qosSlice)) + copy(newSlice, qosSlice) + sortQOSSlice(newSlice) + for _, qos := range newSlice { + if MatchPodAndPodQOS(pod, qos) { + return qos + } + } + return nil +} + +func MatchPodAndPodQOS(pod *v1.Pod, podQOS *ensuranceapi.PodQOS) bool { if podQOS.Spec.ScopeSelector == nil && podQOS.Spec.LabelSelector.MatchLabels == nil && diff --git a/pkg/providers/grpc/pb/provider.pb.go b/pkg/providers/grpc/pb/provider.pb.go index 3b39c4ac1..684a39d71 100644 --- a/pkg/providers/grpc/pb/provider.pb.go +++ b/pkg/providers/grpc/pb/provider.pb.go @@ -7,10 +7,11 @@ package pb import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" ) const ( diff --git a/pkg/providers/grpc/pb/provider_grpc.pb.go b/pkg/providers/grpc/pb/provider_grpc.pb.go index a23383cf6..e30766d6e 100644 --- a/pkg/providers/grpc/pb/provider_grpc.pb.go +++ b/pkg/providers/grpc/pb/provider_grpc.pb.go @@ -8,6 +8,7 @@ package pb import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/pkg/webhooks/pod/mutating.go b/pkg/webhooks/pod/mutating.go index 16eec8c45..8f48172cc 100644 --- a/pkg/webhooks/pod/mutating.go +++ b/pkg/webhooks/pod/mutating.go @@ -4,17 +4,29 @@ import ( "context" "fmt" - corev1 "k8s.io/api/core/v1" + "github.com/gocrane/crane/pkg/ensurance/util" + "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" + "github.com/gocrane/api/ensurance/v1alpha1" "github.com/gocrane/crane/pkg/ensurance/config" ) type MutatingAdmission struct { - Config *config.QOSConfig + Config *config.QOSConfig + listPodQOS func() ([]*v1alpha1.PodQOS, error) +} + +func NewMutatingAdmission(config *config.QOSConfig, listPodQOS func() ([]*v1alpha1.PodQOS, error)) *MutatingAdmission { + return &MutatingAdmission{ + Config: config, + listPodQOS: listPodQOS, + } } // Default implements webhook.Defaulter so a webhook will be registered for the type @@ -23,7 +35,8 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err if !ok { return fmt.Errorf("expected a Pod but got a %T", obj) } - klog.Infof("Into Pod injection %s/%s", pod.Namespace, pod.Name) + + klog.Infof("mutating started for pod %s/%s", pod.Namespace, pod.Name) if m.Config == nil || !m.Config.QOSInitializer.Enable { return nil @@ -38,17 +51,54 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err return err } - if ls.Matches(labels.Set(pod.Labels)) { - if m.Config.QOSInitializer.InitContainerTemplate != nil { - pod.Spec.InitContainers = append(pod.Spec.InitContainers, *m.Config.QOSInitializer.InitContainerTemplate) + if !ls.Matches(labels.Set(pod.Labels)) { + klog.Infof("injection skipped: webhook is not interested in the pod") + return nil + } + + qosSlice, err := m.listPodQOS() + if err != nil { + return errors.WithMessage(err, "list PodQOS failed") + } + + /**************************************************************** + * Check whether the pod has a low CPUPriority (CPUPriority > 0) + ****************************************************************/ + qos := util.MatchPodAndPodQOSSlice(pod, qosSlice) + if qos == nil { + klog.Infof("injection skipped: no podqos matched") + return nil + } + + if qos.Spec.ResourceQOS.CPUQOS == nil || + qos.Spec.ResourceQOS.CPUQOS.CPUPriority == nil || + *qos.Spec.ResourceQOS.CPUQOS.CPUPriority == 0 { + klog.Infof("injection skipped: not a low CPUPriority pod, qos %s", qos.Name) + return nil + } + for _, container := range pod.Spec.InitContainers { + if container.Name == m.Config.QOSInitializer.InitContainerTemplate.Name { + klog.Infof("injection skipped: pod has initializerContainer already") + return nil } + } - if m.Config.QOSInitializer.VolumeTemplate != nil { - pod.Spec.Volumes = append(pod.Spec.Volumes, *m.Config.QOSInitializer.VolumeTemplate) + for _, volume := range pod.Spec.Volumes { + if volume.Name == m.Config.QOSInitializer.VolumeTemplate.Name { + klog.Infof("injection skipped: pod has initializerVolume already") + return nil } + } - klog.Infof("Injected QOSInitializer for Pod %s/%s", pod.Namespace, pod.Name) + if m.Config.QOSInitializer.InitContainerTemplate != nil { + pod.Spec.InitContainers = append(pod.Spec.InitContainers, *m.Config.QOSInitializer.InitContainerTemplate) } + if m.Config.QOSInitializer.VolumeTemplate != nil { + pod.Spec.Volumes = append(pod.Spec.Volumes, *m.Config.QOSInitializer.VolumeTemplate) + } + + klog.Infof("mutating completed for pod %s/%s", pod.Namespace, pod.Name) + return nil } diff --git a/pkg/webhooks/pod/mutating_test.go b/pkg/webhooks/pod/mutating_test.go index 17c478334..8d2d2c4cd 100644 --- a/pkg/webhooks/pod/mutating_test.go +++ b/pkg/webhooks/pod/mutating_test.go @@ -4,6 +4,10 @@ import ( "context" "testing" + "github.com/gocrane/api/ensurance/v1alpha1" + "github.com/stretchr/testify/assert" + "k8s.io/utils/pointer" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" @@ -20,22 +24,80 @@ func TestDefaultingPodQOSInitializer(t *testing.T) { t.Errorf("unmarshal config failed:%v", err) } m := MutatingAdmission{ - Config: config, + Config: config, + listPodQOS: MockListPodQOSFunc, } - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1", - Labels: map[string]string{ - "app": "nginx", + type Case struct { + Pod *v1.Pod + Inject bool + } + + for _, tc := range []Case{ + {Pod: MockPod("offline", "offline", "enable", "app", "nginx"), Inject: true}, + {Pod: MockPod("offline-not-interested", "offline", "enable"), Inject: false}, + {Pod: MockPod("online", "offline", "disable", "app", "nginx"), Inject: false}, + {Pod: MockPod("online-not-interested", "offline", "disable"), Inject: false}, + {Pod: MockPod("default"), Inject: false}, + } { + assert.NoError(t, m.Default(context.Background(), tc.Pod)) + t.Log(tc.Pod.Name) + assert.Equal(t, len(tc.Pod.Spec.InitContainers) == 1, tc.Inject) + assert.Equal(t, len(tc.Pod.Spec.Volumes) == 1, tc.Inject) + } +} + +func MockListPodQOSFunc() ([]*v1alpha1.PodQOS, error) { + return []*v1alpha1.PodQOS{ + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: v1alpha1.PodQOSSpec{ + LabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"offline": "enable"}, + }, + ResourceQOS: v1alpha1.ResourceQOS{ + CPUQOS: &v1alpha1.CPUQOS{ + CPUPriority: pointer.Int32(7), + }, + }, + }, + }, { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: v1alpha1.PodQOSSpec{ + LabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"offline": "disable"}, + }, + ResourceQOS: v1alpha1.ResourceQOS{ + CPUQOS: &v1alpha1.CPUQOS{ + CPUPriority: pointer.Int32(0), + }, + }, + }, + }, { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: v1alpha1.PodQOSSpec{ + ResourceQOS: v1alpha1.ResourceQOS{ + CPUQOS: &v1alpha1.CPUQOS{ + CPUPriority: pointer.Int32(7), + }, + }, }, }, + }, nil +} + +func MockPod(name string, labels ...string) *v1.Pod { + labelmap := map[string]string{} + for i := 0; i < len(labels)-1; i += 2 { + labelmap[labels[i]] = labels[i+1] } - err = m.Default(context.TODO(), pod) - if err != nil { - t.Fatalf("inject pod failed: %v", err) - } - if len(pod.Spec.InitContainers) == 0 { - t.Fatalf("should inject containers") + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: labelmap, + }, } } diff --git a/pkg/webhooks/webhook.go b/pkg/webhooks/webhook.go index fdea80860..977f9f738 100644 --- a/pkg/webhooks/webhook.go +++ b/pkg/webhooks/webhook.go @@ -17,14 +17,15 @@ limitations under the License. package webhooks import ( - corev1 "k8s.io/api/core/v1" - "k8s.io/klog/v2" - ctrl "sigs.k8s.io/controller-runtime" + "context" analysisapi "github.com/gocrane/api/analysis/v1alpha1" autoscalingapi "github.com/gocrane/api/autoscaling/v1alpha1" ensuranceapi "github.com/gocrane/api/ensurance/v1alpha1" predictionapi "github.com/gocrane/api/prediction/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" "github.com/gocrane/crane/pkg/ensurance/config" "github.com/gocrane/crane/pkg/webhooks/autoscaling" @@ -109,12 +110,10 @@ func SetupWebhookWithManager(mgr ctrl.Manager, autoscalingEnabled, nodeResourceE klog.Errorf("Failed to load qos initializer config: %v", err) } - podMutatingAdmission := pod.MutatingAdmission{ - Config: qosConfig, - } + podMutatingAdmission := pod.NewMutatingAdmission(qosConfig, BuildPodQosListFunction(mgr)) err = ctrl.NewWebhookManagedBy(mgr). For(&corev1.Pod{}). - WithDefaulter(&podMutatingAdmission). + WithDefaulter(podMutatingAdmission). Complete() if err != nil { klog.Errorf("Failed to setup qos initializer webhook: %v", err) @@ -124,3 +123,16 @@ func SetupWebhookWithManager(mgr ctrl.Manager, autoscalingEnabled, nodeResourceE return nil } + +func BuildPodQosListFunction(mgr ctrl.Manager) func() ([]*ensuranceapi.PodQOS, error) { + return func() (qosSlice []*ensuranceapi.PodQOS, err error) { + podQOSList := ensuranceapi.PodQOSList{} + if err := mgr.GetCache().List(context.Background(), &podQOSList); err != nil { + return nil, err + } + for _, qos := range podQOSList.Items { + qosSlice = append(qosSlice, qos.DeepCopy()) + } + return qosSlice, err + } +} diff --git a/tools/initializer/resource.yaml b/tools/initializer/resource.yaml index 371d1483d..98085abd3 100644 --- a/tools/initializer/resource.yaml +++ b/tools/initializer/resource.yaml @@ -99,28 +99,36 @@ data: kind: QOSConfig qosInitializer: enable: true - selector: + selector: matchLabels: app: nginx initContainerTemplate: - name: crane-qos-initializer + name: qos-initializer-container image: docker.io/gocrane/qos-init:v0.1.4 imagePullPolicy: IfNotPresent + args: + - "while ! grep -q gocrane.io/cpu-qos /etc/podinfo/annotations; do sleep 1; done; echo cpu qos setting competed;" command: - - sh - - -x - - /qos-checking.sh + - /bin/bash + - -c + resources: + requests: + cpu: 10m + memory: 10Mi + limits: + cpu: 10m + memory: 10Mi volumeMounts: - - name: podinfo + - name: qos-initializer-volume mountPath: /etc/podinfo volumeTemplate: - name: podinfo + name: qos-initializer-volume downwardAPI: items: - path: "annotations" fieldRef: fieldPath: metadata.annotations - + --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration From 7be30c93b0a23f80f9026db0dffc9fae316b59fd Mon Sep 17 00:00:00 2001 From: payall4u Date: Wed, 29 Nov 2023 18:41:10 +0800 Subject: [PATCH 5/6] Fix log level. --- pkg/webhooks/pod/mutating.go | 14 +++++++------- tools/initializer/resource.yaml | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/webhooks/pod/mutating.go b/pkg/webhooks/pod/mutating.go index 8f48172cc..6abd05374 100644 --- a/pkg/webhooks/pod/mutating.go +++ b/pkg/webhooks/pod/mutating.go @@ -36,7 +36,7 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err return fmt.Errorf("expected a Pod but got a %T", obj) } - klog.Infof("mutating started for pod %s/%s", pod.Namespace, pod.Name) + klog.V(2).Infof("Mutating started for pod %s/%s", pod.Namespace, pod.Name) if m.Config == nil || !m.Config.QOSInitializer.Enable { return nil @@ -52,7 +52,7 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err } if !ls.Matches(labels.Set(pod.Labels)) { - klog.Infof("injection skipped: webhook is not interested in the pod") + klog.V(2).Infof("Injection skipped: webhook is not interested in the pod") return nil } @@ -66,26 +66,26 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err ****************************************************************/ qos := util.MatchPodAndPodQOSSlice(pod, qosSlice) if qos == nil { - klog.Infof("injection skipped: no podqos matched") + klog.V(2).Infof("Injection skipped: no podqos matched") return nil } if qos.Spec.ResourceQOS.CPUQOS == nil || qos.Spec.ResourceQOS.CPUQOS.CPUPriority == nil || *qos.Spec.ResourceQOS.CPUQOS.CPUPriority == 0 { - klog.Infof("injection skipped: not a low CPUPriority pod, qos %s", qos.Name) + klog.V(2).Infof("Injection skipped: not a low CPUPriority pod, qos %s", qos.Name) return nil } for _, container := range pod.Spec.InitContainers { if container.Name == m.Config.QOSInitializer.InitContainerTemplate.Name { - klog.Infof("injection skipped: pod has initializerContainer already") + klog.V(2).Infof("Injection skipped: pod has initializerContainer already") return nil } } for _, volume := range pod.Spec.Volumes { if volume.Name == m.Config.QOSInitializer.VolumeTemplate.Name { - klog.Infof("injection skipped: pod has initializerVolume already") + klog.V(2).Infof("Injection skipped: pod has initializerVolume already") return nil } } @@ -98,7 +98,7 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err pod.Spec.Volumes = append(pod.Spec.Volumes, *m.Config.QOSInitializer.VolumeTemplate) } - klog.Infof("mutating completed for pod %s/%s", pod.Namespace, pod.Name) + klog.V(2).Infof("Mutating completed for pod %s/%s", pod.Namespace, pod.Name) return nil } diff --git a/tools/initializer/resource.yaml b/tools/initializer/resource.yaml index 98085abd3..db3061ecb 100644 --- a/tools/initializer/resource.yaml +++ b/tools/initializer/resource.yaml @@ -103,7 +103,7 @@ data: matchLabels: app: nginx initContainerTemplate: - name: qos-initializer-container + name: crane-qos-initializer image: docker.io/gocrane/qos-init:v0.1.4 imagePullPolicy: IfNotPresent args: @@ -119,10 +119,10 @@ data: cpu: 10m memory: 10Mi volumeMounts: - - name: qos-initializer-volume + - name: crane-qos-initializer-volume mountPath: /etc/podinfo volumeTemplate: - name: qos-initializer-volume + name: crane-qos-initializer-volume downwardAPI: items: - path: "annotations" From 008474b0094d12b860a04154ef671660c41c4e73 Mon Sep 17 00:00:00 2001 From: payall4u Date: Wed, 29 Nov 2023 20:23:44 +0800 Subject: [PATCH 6/6] Add nil check and add unit test --- pkg/ensurance/analyzer/analyzer.go | 3 +-- pkg/webhooks/pod/mutating.go | 20 +++++++++------ pkg/webhooks/pod/mutating_test.go | 40 +++++++++++++++++++++++------- 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/pkg/ensurance/analyzer/analyzer.go b/pkg/ensurance/analyzer/analyzer.go index 561e18a46..55aad6ffe 100644 --- a/pkg/ensurance/analyzer/analyzer.go +++ b/pkg/ensurance/analyzer/analyzer.go @@ -6,8 +6,6 @@ import ( "strings" "time" - "github.com/gocrane/crane/pkg/ensurance/util" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -29,6 +27,7 @@ import ( ecache "github.com/gocrane/crane/pkg/ensurance/cache" "github.com/gocrane/crane/pkg/ensurance/executor" "github.com/gocrane/crane/pkg/ensurance/executor/podinfo" + "github.com/gocrane/crane/pkg/ensurance/util" "github.com/gocrane/crane/pkg/known" "github.com/gocrane/crane/pkg/metrics" "github.com/gocrane/crane/pkg/utils" diff --git a/pkg/webhooks/pod/mutating.go b/pkg/webhooks/pod/mutating.go index 6abd05374..89bd80222 100644 --- a/pkg/webhooks/pod/mutating.go +++ b/pkg/webhooks/pod/mutating.go @@ -4,17 +4,17 @@ import ( "context" "fmt" - "github.com/gocrane/crane/pkg/ensurance/util" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" "github.com/gocrane/api/ensurance/v1alpha1" + "github.com/gocrane/crane/pkg/ensurance/config" + "github.com/gocrane/crane/pkg/ensurance/util" ) type MutatingAdmission struct { @@ -38,11 +38,7 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err klog.V(2).Infof("Mutating started for pod %s/%s", pod.Namespace, pod.Name) - if m.Config == nil || !m.Config.QOSInitializer.Enable { - return nil - } - - if pod.Labels == nil { + if !m.available() { return nil } @@ -76,6 +72,7 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err klog.V(2).Infof("Injection skipped: not a low CPUPriority pod, qos %s", qos.Name) return nil } + for _, container := range pod.Spec.InitContainers { if container.Name == m.Config.QOSInitializer.InitContainerTemplate.Name { klog.V(2).Infof("Injection skipped: pod has initializerContainer already") @@ -102,3 +99,10 @@ func (m *MutatingAdmission) Default(ctx context.Context, obj runtime.Object) err return nil } + +func (m *MutatingAdmission) available() bool { + return m.Config != nil && + m.Config.QOSInitializer.Enable && + m.Config.QOSInitializer.InitContainerTemplate != nil && + m.Config.QOSInitializer.VolumeTemplate != nil +} diff --git a/pkg/webhooks/pod/mutating_test.go b/pkg/webhooks/pod/mutating_test.go index 8d2d2c4cd..e060f1ce5 100644 --- a/pkg/webhooks/pod/mutating_test.go +++ b/pkg/webhooks/pod/mutating_test.go @@ -4,14 +4,14 @@ import ( "context" "testing" - "github.com/gocrane/api/ensurance/v1alpha1" "github.com/stretchr/testify/assert" - "k8s.io/utils/pointer" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" "sigs.k8s.io/yaml" + "github.com/gocrane/api/ensurance/v1alpha1" + "github.com/gocrane/crane/pkg/ensurance/config" ) @@ -47,6 +47,21 @@ func TestDefaultingPodQOSInitializer(t *testing.T) { } } +func TestPrecheck(t *testing.T) { + configYaml := "apiVersion: ensurance.crane.io/v1alpha1\nkind: QOSConfig\nqosInitializer:\n enable: true\n selector: \n matchLabels:\n app: nginx\n" + + config := &config.QOSConfig{} + err := yaml.Unmarshal([]byte(configYaml), config) + if err != nil { + t.Errorf("unmarshal config failed:%v", err) + } + m := MutatingAdmission{ + Config: config, + listPodQOS: MockListPodQOSFunc, + } + assert.False(t, m.available()) +} + func MockListPodQOSFunc() ([]*v1alpha1.PodQOS, error) { return []*v1alpha1.PodQOS{ { @@ -90,14 +105,21 @@ func MockListPodQOSFunc() ([]*v1alpha1.PodQOS, error) { } func MockPod(name string, labels ...string) *v1.Pod { - labelmap := map[string]string{} - for i := 0; i < len(labels)-1; i += 2 { - labelmap[labels[i]] = labels[i+1] - } - return &v1.Pod{ + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Labels: labelmap, + Labels: nil, }, } + + if len(labels) < 2 { + return pod + } + + labelmap := map[string]string{} + for i := 0; i < len(labels)-1; i += 2 { + labelmap[labels[i]] = labels[i+1] + } + pod.Labels = labelmap + return pod }