From 6ea4fcc10e801a2c0d2852241a3af448288b1574 Mon Sep 17 00:00:00 2001 From: bobzetian Date: Thu, 11 Dec 2025 06:40:35 +0000 Subject: [PATCH] use kustomize to install all the crds. --- go.mod | 5 ++ go.sum | 11 +++++ test/e2e/epp/e2e_suite_test.go | 29 ++--------- test/e2e/epp/e2e_test.go | 2 +- test/utils/utils.go | 90 ++++++++++++++++++++++------------ 5 files changed, 79 insertions(+), 58 deletions(-) diff --git a/go.mod b/go.mod index 31309e6bc..ac44d7409 100644 --- a/go.mod +++ b/go.mod @@ -65,6 +65,7 @@ require ( github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-errors/errors v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.2 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect @@ -92,6 +93,7 @@ require ( github.com/moby/spdystream v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect @@ -100,6 +102,7 @@ require ( github.com/spf13/cobra v1.9.1 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/xlab/treeprint v1.2.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect @@ -132,5 +135,7 @@ require ( k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/kustomize/api v0.21.0 // indirect + sigs.k8s.io/kustomize/kyaml v0.21.0 // indirect sigs.k8s.io/randfill v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 84f7a4963..1623dc60a 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,8 @@ github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZ github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -195,6 +197,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= @@ -251,6 +255,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -266,6 +271,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -413,6 +420,10 @@ sigs.k8s.io/gateway-api v1.3.1-0.20251106052652-079e4774d76b h1:CoVExRHGK0xoewqK sigs.k8s.io/gateway-api v1.3.1-0.20251106052652-079e4774d76b/go.mod h1:eEYVpDGr0WPqR/35ZTBIWWpwKL7uUzOqlT92mmv3fus= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kustomize/api v0.21.0 h1:I7nry5p8iDJbuRdYS7ez8MUvw7XVNPcIP5GkzzuXIIQ= +sigs.k8s.io/kustomize/api v0.21.0/go.mod h1:XGVQuR5n2pXKWbzXHweZU683pALGw/AMVO4zU4iS8SE= +sigs.k8s.io/kustomize/kyaml v0.21.0 h1:7mQAf3dUwf0wBerWJd8rXhVcnkk5Tvn/q91cGkaP6HQ= +sigs.k8s.io/kustomize/kyaml v0.21.0/go.mod h1:hmxADesM3yUN2vbA5z1/YTBnzLJ1dajdqpQonwBL1FQ= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= diff --git a/test/e2e/epp/e2e_suite_test.go b/test/e2e/epp/e2e_suite_test.go index bb8c0eb6b..d5c680683 100644 --- a/test/e2e/epp/e2e_suite_test.go +++ b/test/e2e/epp/e2e_suite_test.go @@ -64,14 +64,6 @@ const ( clientManifest = "../../testdata/client.yaml" // modelServerSecretManifest is the manifest for the model server secret resource. modelServerSecretManifest = "../../testdata/model-secret.yaml" - // xInferPoolManifest is the manifest for the inference pool CRD with 'inference.networking.x-k8s.io' group. - xInferPoolManifest = "../../../config/crd/bases/inference.networking.x-k8s.io_inferencepools.yaml" - // xInferObjectiveManifest is the manifest for the inference model CRD with 'inference.networking.x-k8s.io' group. - xInferObjectiveManifest = "../../../config/crd/bases/inference.networking.x-k8s.io_inferenceobjectives.yaml" - // xInferenceModelRewritesManifest is the manifest for the inference rewrites CRD with 'inference.networking.x-k8s.io' group. - xInferenceModelRewritesManifest = "../../../config/crd/bases/inference.networking.x-k8s.io_inferencemodelrewrites.yaml" - // inferPoolManifest is the manifest for the inference pool CRD with 'inference.networking.k8s.io' group. - inferPoolManifest = "../../../config/crd/bases/inference.networking.k8s.io_inferencepools.yaml" // inferExtManifestDefault is the manifest for the default inference extension test resources (single replica). inferExtManifestDefault = "../../testdata/inferencepool-e2e.yaml" // inferExtManifestLeaderElection is the manifest for the inference extension test resources with leader election enabled (3 replicas). @@ -82,6 +74,8 @@ const ( metricsRbacManifest = "../../testdata/metrics-rbac.yaml" // modelServerManifestFilepathEnvVar is the env var that holds absolute path to the manifest for the model server test resource. modelServerManifestFilepathEnvVar = "MANIFEST_PATH" + // crdKustomizePath is the kustomize folder path for the required CRDs. + crdKustomizePath = "../../../config/crd/" ) const e2eLeaderElectionEnabledEnvVar = "E2E_LEADER_ELECTION_ENABLED" @@ -133,14 +127,7 @@ func setupInfra() { if strings.Contains(modelServerManifestArray[0], "hf-token") { createHfSecret(testConfig, modelServerSecretManifest) } - crds := map[string]string{ - "inferencepools.inference.networking.x-k8s.io": xInferPoolManifest, - "inferenceobjectives.inference.networking.x-k8s.io": xInferObjectiveManifest, - "inferencemodelrewrites.inference.networking.x-k8s.io": xInferenceModelRewritesManifest, - "inferencepools.inference.networking.k8s.io": inferPoolManifest, - } - - createCRDs(testConfig, crds) + testutils.CreateCrdsFromKustomize(testConfig, crdKustomizePath) inferExtManifestPath := inferExtManifestDefault if leaderElectionEnabled { @@ -201,7 +188,7 @@ func cleanupResources() { gomega.Expect(testutils.DeleteNamespacedResources(testConfig)).To(gomega.Succeed()) } -func cleanupInferModelResources() { +func cleanupInferObjectiveResources() { gomega.Expect(testutils.DeleteInferenceObjectiveResources(testConfig)).To(gomega.Succeed()) } @@ -245,14 +232,6 @@ func getYamlsFromModelServerManifest(modelServerManifestPath string) []string { return modelServerManifestArray } -// createCRDs creates the Inference Extension CRDs used for testing. -func createCRDs(testConfig *testutils.TestConfig, crds map[string]string) { - for _, path := range crds { - ginkgo.By("Creating CRD resource from manifest: " + path) - testutils.ApplyYAMLFile(testConfig, path) - } -} - // createClient creates the client pod used for testing from the given filePath. func createClient(testConfig *testutils.TestConfig, filePath string) { ginkgo.By("Creating client resources from manifest: " + filePath) diff --git a/test/e2e/epp/e2e_test.go b/test/e2e/epp/e2e_test.go index 4d34dfd9b..4fbaed4d9 100644 --- a/test/e2e/epp/e2e_test.go +++ b/test/e2e/epp/e2e_test.go @@ -57,7 +57,7 @@ var _ = ginkgo.Describe("InferencePool", func() { ginkgo.AfterEach(func() { ginkgo.By("Deleting the InferenceObjective test resource.") - cleanupInferModelResources() + cleanupInferObjectiveResources() gomega.Eventually(func() error { err := testConfig.K8sClient.Get(testConfig.Context, types.NamespacedName{Namespace: infObjective.Namespace, Name: infObjective.Name}, infObjective) if err == nil { diff --git a/test/utils/utils.go b/test/utils/utils.go index c2d97696c..d2bc198ce 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -41,6 +41,8 @@ import ( "k8s.io/client-go/tools/remotecommand" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/kustomize/api/krusty" + "sigs.k8s.io/kustomize/kyaml/filesys" v1 "sigs.k8s.io/gateway-api-inference-extension/api/v1" "sigs.k8s.io/gateway-api-inference-extension/apix/v1alpha2" @@ -381,44 +383,20 @@ func EventuallyExists(testConfig *TestConfig, getResource func() error) { }, testConfig.ExistsTimeout, testConfig.Interval).Should(gomega.Succeed()) } -// CreateObjsFromYaml creates K8S objects from yaml and waits for them to be instantiated -func CreateObjsFromYaml(testConfig *TestConfig, docs []string) []string { +func createAndVerifyObjs(testConfig *TestConfig, objs []*unstructured.Unstructured) []string { objNames := []string{} - - // For each doc, decode and create - decoder := serializer.NewCodecFactory(testConfig.Scheme).UniversalDeserializer() - for _, doc := range docs { - trimmed := strings.TrimSpace(doc) - if trimmed == "" { - continue - } - // Decode into a runtime.Object - obj, gvk, decodeErr := decoder.Decode([]byte(trimmed), nil, nil) - gomega.Expect(decodeErr).NotTo(gomega.HaveOccurred(), - "Failed to decode YAML document to a Kubernetes object") - - ginkgo.By(fmt.Sprintf("Decoded GVK: %s", gvk)) - - unstrObj, ok := obj.(*unstructured.Unstructured) - if !ok { - // Fallback if it's a typed object - unstrObj = &unstructured.Unstructured{} - // Convert typed to unstructured - err := testConfig.Scheme.Convert(obj, unstrObj, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - + for _, unstrObj := range objs { + ginkgo.By(fmt.Sprintf("Processing GVK: %s", unstrObj.GroupVersionKind())) unstrObj.SetNamespace(testConfig.NsName) + kind := unstrObj.GetKind() name := unstrObj.GetName() objNames = append(objNames, kind+"/"+name) - // Create the object err := testConfig.K8sClient.Create(testConfig.Context, unstrObj, &client.CreateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred(), - "Failed to create object from YAML") + fmt.Sprintf("Failed to create %s %s", kind, name)) - // Wait for the created object to exist. clientObj := getClientObject(kind) EventuallyExists(testConfig, func() error { return testConfig.K8sClient.Get(testConfig.Context, @@ -427,19 +405,67 @@ func CreateObjsFromYaml(testConfig *TestConfig, docs []string) []string { switch kind { case "CustomResourceDefinition": - // Wait for the CRD to be established. CRDEstablished(testConfig, clientObj.(*apiextv1.CustomResourceDefinition)) case "Deployment": - // Wait for the deployment to be available. DeploymentAvailable(testConfig, clientObj.(*appsv1.Deployment)) case "Pod": - // Wait for the pod to be ready. PodReady(testConfig, clientObj.(*corev1.Pod)) } } return objNames } +func CreateCrdsFromKustomize(testConfig *TestConfig, kustomizePath string) []string { + ginkgo.By("Running Kustomize build on: " + kustomizePath) + + fSys := filesys.MakeFsOnDisk() + opts := krusty.MakeDefaultOptions() + opts.PluginConfig = krusty.MakeDefaultOptions().PluginConfig + k := krusty.MakeKustomizer(opts) + + resMap, err := k.Run(fSys, kustomizePath) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to run kustomize build") + + resources := resMap.Resources() + objs := make([]*unstructured.Unstructured, 0, len(resources)) + for _, res := range resources { + resMap, err := res.Map() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to run kustomize get map") + objs = append(objs, &unstructured.Unstructured{Object: resMap}) + } + return createAndVerifyObjs(testConfig, objs) +} + +// CreateObjsFromYaml creates K8S objects from yaml and waits for them to be instantiated +func CreateObjsFromYaml(testConfig *TestConfig, docs []string) []string { + objs := make([]*unstructured.Unstructured, 0, len(docs)) + decoder := serializer.NewCodecFactory(testConfig.Scheme).UniversalDeserializer() + + for _, doc := range docs { + trimmed := strings.TrimSpace(doc) + if trimmed == "" { + continue + } + // Decode into a runtime.Object + obj, gvk, decodeErr := decoder.Decode([]byte(trimmed), nil, nil) + gomega.Expect(decodeErr).NotTo(gomega.HaveOccurred(), + "Failed to decode YAML document to a Kubernetes object") + + ginkgo.By(fmt.Sprintf("Decoded GVK: %s", gvk)) + + unstrObj, ok := obj.(*unstructured.Unstructured) + if !ok { + // Fallback if it's a typed object + unstrObj = &unstructured.Unstructured{} + // Convert typed to unstructured + err := testConfig.Scheme.Convert(obj, unstrObj, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + objs = append(objs, unstrObj) + } + return createAndVerifyObjs(testConfig, objs) +} + // DeleteObjects deletes set of Kubernetes objects in the form of kind/name func DeleteObjects(testConfig *TestConfig, kindAndNames []string) { for _, kindAndName := range kindAndNames {