Skip to content

Commit f907cd0

Browse files
committed
use kustomize to install all the crds.
1 parent 883c40a commit f907cd0

File tree

5 files changed

+80
-58
lines changed

5 files changed

+80
-58
lines changed

go.mod

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ require (
6565
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
6666
github.com/felixge/httpsnoop v1.0.4 // indirect
6767
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
68+
github.com/go-errors/errors v1.4.2 // indirect
6869
github.com/go-logr/stdr v1.2.2 // indirect
6970
github.com/go-openapi/jsonpointer v0.21.2 // indirect
7071
github.com/go-openapi/jsonreference v0.21.0 // indirect
@@ -92,6 +93,7 @@ require (
9293
github.com/moby/spdystream v0.5.0 // indirect
9394
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
9495
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
96+
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
9597
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
9698
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
9799
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
@@ -100,6 +102,7 @@ require (
100102
github.com/spf13/cobra v1.9.1 // indirect
101103
github.com/stoewer/go-strcase v1.3.0 // indirect
102104
github.com/x448/float16 v0.8.4 // indirect
105+
github.com/xlab/treeprint v1.2.0 // indirect
103106
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
104107
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
105108
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
@@ -132,5 +135,7 @@ require (
132135
k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 // indirect
133136
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
134137
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
138+
sigs.k8s.io/kustomize/api v0.21.0 // indirect
139+
sigs.k8s.io/kustomize/kyaml v0.21.0 // indirect
135140
sigs.k8s.io/randfill v1.0.0 // indirect
136141
)

go.sum

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,8 @@ github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZ
9797
github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk=
9898
github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
9999
github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
100+
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
101+
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
100102
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
101103
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
102104
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -195,6 +197,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
195197
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
196198
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
197199
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
200+
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
201+
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
198202
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
199203
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
200204
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
@@ -251,6 +255,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE
251255
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
252256
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
253257
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
258+
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
254259
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
255260
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
256261
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
@@ -266,6 +271,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
266271
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
267272
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
268273
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
274+
github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
275+
github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
269276
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
270277
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
271278
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
@@ -413,6 +420,10 @@ sigs.k8s.io/gateway-api v1.3.1-0.20251106052652-079e4774d76b h1:CoVExRHGK0xoewqK
413420
sigs.k8s.io/gateway-api v1.3.1-0.20251106052652-079e4774d76b/go.mod h1:eEYVpDGr0WPqR/35ZTBIWWpwKL7uUzOqlT92mmv3fus=
414421
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
415422
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
423+
sigs.k8s.io/kustomize/api v0.21.0 h1:I7nry5p8iDJbuRdYS7ez8MUvw7XVNPcIP5GkzzuXIIQ=
424+
sigs.k8s.io/kustomize/api v0.21.0/go.mod h1:XGVQuR5n2pXKWbzXHweZU683pALGw/AMVO4zU4iS8SE=
425+
sigs.k8s.io/kustomize/kyaml v0.21.0 h1:7mQAf3dUwf0wBerWJd8rXhVcnkk5Tvn/q91cGkaP6HQ=
426+
sigs.k8s.io/kustomize/kyaml v0.21.0/go.mod h1:hmxADesM3yUN2vbA5z1/YTBnzLJ1dajdqpQonwBL1FQ=
416427
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
417428
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
418429
sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E=

test/e2e/epp/e2e_suite_test.go

Lines changed: 4 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -64,14 +64,6 @@ const (
6464
clientManifest = "../../testdata/client.yaml"
6565
// modelServerSecretManifest is the manifest for the model server secret resource.
6666
modelServerSecretManifest = "../../testdata/model-secret.yaml"
67-
// xInferPoolManifest is the manifest for the inference pool CRD with 'inference.networking.x-k8s.io' group.
68-
xInferPoolManifest = "../../../config/crd/bases/inference.networking.x-k8s.io_inferencepools.yaml"
69-
// xInferObjectiveManifest is the manifest for the inference model CRD with 'inference.networking.x-k8s.io' group.
70-
xInferObjectiveManifest = "../../../config/crd/bases/inference.networking.x-k8s.io_inferenceobjectives.yaml"
71-
// xInferenceModelRewritesManifest is the manifest for the inference rewrites CRD with 'inference.networking.x-k8s.io' group.
72-
xInferenceModelRewritesManifest = "../../../config/crd/bases/inference.networking.x-k8s.io_inferencemodelrewrites.yaml"
73-
// inferPoolManifest is the manifest for the inference pool CRD with 'inference.networking.k8s.io' group.
74-
inferPoolManifest = "../../../config/crd/bases/inference.networking.k8s.io_inferencepools.yaml"
7567
// inferExtManifestDefault is the manifest for the default inference extension test resources (single replica).
7668
inferExtManifestDefault = "../../testdata/inferencepool-e2e.yaml"
7769
// inferExtManifestLeaderElection is the manifest for the inference extension test resources with leader election enabled (3 replicas).
@@ -82,6 +74,8 @@ const (
8274
metricsRbacManifest = "../../testdata/metrics-rbac.yaml"
8375
// modelServerManifestFilepathEnvVar is the env var that holds absolute path to the manifest for the model server test resource.
8476
modelServerManifestFilepathEnvVar = "MANIFEST_PATH"
77+
// crdKustomizePath is the kustomize folder path for the required CRDs.
78+
crdKustomizePath = "../../../config/crd/"
8579
)
8680

8781
const e2eLeaderElectionEnabledEnvVar = "E2E_LEADER_ELECTION_ENABLED"
@@ -133,14 +127,7 @@ func setupInfra() {
133127
if strings.Contains(modelServerManifestArray[0], "hf-token") {
134128
createHfSecret(testConfig, modelServerSecretManifest)
135129
}
136-
crds := map[string]string{
137-
"inferencepools.inference.networking.x-k8s.io": xInferPoolManifest,
138-
"inferenceobjectives.inference.networking.x-k8s.io": xInferObjectiveManifest,
139-
"inferencemodelrewrites.inference.networking.x-k8s.io": xInferenceModelRewritesManifest,
140-
"inferencepools.inference.networking.k8s.io": inferPoolManifest,
141-
}
142-
143-
createCRDs(testConfig, crds)
130+
testutils.CreateCrdsFromKustomize(testConfig, crdKustomizePath)
144131

145132
inferExtManifestPath := inferExtManifestDefault
146133
if leaderElectionEnabled {
@@ -201,7 +188,7 @@ func cleanupResources() {
201188
gomega.Expect(testutils.DeleteNamespacedResources(testConfig)).To(gomega.Succeed())
202189
}
203190

204-
func cleanupInferModelResources() {
191+
func cleanupInferObjectiveResources() {
205192
gomega.Expect(testutils.DeleteInferenceObjectiveResources(testConfig)).To(gomega.Succeed())
206193
}
207194

@@ -245,14 +232,6 @@ func getYamlsFromModelServerManifest(modelServerManifestPath string) []string {
245232
return modelServerManifestArray
246233
}
247234

248-
// createCRDs creates the Inference Extension CRDs used for testing.
249-
func createCRDs(testConfig *testutils.TestConfig, crds map[string]string) {
250-
for _, path := range crds {
251-
ginkgo.By("Creating CRD resource from manifest: " + path)
252-
testutils.ApplyYAMLFile(testConfig, path)
253-
}
254-
}
255-
256235
// createClient creates the client pod used for testing from the given filePath.
257236
func createClient(testConfig *testutils.TestConfig, filePath string) {
258237
ginkgo.By("Creating client resources from manifest: " + filePath)

test/e2e/epp/e2e_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ var _ = ginkgo.Describe("InferencePool", func() {
5757

5858
ginkgo.AfterEach(func() {
5959
ginkgo.By("Deleting the InferenceObjective test resource.")
60-
cleanupInferModelResources()
60+
cleanupInferObjectiveResources()
6161
gomega.Eventually(func() error {
6262
err := testConfig.K8sClient.Get(testConfig.Context, types.NamespacedName{Namespace: infObjective.Namespace, Name: infObjective.Name}, infObjective)
6363
if err == nil {

test/utils/utils.go

Lines changed: 59 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ import (
4141
"k8s.io/client-go/tools/remotecommand"
4242
"sigs.k8s.io/controller-runtime/pkg/client"
4343
"sigs.k8s.io/controller-runtime/pkg/client/config"
44+
"sigs.k8s.io/kustomize/api/krusty"
45+
"sigs.k8s.io/kustomize/kyaml/filesys"
4446

4547
v1 "sigs.k8s.io/gateway-api-inference-extension/api/v1"
4648
"sigs.k8s.io/gateway-api-inference-extension/apix/v1alpha2"
@@ -381,44 +383,20 @@ func EventuallyExists(testConfig *TestConfig, getResource func() error) {
381383
}, testConfig.ExistsTimeout, testConfig.Interval).Should(gomega.Succeed())
382384
}
383385

384-
// CreateObjsFromYaml creates K8S objects from yaml and waits for them to be instantiated
385-
func CreateObjsFromYaml(testConfig *TestConfig, docs []string) []string {
386+
func createAndVerifyObjs(testConfig *TestConfig, objs []*unstructured.Unstructured) []string {
386387
objNames := []string{}
387-
388-
// For each doc, decode and create
389-
decoder := serializer.NewCodecFactory(testConfig.Scheme).UniversalDeserializer()
390-
for _, doc := range docs {
391-
trimmed := strings.TrimSpace(doc)
392-
if trimmed == "" {
393-
continue
394-
}
395-
// Decode into a runtime.Object
396-
obj, gvk, decodeErr := decoder.Decode([]byte(trimmed), nil, nil)
397-
gomega.Expect(decodeErr).NotTo(gomega.HaveOccurred(),
398-
"Failed to decode YAML document to a Kubernetes object")
399-
400-
ginkgo.By(fmt.Sprintf("Decoded GVK: %s", gvk))
401-
402-
unstrObj, ok := obj.(*unstructured.Unstructured)
403-
if !ok {
404-
// Fallback if it's a typed object
405-
unstrObj = &unstructured.Unstructured{}
406-
// Convert typed to unstructured
407-
err := testConfig.Scheme.Convert(obj, unstrObj, nil)
408-
gomega.Expect(err).NotTo(gomega.HaveOccurred())
409-
}
410-
388+
for _, unstrObj := range objs {
389+
ginkgo.By(fmt.Sprintf("Processing GVK: %s", unstrObj.GroupVersionKind()))
411390
unstrObj.SetNamespace(testConfig.NsName)
391+
412392
kind := unstrObj.GetKind()
413393
name := unstrObj.GetName()
414394
objNames = append(objNames, kind+"/"+name)
415395

416-
// Create the object
417396
err := testConfig.K8sClient.Create(testConfig.Context, unstrObj, &client.CreateOptions{})
418397
gomega.Expect(err).NotTo(gomega.HaveOccurred(),
419-
"Failed to create object from YAML")
398+
fmt.Sprintf("Failed to create %s %s", kind, name))
420399

421-
// Wait for the created object to exist.
422400
clientObj := getClientObject(kind)
423401
EventuallyExists(testConfig, func() error {
424402
return testConfig.K8sClient.Get(testConfig.Context,
@@ -427,19 +405,67 @@ func CreateObjsFromYaml(testConfig *TestConfig, docs []string) []string {
427405

428406
switch kind {
429407
case "CustomResourceDefinition":
430-
// Wait for the CRD to be established.
431408
CRDEstablished(testConfig, clientObj.(*apiextv1.CustomResourceDefinition))
432409
case "Deployment":
433-
// Wait for the deployment to be available.
434410
DeploymentAvailable(testConfig, clientObj.(*appsv1.Deployment))
435411
case "Pod":
436-
// Wait for the pod to be ready.
437412
PodReady(testConfig, clientObj.(*corev1.Pod))
438413
}
439414
}
440415
return objNames
441416
}
442417

418+
func CreateCrdsFromKustomize(testConfig *TestConfig, kustomizePath string) []string {
419+
ginkgo.By("Running Kustomize build on: " + kustomizePath)
420+
421+
fSys := filesys.MakeFsOnDisk()
422+
opts := krusty.MakeDefaultOptions()
423+
opts.PluginConfig = krusty.MakeDefaultOptions().PluginConfig
424+
k := krusty.MakeKustomizer(opts)
425+
426+
resMap, err := k.Run(fSys, kustomizePath)
427+
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to run kustomize build")
428+
429+
resources := resMap.Resources()
430+
objs := make([]*unstructured.Unstructured, 0, len(resources))
431+
for _, res := range resources {
432+
resMap, err := res.Map()
433+
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to run kustomize get map")
434+
objs = append(objs, &unstructured.Unstructured{Object: resMap})
435+
}
436+
return createAndVerifyObjs(testConfig, objs)
437+
}
438+
439+
// CreateObjsFromYaml creates K8S objects from yaml and waits for them to be instantiated
440+
func CreateObjsFromYaml(testConfig *TestConfig, docs []string) []string {
441+
objs := make([]*unstructured.Unstructured, 0, len(docs))
442+
decoder := serializer.NewCodecFactory(testConfig.Scheme).UniversalDeserializer()
443+
444+
for _, doc := range docs {
445+
trimmed := strings.TrimSpace(doc)
446+
if trimmed == "" {
447+
continue
448+
}
449+
// Decode into a runtime.Object
450+
obj, gvk, decodeErr := decoder.Decode([]byte(trimmed), nil, nil)
451+
gomega.Expect(decodeErr).NotTo(gomega.HaveOccurred(),
452+
"Failed to decode YAML document to a Kubernetes object")
453+
454+
ginkgo.By(fmt.Sprintf("Decoded GVK: %s", gvk))
455+
456+
unstrObj, ok := obj.(*unstructured.Unstructured)
457+
if !ok {
458+
// Fallback if it's a typed object
459+
unstrObj = &unstructured.Unstructured{}
460+
// Convert typed to unstructured
461+
err := testConfig.Scheme.Convert(obj, unstrObj, nil)
462+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
463+
}
464+
objs = append(objs, unstrObj)
465+
}
466+
return createAndVerifyObjs(testConfig, objs)
467+
}
468+
443469
// DeleteObjects deletes set of Kubernetes objects in the form of kind/name
444470
func DeleteObjects(testConfig *TestConfig, kindAndNames []string) {
445471
for _, kindAndName := range kindAndNames {
@@ -462,6 +488,7 @@ func DeleteObjects(testConfig *TestConfig, kindAndNames []string) {
462488
// ApplyYAMLFile reads a file containing YAML (possibly multiple docs)
463489
// and applies each object to the cluster.
464490
func ApplyYAMLFile(testConfig *TestConfig, filePath string) []string {
491+
// return CreateObjsFromKustomize(testConfig, filePath)
465492
// Create the resources from the manifest file
466493
return CreateObjsFromYaml(testConfig, ReadYaml(filePath))
467494
}

0 commit comments

Comments
 (0)