From a617750d0f82ad14375258f031c8a1396a642b65 Mon Sep 17 00:00:00 2001
From: Tobias Giese <tgiese@nvidia.com>
Date: Tue, 29 Oct 2024 23:37:31 +0100
Subject: [PATCH 1/2] feat: Remove kubectl binary from container and apply CRDs
with client-go
Due too multiple CVEs in the kubectl binary we remove the dependency to it.
To be able to update our CRDs we introduce a custom Go binary that uses client-go
to apply our CRDs and also from our dependencies, like NFD and SRIOV.
Additionally we remove the scale-down Helm hook as it is no longer needed.
Signed-off-by: Tobias Giese <tgiese@nvidia.com>
---
Dockerfile | 53 +++--
cmd/apply-crds/go.mod | 72 +++++++
cmd/apply-crds/go.sum | 194 ++++++++++++++++++
cmd/apply-crds/main.go | 177 ++++++++++++++++
cmd/apply-crds/main_test.go | 80 ++++++++
cmd/apply-crds/suite_test.go | 65 ++++++
cmd/apply-crds/test-files/test-crds.yaml | 47 +++++
.../test-files/updated-test-crds.yaml | 51 +++++
.../templates/scale-down-hook.yaml | 94 ---------
.../templates/upgrade-crd-hook.yaml | 11 +-
10 files changed, 726 insertions(+), 118 deletions(-)
create mode 100644 cmd/apply-crds/go.mod
create mode 100644 cmd/apply-crds/go.sum
create mode 100644 cmd/apply-crds/main.go
create mode 100644 cmd/apply-crds/main_test.go
create mode 100644 cmd/apply-crds/suite_test.go
create mode 100644 cmd/apply-crds/test-files/test-crds.yaml
create mode 100644 cmd/apply-crds/test-files/updated-test-crds.yaml
delete mode 100644 deployment/network-operator/templates/scale-down-hook.yaml
diff --git a/Dockerfile b/Dockerfile
index 041d576e2..4891132ac 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,16 +16,9 @@
ARG ARCH
# Build the manager binary
-FROM golang:1.23@sha256:ad5c126b5cf501a8caef751a243bb717ec204ab1aa56dc41dc11be089fafcb4f AS builder
+FROM golang:1.23@sha256:ad5c126b5cf501a8caef751a243bb717ec204ab1aa56dc41dc11be089fafcb4f AS manager-builder
WORKDIR /workspace
-# Add kubectl tool
-# Using the $ARCH in the name of the binary here ensures we don't get any cross-arch caching after this binary is downloaded.
-ARG ARCH
-# kubectl latest version can be retrieved by curl -L -s https://dl.k8s.io/release/stable.txt
-ARG KUBECTL_VERSION=v1.31.1
-RUN curl -L "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl" -o kubectl-${ARCH} && \
- chmod +x ./kubectl-${ARCH}
# Copy the Go Modules manifests
COPY go.mod go.mod
@@ -38,30 +31,54 @@ RUN --mount=type=cache,target=/go/pkg/mod \
# Copy the go source
COPY ./ ./
+# Build
+ARG ARCH
+ARG LDFLAGS
+ARG GCFLAGS
+RUN --mount=type=cache,target=/go/pkg/mod \
+ --mount=type=cache,target=/root/.cache/go-build \
+ CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -ldflags="${LDFLAGS}" -gcflags="${GCFLAGS}" -o manager main.go
+
+# Build the apply-crds binary
+FROM golang:1.23@sha256:ad5c126b5cf501a8caef751a243bb717ec204ab1aa56dc41dc11be089fafcb4f AS apply-crds-builder
+
+WORKDIR /workspace
+
+# Copy the Go Modules manifests
+COPY cmd/apply-crds/go.mod go.mod
+COPY cmd/apply-crds/go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN --mount=type=cache,target=/go/pkg/mod \
+ go mod download -x
+
+# Copy the go source
+COPY cmd/apply-crds/ ./
+COPY deployment/network-operator/ ./network-operator-chart/
+
# copy CRDs from helm charts
RUN mkdir crds && \
- cp -r deployment/network-operator/crds /workspace/crds/network-operator/ && \
- cp -r deployment/network-operator/charts/sriov-network-operator/crds /workspace/crds/sriov-network-operator/ && \
- cp -r deployment/network-operator/charts/node-feature-discovery/crds /workspace/crds/node-feature-discovery/
+ cp -r network-operator-chart/crds /workspace/crds/network-operator/ && \
+ cp -r network-operator-chart/charts/sriov-network-operator/crds /workspace/crds/sriov-network-operator/ && \
+ cp -r network-operator-chart/charts/node-feature-discovery/crds /workspace/crds/node-feature-discovery/
# Build
+ARG ARCH
ARG LDFLAGS
ARG GCFLAGS
RUN --mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
- CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -ldflags="${LDFLAGS}" -gcflags="${GCFLAGS}" -o manager main.go
+ CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -ldflags="${LDFLAGS}" -gcflags="${GCFLAGS}" -o apply-crds main.go
FROM --platform=linux/${ARCH} registry.access.redhat.com/ubi8-micro:8.10
-ARG ARCH
-
WORKDIR /
-COPY --from=builder /workspace/manager .
-COPY --from=builder /workspace/kubectl-${ARCH} /usr/local/bin/kubectl
-COPY --from=builder /workspace/crds /crds
+COPY --from=manager-builder /workspace/manager .
+COPY --from=apply-crds-builder /workspace/apply-crds .
+COPY --from=apply-crds-builder /workspace/crds /crds
# Default Certificates are missing in micro-ubi. These are need to fetch DOCA drivers image tags
-COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
+COPY --from=manager-builder /etc/ssl/certs/ca-certificates.crt /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
COPY /webhook-schemas /webhook-schemas
COPY manifests/ manifests/
USER 65532:65532
diff --git a/cmd/apply-crds/go.mod b/cmd/apply-crds/go.mod
new file mode 100644
index 000000000..bb3fda8e5
--- /dev/null
+++ b/cmd/apply-crds/go.mod
@@ -0,0 +1,72 @@
+module github.com/mellanox/network-operator/cmd/apply-crds
+
+go 1.23.2
+
+require (
+ github.com/onsi/ginkgo/v2 v2.19.0
+ github.com/onsi/gomega v1.33.1
+ k8s.io/apiextensions-apiserver v0.31.2
+ k8s.io/apimachinery v0.31.2
+ k8s.io/client-go v0.31.2
+ sigs.k8s.io/controller-runtime v0.19.1
+)
+
+require (
+ github.com/beorn7/perks v1.0.1 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
+ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+ github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+ github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+ github.com/fsnotify/fsnotify v1.7.0 // indirect
+ github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+ github.com/go-logr/logr v1.4.2 // indirect
+ github.com/go-logr/zapr v1.3.0 // indirect
+ github.com/go-openapi/jsonpointer v0.19.6 // indirect
+ github.com/go-openapi/jsonreference v0.20.2 // indirect
+ github.com/go-openapi/swag v0.22.4 // indirect
+ github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+ github.com/golang/protobuf v1.5.4 // indirect
+ github.com/google/gnostic-models v0.6.8 // indirect
+ github.com/google/go-cmp v0.6.0 // indirect
+ github.com/google/gofuzz v1.2.0 // indirect
+ github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect
+ github.com/google/uuid v1.6.0 // indirect
+ github.com/imdario/mergo v0.3.6 // indirect
+ github.com/josharian/intern v1.0.0 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
+ github.com/mailru/easyjson v0.7.7 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
+ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+ github.com/pkg/errors v0.9.1 // indirect
+ github.com/prometheus/client_golang v1.19.1 // indirect
+ github.com/prometheus/client_model v0.6.1 // indirect
+ github.com/prometheus/common v0.55.0 // indirect
+ github.com/prometheus/procfs v0.15.1 // indirect
+ github.com/spf13/pflag v1.0.5 // indirect
+ github.com/x448/float16 v0.8.4 // indirect
+ go.uber.org/multierr v1.11.0 // indirect
+ go.uber.org/zap v1.26.0 // indirect
+ golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
+ golang.org/x/net v0.26.0 // indirect
+ golang.org/x/oauth2 v0.21.0 // indirect
+ golang.org/x/sys v0.21.0 // indirect
+ golang.org/x/term v0.21.0 // indirect
+ golang.org/x/text v0.16.0 // indirect
+ golang.org/x/time v0.3.0 // indirect
+ golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
+ gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+ google.golang.org/protobuf v1.34.2 // indirect
+ gopkg.in/inf.v0 v0.9.1 // indirect
+ gopkg.in/yaml.v2 v2.4.0 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
+ k8s.io/api v0.31.2 // indirect
+ k8s.io/klog/v2 v2.130.1 // indirect
+ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+ k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
+ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+ sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+ sigs.k8s.io/yaml v1.4.0 // indirect
+)
diff --git a/cmd/apply-crds/go.sum b/cmd/apply-crds/go.sum
new file mode 100644
index 000000000..78554f1fe
--- /dev/null
+++ b/cmd/apply-crds/go.sum
@@ -0,0 +1,194 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
+github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
+github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
+github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
+github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
+github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
+github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
+github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU=
+github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
+github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM=
+github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
+github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA=
+github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To=
+github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk=
+github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
+github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
+go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU=
+golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
+golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
+gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
+gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0=
+k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk=
+k8s.io/apiextensions-apiserver v0.31.2 h1:W8EwUb8+WXBLu56ser5IudT2cOho0gAKeTOnywBLxd0=
+k8s.io/apiextensions-apiserver v0.31.2/go.mod h1:i+Geh+nGCJEGiCGR3MlBDkS7koHIIKWVfWeRFiOsUcM=
+k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw=
+k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc=
+k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
+k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
+k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/controller-runtime v0.19.1 h1:Son+Q40+Be3QWb+niBXAg2vFiYWolDjjRfO8hn/cxOk=
+sigs.k8s.io/controller-runtime v0.19.1/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
diff --git a/cmd/apply-crds/main.go b/cmd/apply-crds/main.go
new file mode 100644
index 000000000..3e769b3d9
--- /dev/null
+++ b/cmd/apply-crds/main.go
@@ -0,0 +1,177 @@
+/*
+2024 NVIDIA CORPORATION & AFFILIATES
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+
+ apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+ "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/wait"
+ "k8s.io/apimachinery/pkg/util/yaml"
+ "k8s.io/client-go/util/retry"
+ ctrl "sigs.k8s.io/controller-runtime"
+)
+
+type StringList []string
+
+func (s *StringList) String() string {
+ return strings.Join(*s, ", ")
+}
+
+func (s *StringList) Set(value string) error {
+ *s = append(*s, value)
+ return nil
+}
+
+var (
+ crdsDir StringList
+)
+
+func initFlags() {
+ flag.Var(&crdsDir, "crds-dir", "Path to the directory containing the CRD manifests")
+ flag.Parse()
+
+ if len(crdsDir) == 0 {
+ log.Fatalf("CRDs directory is required")
+ }
+
+ for _, crdDir := range crdsDir {
+ if _, err := os.Stat(crdDir); os.IsNotExist(err) {
+ log.Fatalf("CRDs directory %s does not exist", crdsDir)
+ }
+ }
+}
+
+func main() {
+ ctx := context.Background()
+
+ initFlags()
+
+ config, err := ctrl.GetConfig()
+ if err != nil {
+ log.Fatalf("Failed to get Kubernetes config: %v", err)
+ }
+
+ client, err := clientset.NewForConfig(config)
+ if err != nil {
+ log.Fatalf("Failed to create API extensions client: %v", err)
+ }
+
+ if err := walkCrdsDir(ctx, client); err != nil {
+ log.Fatalf("Failed to apply CRDs: %v", err)
+ }
+}
+
+// walkCrdsDir walks the CRDs directory and applies each YAML file.
+func walkCrdsDir(ctx context.Context, client *clientset.Clientset) error {
+ for _, crdDir := range crdsDir {
+ // Walk the directory recursively and apply each YAML file.
+ err := filepath.Walk(crdDir, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() || filepath.Ext(path) != ".yaml" {
+ return nil
+ }
+
+ log.Printf("Apply CRDs from file: %s", path)
+ if err := applyCRDsFromFile(ctx, client, path); err != nil {
+ return fmt.Errorf("apply CRD %s: %v", path, err)
+ }
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("walk the path %s: %v", crdsDir, err)
+ }
+ }
+ return nil
+}
+
+// applyCRDsFromFile reads a YAML file, splits it into documents, and applies each CRD to the cluster.
+func applyCRDsFromFile(ctx context.Context, client *clientset.Clientset, filePath string) error {
+ file, err := os.Open(filePath)
+ if err != nil {
+ return fmt.Errorf("open file %q: %v", filePath, err)
+ }
+ defer file.Close()
+
+ // Create a decoder that reads multiple YAML documents.
+ decoder := yaml.NewYAMLOrJSONDecoder(file, 4096)
+ var crdsToApply []*apiextensionsv1.CustomResourceDefinition
+ for {
+ crd := &apiextensionsv1.CustomResourceDefinition{}
+ if err := decoder.Decode(crd); err != nil {
+ if err == io.EOF {
+ break
+ }
+ return fmt.Errorf("decode YAML: %v", err)
+ }
+ if crd.GetObjectKind().GroupVersionKind().Kind != "CustomResourceDefinition" {
+ log.Printf("Skipping non-CRD object %s", crd.GetName())
+ continue
+ }
+ crdsToApply = append(crdsToApply, crd)
+ }
+
+ // Apply each CRD separately.
+ for _, crd := range crdsToApply {
+ err := wait.ExponentialBackoffWithContext(ctx, retry.DefaultBackoff, func(context.Context) (bool, error) {
+ if err := applyCRD(ctx, client, crd); err != nil {
+ return false, nil
+ }
+ return true, nil
+ })
+ if err != nil {
+ return fmt.Errorf("apply CRD %s: %v", crd.Name, err)
+ }
+ }
+ return nil
+}
+
+// applyCRD creates or updates the CRD.
+func applyCRD(ctx context.Context, client *clientset.Clientset, crd *apiextensionsv1.CustomResourceDefinition) error {
+ crdClient := client.ApiextensionsV1().CustomResourceDefinitions()
+
+ // Check if CRD already exists in cluster and create if not found.
+ curCRD, err := crdClient.Get(ctx, crd.Name, metav1.GetOptions{})
+ if apierrors.IsNotFound(err) {
+ log.Printf("Create CRD %s", crd.Name)
+ _, err = crdClient.Create(ctx, crd, metav1.CreateOptions{})
+ if err != nil {
+ return fmt.Errorf("create CRD %s: %v", crd.Name, err)
+ }
+ } else {
+ log.Printf("Update CRD %s", crd.Name)
+ // Set resource version to update an existing CRD.
+ crd.SetResourceVersion(curCRD.GetResourceVersion())
+ _, err = crdClient.Update(ctx, crd, metav1.UpdateOptions{})
+ if err != nil {
+ return fmt.Errorf("update CRD %s: %v", crd.Name, err)
+ }
+ }
+ return nil
+}
diff --git a/cmd/apply-crds/main_test.go b/cmd/apply-crds/main_test.go
new file mode 100644
index 000000000..0cef92f55
--- /dev/null
+++ b/cmd/apply-crds/main_test.go
@@ -0,0 +1,80 @@
+/*
+2024 NVIDIA CORPORATION & AFFILIATES
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+var _ = Describe("CRD Application", func() {
+ var (
+ ctx context.Context
+ )
+
+ BeforeEach(func() {
+ ctx = context.Background()
+ })
+
+ AfterEach(func() {
+ Expect(testClient.ApiextensionsV1().CustomResourceDefinitions().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{})).NotTo(HaveOccurred())
+ })
+
+ Describe("applyCRDsFromFile", func() {
+ It("should apply CRDs multiple times from a valid YAML file", func() {
+ By("applying CRDs")
+ Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed())
+ Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed())
+ Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed())
+ Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed())
+
+ By("verifying CRDs are applied")
+ crds, err := testClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{})
+ Expect(err).NotTo(HaveOccurred())
+ Expect(crds.Items).To(HaveLen(2))
+ })
+
+ It("should update CRDs", func() {
+ By("applying CRDs")
+ Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed())
+
+ By("verifying CRDs do not have spec.foobar")
+ for _, crdName := range []string{"bars.example.com", "foos.example.com"} {
+ crd, err := testClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crdName, metav1.GetOptions{})
+ Expect(err).NotTo(HaveOccurred())
+ props := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties
+ Expect(props).To(HaveKey("spec"))
+ Expect(props["spec"].Properties).NotTo(HaveKey("foobar"))
+ }
+
+ By("updating CRDs")
+ Expect(applyCRDsFromFile(ctx, testClient, "test-files/updated-test-crds.yaml")).To(Succeed())
+
+ By("verifying CRDs are updated")
+ for _, crdName := range []string{"bars.example.com", "foos.example.com"} {
+ crd, err := testClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crdName, metav1.GetOptions{})
+ Expect(err).NotTo(HaveOccurred())
+ props := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties
+ Expect(props["spec"].Properties).To(HaveKey("foobar"))
+ }
+ })
+ })
+})
diff --git a/cmd/apply-crds/suite_test.go b/cmd/apply-crds/suite_test.go
new file mode 100644
index 000000000..22acfd415
--- /dev/null
+++ b/cmd/apply-crds/suite_test.go
@@ -0,0 +1,65 @@
+/*
+2024 NVIDIA CORPORATION & AFFILIATES
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "testing"
+
+ "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+ "sigs.k8s.io/controller-runtime/pkg/envtest"
+ logf "sigs.k8s.io/controller-runtime/pkg/log"
+ "sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+var (
+ testClient *clientset.Clientset
+ testEnv *envtest.Environment
+)
+
+func TestApplyCrds(t *testing.T) {
+ RegisterFailHandler(Fail)
+ RunSpecs(t, "ApplyCrds Suite")
+}
+
+var _ = BeforeSuite(func() {
+ logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+
+ By("bootstrapping test environment")
+ testEnv = &envtest.Environment{}
+
+ cfg, err := testEnv.Start()
+ Expect(err).NotTo(HaveOccurred())
+ Expect(cfg).NotTo(BeNil())
+
+ // create clientset with scheme
+ testClient, err = clientset.NewForConfig(cfg)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(testClient).NotTo(BeNil())
+
+ go func() {
+ defer GinkgoRecover()
+ }()
+})
+
+var _ = AfterSuite(func() {
+ By("tearing down the test environment")
+ err := testEnv.Stop()
+ Expect(err).NotTo(HaveOccurred())
+})
diff --git a/cmd/apply-crds/test-files/test-crds.yaml b/cmd/apply-crds/test-files/test-crds.yaml
new file mode 100644
index 000000000..67a97a772
--- /dev/null
+++ b/cmd/apply-crds/test-files/test-crds.yaml
@@ -0,0 +1,47 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ name: foos.example.com
+spec:
+ group: example.com
+ names:
+ kind: Foo
+ listKind: FooList
+ singular: foo
+ plural: foos
+ scope: Namespaced
+ versions:
+ - name: v1
+ served: true
+ storage: true
+ schema:
+ openAPIV3Schema:
+ type: object
+ properties:
+ spec:
+ type: object
+---
+# non CRD yamls should not be handled and skipped
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ name: bars.example.com
+spec:
+ group: example.com
+ names:
+ kind: Bar
+ listKind: BarList
+ singular: bar
+ plural: bars
+ scope: Namespaced
+ versions:
+ - name: v1
+ served: true
+ storage: true
+ schema:
+ openAPIV3Schema:
+ type: object
+ properties:
+ spec:
+ type: object
diff --git a/cmd/apply-crds/test-files/updated-test-crds.yaml b/cmd/apply-crds/test-files/updated-test-crds.yaml
new file mode 100644
index 000000000..db2829caf
--- /dev/null
+++ b/cmd/apply-crds/test-files/updated-test-crds.yaml
@@ -0,0 +1,51 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ name: foos.example.com
+spec:
+ group: example.com
+ names:
+ kind: Foo
+ listKind: FooList
+ singular: foo
+ plural: foos
+ scope: Namespaced
+ versions:
+ - name: v1
+ served: true
+ storage: true
+ schema:
+ openAPIV3Schema:
+ type: object
+ properties:
+ spec:
+ type: object
+ properties:
+ foobar:
+ type: string
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ name: bars.example.com
+spec:
+ group: example.com
+ names:
+ kind: Bar
+ listKind: BarList
+ singular: bar
+ plural: bars
+ scope: Namespaced
+ versions:
+ - name: v1
+ served: true
+ storage: true
+ schema:
+ openAPIV3Schema:
+ type: object
+ properties:
+ spec:
+ type: object
+ properties:
+ foobar:
+ type: string
diff --git a/deployment/network-operator/templates/scale-down-hook.yaml b/deployment/network-operator/templates/scale-down-hook.yaml
deleted file mode 100644
index d124e77c3..000000000
--- a/deployment/network-operator/templates/scale-down-hook.yaml
+++ /dev/null
@@ -1,94 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: {{ include "network-operator.fullname" . }}-hooks-scale-sa
- annotations:
- helm.sh/hook: pre-install,pre-upgrade
- helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
- helm.sh/hook-weight: "0"
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- name: {{ include "network-operator.fullname" . }}-hooks-scale-role
- annotations:
- helm.sh/hook: pre-install,pre-upgrade
- helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
- helm.sh/hook-weight: "0"
-rules:
- - apiGroups:
- - apps
- resources:
- - deployments/scale
- verbs:
- - "*"
- - apiGroups:
- - ""
- resources:
- - pods
- verbs:
- - "*"
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
- name: {{ include "network-operator.fullname" . }}-hooks-scale-binding
- annotations:
- helm.sh/hook: pre-install,pre-upgrade
- helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
- helm.sh/hook-weight: "0"
-subjects:
- - kind: ServiceAccount
- name: {{ include "network-operator.fullname" . }}-hooks-scale-sa
- namespace: {{ .Release.Namespace }}
-roleRef:
- kind: ClusterRole
- name: {{ include "network-operator.fullname" . }}-hooks-scale-role
- apiGroup: rbac.authorization.k8s.io
----
-apiVersion: batch/v1
-kind: Job
-metadata:
- name: network-operator-scale-down
- namespace: {{ .Release.Namespace }}
- annotations:
- "helm.sh/hook": pre-install,pre-upgrade
- "helm.sh/hook-weight": "1"
- "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
- labels:
- {{- include "network-operator.labels" . | nindent 4 }}
- app.kubernetes.io/component: "network-operator"
-spec:
- template:
- metadata:
- name: network-operator-scale-down
- labels:
- {{- include "network-operator.labels" . | nindent 8 }}
- app.kubernetes.io/component: "network-operator"
- spec:
- {{- with .Values.operator.nodeSelector }}
- nodeSelector:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- with .Values.operator.affinity}}
- affinity:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- {{- with .Values.operator.tolerations }}
- tolerations:
- {{- toYaml . | nindent 8 }}
- {{- end }}
- serviceAccountName: {{ include "network-operator.fullname" . }}-hooks-scale-sa
- imagePullSecrets: {{ include "network-operator.operator.imagePullSecrets" . }}
- containers:
- - name: scale-down
- image: "{{ .Values.operator.repository }}/{{ .Values.operator.image }}:{{ .Values.operator.tag | default .Chart.AppVersion }}"
- imagePullPolicy: IfNotPresent
- command:
- - /bin/sh
- - -c
- - >
- kubectl scale -n {{ .Release.Namespace }} deployment {{ include "network-operator.fullname" . }} --replicas=0 || true;
- kubectl wait -n {{ .Release.Namespace }} pod -l control-plane={{ .Release.Name }}-controller --for=delete --timeout=30s;
- restartPolicy: OnFailure
diff --git a/deployment/network-operator/templates/upgrade-crd-hook.yaml b/deployment/network-operator/templates/upgrade-crd-hook.yaml
index 018aea563..17e86f2ce 100644
--- a/deployment/network-operator/templates/upgrade-crd-hook.yaml
+++ b/deployment/network-operator/templates/upgrade-crd-hook.yaml
@@ -81,15 +81,14 @@ spec:
image: "{{ .Values.operator.repository }}/{{ .Values.operator.image }}:{{ .Values.operator.tag | default .Chart.AppVersion }}"
imagePullPolicy: IfNotPresent
command:
- - /bin/sh
- - -c
- - >
- kubectl apply -f /crds/network-operator;
+ - /apply-crds
+ args:
+ - --crds-dir=/crds/network-operator
{{- if .Values.sriovNetworkOperator.enabled }}
- kubectl apply -f /crds/sriov-network-operator;
+ - --crds-dir=/crds/sriov-network-operator
{{- end }}
{{- if .Values.nfd.enabled }}
- kubectl apply -f /crds/node-feature-discovery;
+ - --crds-dir=/crds/node-feature-discovery
{{- end }}
restartPolicy: OnFailure
{{- end }}
From 29cdc840b15083416e1c0760c1f1e80fd8516c92 Mon Sep 17 00:00:00 2001
From: Tobias Giese <tgiese@nvidia.com>
Date: Thu, 31 Oct 2024 15:08:08 +0100
Subject: [PATCH 2/2] fix: Also update CRDs for the nic-configuration-operator
Signed-off-by: Tobias Giese <tgiese@nvidia.com>
---
Dockerfile | 3 ++-
deployment/network-operator/templates/upgrade-crd-hook.yaml | 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index 4891132ac..6f5ae0c09 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -60,7 +60,8 @@ COPY deployment/network-operator/ ./network-operator-chart/
RUN mkdir crds && \
cp -r network-operator-chart/crds /workspace/crds/network-operator/ && \
cp -r network-operator-chart/charts/sriov-network-operator/crds /workspace/crds/sriov-network-operator/ && \
- cp -r network-operator-chart/charts/node-feature-discovery/crds /workspace/crds/node-feature-discovery/
+ cp -r network-operator-chart/charts/node-feature-discovery/crds /workspace/crds/node-feature-discovery/ && \
+ cp -r network-operator-chart/charts/nic-configuration-operator-chart/crds /workspace/crds/nic-configuration-operator/
# Build
ARG ARCH
diff --git a/deployment/network-operator/templates/upgrade-crd-hook.yaml b/deployment/network-operator/templates/upgrade-crd-hook.yaml
index 17e86f2ce..c12cf13ee 100644
--- a/deployment/network-operator/templates/upgrade-crd-hook.yaml
+++ b/deployment/network-operator/templates/upgrade-crd-hook.yaml
@@ -90,5 +90,8 @@ spec:
{{- if .Values.nfd.enabled }}
- --crds-dir=/crds/node-feature-discovery
{{- end }}
+ {{- if .Values.nicConfigurationOperator.enabled }}
+ - --crds-dir=/crds/nic-configuration-operator
+ {{- end }}
restartPolicy: OnFailure
{{- end }}