diff --git a/.gitignore b/.gitignore index 7da33fc..82d0f0d 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ go.work.sum # build dirs build/ bin/ + +# IDE +.vscode diff --git a/Dockerfile b/Dockerfile index c8c3561..92aa211 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM golang:1.22 AS builder ARG TARGETOS ARG TARGETARCH +ARG GCFLAGS WORKDIR /workspace # Copy sources @@ -9,7 +10,7 @@ COPY go.mod go.mod COPY go.sum go.sum # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer -RUN go mod download +RUN --mount=type=cache,target=/go/pkg/mod/ go mod download # Copy the go source COPY ./ ./ @@ -20,7 +21,7 @@ COPY ./ ./ # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. #RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/maintenance-manager/main.go -RUN make build +RUN --mount=type=cache,target=/go/pkg/mod/ GO_GCFLAGS=${GCFLAGS} make build # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details @@ -30,3 +31,6 @@ COPY --from=builder /workspace/build/manager . USER 65532:65532 ENTRYPOINT ["/manager"] + +LABEL org.opencontainers.image.source=https://github.com/Mellanox/maintenance-operator + diff --git a/Makefile b/Makefile index 00130f1..bd2c605 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,7 @@ TARGETOS ?= $(shell go env GOOS) TARGETARCH ?= $(shell go env GOARCH) GO_BUILD_OPTS ?= CGO_ENABLED=0 GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) GO_LDFLAGS ?= $(VERSION_LDFLAGS) +GO_GCFLAGS ?= # PKGs to test PKGS = $$(go list ./... | grep -v /e2e | grep -v ".*/mocks") @@ -162,7 +163,7 @@ generate-mocks: mockery ## generate mock objects .PHONY: build build: $(BUILDDIR) ## Build manager binary. - $(GO_BUILD_OPTS) go build -ldflags $(GO_LDFLAGS) -o $(BUILDDIR)/manager cmd/maintenance-manager/main.go + $(GO_BUILD_OPTS) go build -ldflags $(GO_LDFLAGS) -gcflags="$(GO_GCFLAGS)" -o $(BUILDDIR)/manager cmd/maintenance-manager/main.go .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. @@ -340,3 +341,62 @@ else OPM = $(shell which opm) endif endif + +SKAFFOLD_VER := v2.12.0 +SKAFFOLD := $(abspath $(LOCALBIN)/skaffold-$(SKAFFOLD_VER)) +.PHONY: skaffold +skaffold: $(SKAFFOLD) ## Download skaffold locally if necessary. +$(SKAFFOLD): | $(LOCALBIN) + @{ \ + set -e;\ + OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ + curl -fsSL https://storage.googleapis.com/skaffold/releases/$(SKAFFOLD_VER)/skaffold-$${OS}-$${ARCH} -o $(SKAFFOLD); \ + chmod +x $(SKAFFOLD);\ + } + +# minikube is used to set-up a local kubernetes cluster for dev work. +MINIKUBE_VER := v1.33.1 +MINIKUBE := $(abspath $(LOCALBIN)/minikube-$(MINIKUBE_VER)) +.PHONY: minikube +minikube: $(MINIKUBE) ## Download minikube locally if necessary. +$(MINIKUBE): | $(LOCALBIN) + @{ \ + set -e;\ + OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ + curl -fsSL https://storage.googleapis.com/minikube/releases/$(MINIKUBE_VER)/minikube-$${OS}-$${ARCH} -o $(MINIKUBE); \ + chmod +x $(MINIKUBE);\ + } + +##@ Dev + +TEST_CLUSTER_NAME = mn-op + +# run $(MINIKUBE) kubectl -p $(TEST_CLUSTER_NAME) -- port-forward --namespace kube-system svc/registry 5000:80 +# to expose in-cluster registry service to the host. +SKAFFOLD_REGISTRY ?= localhost:5000 + +.PHONY: dev-env +dev-env: $(MINIKUBE) ## Create minikube cluster for dev and tests + CLUSTER_NAME=$(TEST_CLUSTER_NAME) MINIKUBE_BIN=$(MINIKUBE) $(CURDIR)/hack/scripts/setup_minikube.sh + +.PHONY: dev-env +dev-env-multinode: $(MINIKUBE) ## Create minikube cluster for dev and tests + CLUSTER_NAME=$(TEST_CLUSTER_NAME) MINIKUBE_BIN=$(MINIKUBE) NUM_NODES=4 USE_MINIKUBE_DOCKER=false $(CURDIR)/hack/scripts/setup_minikube.sh + +.PHONY: clean-dev-env +clean-dev-env: $(MINIKUBE) ## Teardown minikube cluster for dev and tests + $(MINIKUBE) delete -p $(TEST_CLUSTER_NAME) + +.PHONY: dev-operator +dev-operator: $(MINIKUBE) $(SKAFFOLD) ## Deploy maintenance operator controller to dev cluster using skaffold + {\ + eval $$($(MINIKUBE) -p $(TEST_CLUSTER_NAME) docker-env); \ + $(SKAFFOLD) dev -p operator --default-repo=$(SKAFFOLD_REGISTRY) --detect-minikube=false --cleanup=false --trigger=manual; \ + } + +.PHONY: dev-operator-debug +dev-operator-debug: $(MINIKUBE) $(SKAFFOLD) ## Deploy maintenance operator controller to dev cluster using skaffold with remote debug + {\ + eval $$($(MINIKUBE) -p $(TEST_CLUSTER_NAME) docker-env); \ + $(SKAFFOLD) debug -p operator --default-repo=$(SKAFFOLD_REGISTRY) --detect-minikube=false --cleanup=false; \ + } diff --git a/config/debug/kustomization.yaml b/config/debug/kustomization.yaml new file mode 100644 index 0000000..3dc962f --- /dev/null +++ b/config/debug/kustomization.yaml @@ -0,0 +1,11 @@ +# Adds namespace to all resources. +namespace: maintenance-operator-system +namePrefix: maintenance-operator- + +resources: +- ../crd +- ../rbac +- ../manager + +patches: +- path: manager_debug_patch.yaml diff --git a/config/debug/manager_debug_patch.yaml b/config/debug/manager_debug_patch.yaml new file mode 100644 index 0000000..2f840a5 --- /dev/null +++ b/config/debug/manager_debug_patch.yaml @@ -0,0 +1,16 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + securityContext: null + containers: + - name: manager + securityContext: null + livenessProbe: null + readinessProbe: null + resources: null + diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 96bc51b..8f15c40 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: harbor.mellanox.com/cloud-orchestration-dev/adrianc/maintenance-operator + newName: nvidia.com/maintenance-operator newTag: latest diff --git a/hack/scripts/setup_minikube.sh b/hack/scripts/setup_minikube.sh new file mode 100755 index 0000000..73d849e --- /dev/null +++ b/hack/scripts/setup_minikube.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +# 2024 NVIDIA CORPORATION & AFFILIATES +# +# Licensed under the Apache License, Version 2.0 (the License); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset +set -o pipefail +set -o errexit + +if [[ "${TRACE-0}" == "1" ]]; then + set -o xtrace +fi + +CLUSTER_NAME="${CLUSTER_NAME:-"mn-op"}" +MINIKUBE_BIN="${MINIKUBE_BIN:-"unknown"}" +# Docker is used as the minikube driver to enable using hollow nodes. +MINIKUBE_DRIVER="${MINIKUBE_DRIVER:-"unknown"}" +USE_MINIKUBE_DOCKER="${USE_MINIKUBE_DOCKER:-"true"}" +NUM_NODES="${NUM_NODES:-"1"}" +NODE_MEMORY="${NODE_MEMORY:-"8g"}" +NODE_CPUS="${NODE_CPUS:-"4"}" +NODE_DISK="${NODE_DISK:-"100g"}" +MINIKUBE_CNI="${MINIKUBE_CNI:-kindnet}" +MINIKUBE_KUBERNETES_VERSION="${MINIKUBE_KUBERNETES_VERSION:-"v1.30.0"}" + +## Detect the OS. +OS="unknown" +if [[ "${OSTYPE}" == "linux"* ]]; then + OS="linux" +elif [[ "${OSTYPE}" == "darwin"* ]]; then + OS="darwin" +fi + +# Exit if the OS is not supported. +if [[ "$OS" == "unknown" ]]; then + echo "os '$OSTYPE' not supported. Aborting." >&2 + exit 1 +fi + +## Set the driver used for minikube machines. By default this script will select the preferred VM driver for each OS. +## Users can override this using the MINIKUBE_DRIVER env variable. +if [[ "$MINIKUBE_DRIVER" == "unknown" ]]; then + if [[ "${OSTYPE}" == "linux"* ]]; then + MINIKUBE_DRIVER="kvm2" + elif [[ "${OSTYPE}" == "darwin"* ]]; then + MINIKUBE_DRIVER="qemu" + fi +fi + +MINIKUBE_ARGS="${MINIKUBE_ARGS:-"\ + --driver $MINIKUBE_DRIVER \ + --cpus=$NODE_CPUS \ + --memory=$NODE_MEMORY \ + --disk-size=$NODE_DISK \ + --nodes=$NUM_NODES \ + --cni $MINIKUBE_CNI \ + --kubernetes-version $MINIKUBE_KUBERNETES_VERSION \ + --preload=true \ + --cache-images=true \ + --addons registry"}" + +MINIKUBE_EXTRA_ARGS="${MINIKUBE_EXTRA_ARGS:-""}" + +echo "Setting up Minikube cluster..." + +## Exit early if the cluster already exists. +if [[ $($MINIKUBE_BIN status -p $CLUSTER_NAME -f '{{.Name}}') == $CLUSTER_NAME ]]; then + echo "Minikube cluster '$CLUSTER_NAME' found. Skipping cluster set-up" + ## Set environment variables to use the Minikube VM docker build for + if [[ "$USE_MINIKUBE_DOCKER" == "true" ]]; then + echo "Setting environment variables to use $CLUSTER_NAME as docker build environment." + eval $($MINIKUBE_BIN -p $CLUSTER_NAME docker-env) + fi + exit 0 +fi + +$MINIKUBE_BIN start --profile $CLUSTER_NAME $MINIKUBE_ARGS $MINIKUBE_EXTRA_ARGS + +## Set environment variables to use the Minikube VM docker build for +if [[ "$USE_MINIKUBE_DOCKER" == "true" ]]; then + echo "Setting environment variables to use $CLUSTER_NAME as docker build environment." + eval $($MINIKUBE_BIN -p $CLUSTER_NAME docker-env) +fi diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000..d5f071f --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,31 @@ +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: maintenance-operator + +profiles: + - name: operator + build: + local: + useBuildkit: true + artifacts: + - image: nvidia.com/maintenance-operator + runtimeType: go + docker: + dockerfile: Dockerfile + buildArgs: + # Used for debugging. Disable inlining and optimizations in the go compiler. Passed to go build using `-gcflags` + GCFLAGS: "all=-N -l" + manifests: + ## Deploy the manager. + kustomize: + paths: + - config/debug + + portForward: + - resourceType: deployment + resourceName: maintenance-operator-controller-manager + namespace: maintenance-operator-system + port: 56268 + localPort: 8082 + address: 0.0.0.0