Merge pull request #2 from adrianchiris/create-minikube-dev-env

Create minikube dev env

.gitignore

@@ -24,3 +24,6 @@ go.work.sum
 # build dirs
 build/
 bin/
+
+# IDE
+.vscode

Dockerfile

@@ -2,14 +2,15 @@
 FROM golang:1.22 AS builder
 ARG TARGETOS
 ARG TARGETARCH
+ARG GCFLAGS
 
 WORKDIR /workspace
 # Copy sources
 COPY go.mod go.mod
 COPY go.sum go.sum
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod/ go mod download
 
 # Copy the go source
 COPY ./ ./
@@ -20,7 +21,7 @@ COPY ./ ./
 # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
 #RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/maintenance-manager/main.go
-RUN make build
+RUN --mount=type=cache,target=/go/pkg/mod/ GO_GCFLAGS=${GCFLAGS} make build
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
@@ -30,3 +31,6 @@ COPY --from=builder /workspace/build/manager .
 USER 65532:65532
 
 ENTRYPOINT ["/manager"]
+
+LABEL org.opencontainers.image.source=https://github.com/Mellanox/maintenance-operator
+

Makefile

@@ -81,6 +81,7 @@ TARGETOS ?= $(shell go env GOOS)
 TARGETARCH ?= $(shell go env GOARCH)
 GO_BUILD_OPTS ?= CGO_ENABLED=0 GOOS=$(TARGETOS) GOARCH=$(TARGETARCH)
 GO_LDFLAGS ?= $(VERSION_LDFLAGS)
+GO_GCFLAGS ?=
 
 # PKGs to test
 PKGS = $$(go list ./... | grep -v /e2e | grep -v ".*/mocks")
@@ -162,7 +163,7 @@ generate-mocks: mockery ## generate mock objects
 
 .PHONY: build
 build: $(BUILDDIR) ## Build manager binary.
-	$(GO_BUILD_OPTS) go build -ldflags $(GO_LDFLAGS) -o $(BUILDDIR)/manager cmd/maintenance-manager/main.go
+	$(GO_BUILD_OPTS) go build -ldflags $(GO_LDFLAGS) -gcflags="$(GO_GCFLAGS)" -o $(BUILDDIR)/manager cmd/maintenance-manager/main.go
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
@@ -340,3 +341,62 @@ else
 OPM = $(shell which opm)
 endif
 endif
+
+SKAFFOLD_VER := v2.12.0
+SKAFFOLD := $(abspath $(LOCALBIN)/skaffold-$(SKAFFOLD_VER))
+.PHONY: skaffold
+skaffold: $(SKAFFOLD) ## Download skaffold locally if necessary.
+$(SKAFFOLD): | $(LOCALBIN)
+	@{ \
+		set -e;\
+		OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
+		curl -fsSL https://storage.googleapis.com/skaffold/releases/$(SKAFFOLD_VER)/skaffold-$${OS}-$${ARCH} -o $(SKAFFOLD); \
+		chmod +x $(SKAFFOLD);\
+	}
+
+# minikube is used to set-up a local kubernetes cluster for dev work.
+MINIKUBE_VER := v1.33.1
+MINIKUBE := $(abspath $(LOCALBIN)/minikube-$(MINIKUBE_VER))
+.PHONY: minikube
+minikube: $(MINIKUBE) ## Download minikube locally if necessary.
+$(MINIKUBE): | $(LOCALBIN)
+	@{ \
+		set -e;\
+		OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
+		curl -fsSL https://storage.googleapis.com/minikube/releases/$(MINIKUBE_VER)/minikube-$${OS}-$${ARCH} -o $(MINIKUBE); \
+		chmod +x $(MINIKUBE);\
+	}
+
+##@ Dev
+
+TEST_CLUSTER_NAME = mn-op
+
+# run $(MINIKUBE) kubectl -p $(TEST_CLUSTER_NAME) -- port-forward --namespace kube-system svc/registry 5000:80
+# to expose in-cluster registry service to the host.
+SKAFFOLD_REGISTRY ?= localhost:5000
+
+.PHONY: dev-env
+dev-env: $(MINIKUBE) ## Create minikube cluster for dev and tests
+	CLUSTER_NAME=$(TEST_CLUSTER_NAME) MINIKUBE_BIN=$(MINIKUBE) $(CURDIR)/hack/scripts/setup_minikube.sh
+
+.PHONY: dev-env
+dev-env-multinode: $(MINIKUBE) ## Create minikube cluster for dev and tests
+	CLUSTER_NAME=$(TEST_CLUSTER_NAME) MINIKUBE_BIN=$(MINIKUBE) NUM_NODES=4 USE_MINIKUBE_DOCKER=false $(CURDIR)/hack/scripts/setup_minikube.sh
+
+.PHONY: clean-dev-env
+clean-dev-env: $(MINIKUBE) ## Teardown minikube cluster for dev and tests
+	$(MINIKUBE) delete -p $(TEST_CLUSTER_NAME)
+
+.PHONY: dev-operator
+dev-operator: $(MINIKUBE) $(SKAFFOLD) ## Deploy maintenance operator controller to dev cluster using skaffold
+	{\
+		eval $$($(MINIKUBE) -p $(TEST_CLUSTER_NAME) docker-env); \
+		$(SKAFFOLD) dev -p operator --default-repo=$(SKAFFOLD_REGISTRY) --detect-minikube=false --cleanup=false --trigger=manual; \
+	}
+
+.PHONY: dev-operator-debug
+dev-operator-debug: $(MINIKUBE) $(SKAFFOLD) ## Deploy maintenance operator controller to dev cluster using skaffold with remote debug
+	{\
+		eval $$($(MINIKUBE) -p $(TEST_CLUSTER_NAME) docker-env); \
+		$(SKAFFOLD) debug -p operator --default-repo=$(SKAFFOLD_REGISTRY) --detect-minikube=false --cleanup=false; \
+	}

config/debug/kustomization.yaml

@@ -0,0 +1,11 @@
+# Adds namespace to all resources.
+namespace: maintenance-operator-system
+namePrefix: maintenance-operator-
+
+resources:
+- ../crd
+- ../rbac
+- ../manager
+
+patches:
+- path: manager_debug_patch.yaml

config/debug/manager_debug_patch.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      securityContext: null
+      containers:
+      - name: manager
+        securityContext: null
+        livenessProbe: null
+        readinessProbe: null
+        resources: null
+

config/manager/kustomization.yaml

@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: harbor.mellanox.com/cloud-orchestration-dev/adrianc/maintenance-operator
+  newName: nvidia.com/maintenance-operator
   newTag: latest

hack/scripts/setup_minikube.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+#  2024 NVIDIA CORPORATION & AFFILIATES
+#
+#  Licensed under the Apache License, Version 2.0 (the License);
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an AS IS BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+set -o nounset
+set -o pipefail
+set -o errexit
+
+if [[ "${TRACE-0}" == "1" ]]; then
+    set -o xtrace
+fi
+
+CLUSTER_NAME="${CLUSTER_NAME:-"mn-op"}"
+MINIKUBE_BIN="${MINIKUBE_BIN:-"unknown"}"
+# Docker is used as the minikube driver to enable using hollow nodes.
+MINIKUBE_DRIVER="${MINIKUBE_DRIVER:-"unknown"}"
+USE_MINIKUBE_DOCKER="${USE_MINIKUBE_DOCKER:-"true"}"
+NUM_NODES="${NUM_NODES:-"1"}"
+NODE_MEMORY="${NODE_MEMORY:-"8g"}"
+NODE_CPUS="${NODE_CPUS:-"4"}"
+NODE_DISK="${NODE_DISK:-"100g"}"
+MINIKUBE_CNI="${MINIKUBE_CNI:-kindnet}"
+MINIKUBE_KUBERNETES_VERSION="${MINIKUBE_KUBERNETES_VERSION:-"v1.30.0"}"
+
+## Detect the OS.
+OS="unknown"
+if [[ "${OSTYPE}" == "linux"* ]]; then
+  OS="linux"
+elif [[ "${OSTYPE}" == "darwin"* ]]; then
+  OS="darwin"
+fi
+
+# Exit if the OS is not supported.
+if [[ "$OS" == "unknown" ]]; then
+  echo "os '$OSTYPE' not supported. Aborting." >&2
+  exit 1
+fi
+
+## Set the driver used for minikube machines. By default this script will select the preferred VM driver for each OS.
+## Users can override this using the MINIKUBE_DRIVER env variable.
+if [[ "$MINIKUBE_DRIVER" == "unknown" ]]; then
+  if [[ "${OSTYPE}" == "linux"* ]]; then
+    MINIKUBE_DRIVER="kvm2"
+  elif [[ "${OSTYPE}" == "darwin"* ]]; then
+    MINIKUBE_DRIVER="qemu"
+  fi
+fi
+
+MINIKUBE_ARGS="${MINIKUBE_ARGS:-"\
+  --driver $MINIKUBE_DRIVER \
+  --cpus=$NODE_CPUS \
+  --memory=$NODE_MEMORY \
+  --disk-size=$NODE_DISK \
+  --nodes=$NUM_NODES \
+  --cni $MINIKUBE_CNI \
+  --kubernetes-version $MINIKUBE_KUBERNETES_VERSION \
+  --preload=true \
+  --cache-images=true \
+  --addons registry"}"
+
+MINIKUBE_EXTRA_ARGS="${MINIKUBE_EXTRA_ARGS:-""}"
+
+echo "Setting up Minikube cluster..."
+
+## Exit early if the cluster already exists.
+if [[ $($MINIKUBE_BIN status -p $CLUSTER_NAME  -f '{{.Name}}') == $CLUSTER_NAME ]]; then
+  echo "Minikube cluster '$CLUSTER_NAME' found. Skipping cluster set-up"
+  ## Set environment variables to use the Minikube VM docker build for
+  if [[ "$USE_MINIKUBE_DOCKER" == "true" ]]; then
+    echo "Setting environment variables to use $CLUSTER_NAME as docker build environment."
+    eval $($MINIKUBE_BIN -p $CLUSTER_NAME docker-env)
+  fi
+  exit 0
+fi
+
+$MINIKUBE_BIN start --profile $CLUSTER_NAME $MINIKUBE_ARGS $MINIKUBE_EXTRA_ARGS
+
+## Set environment variables to use the Minikube VM docker build for
+if [[ "$USE_MINIKUBE_DOCKER" == "true" ]]; then
+  echo "Setting environment variables to use $CLUSTER_NAME as docker build environment."
+  eval $($MINIKUBE_BIN -p $CLUSTER_NAME docker-env)
+fi

skaffold.yaml

@@ -0,0 +1,31 @@
+apiVersion: skaffold/v4beta11
+kind: Config
+metadata:
+  name: maintenance-operator
+
+profiles:
+  - name: operator
+    build:
+      local:
+        useBuildkit: true
+      artifacts:
+        - image: nvidia.com/maintenance-operator
+          runtimeType: go
+          docker:
+            dockerfile: Dockerfile
+            buildArgs:
+              # Used for debugging. Disable inlining and optimizations in the go compiler. Passed to go build using `-gcflags`
+              GCFLAGS: "all=-N -l"
+    manifests:
+      ## Deploy the manager.
+      kustomize:
+        paths:
+          - config/debug
+
+    portForward:
+      - resourceType: deployment
+        resourceName: maintenance-operator-controller-manager
+        namespace: maintenance-operator-system
+        port: 56268
+        localPort: 8082
+        address: 0.0.0.0