Skip to content

Commit

Permalink
Merge pull request #2 from adrianchiris/create-minikube-dev-env
Browse files Browse the repository at this point in the history
Create minikube dev env
  • Loading branch information
adrianchiris authored Jul 7, 2024
2 parents 5e1cb4f + 50f7d25 commit 1fccbbc
Show file tree
Hide file tree
Showing 8 changed files with 223 additions and 4 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,6 @@ go.work.sum
# build dirs
build/
bin/

# IDE
.vscode
8 changes: 6 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@
FROM golang:1.22 AS builder
ARG TARGETOS
ARG TARGETARCH
ARG GCFLAGS

WORKDIR /workspace
# Copy sources
COPY go.mod go.mod
COPY go.sum go.sum
# cache deps before building and copying source so that we don't need to re-download as much
# and so that source changes don't invalidate our downloaded layer
RUN go mod download
RUN --mount=type=cache,target=/go/pkg/mod/ go mod download

# Copy the go source
COPY ./ ./
Expand All @@ -20,7 +21,7 @@ COPY ./ ./
# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
#RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/maintenance-manager/main.go
RUN make build
RUN --mount=type=cache,target=/go/pkg/mod/ GO_GCFLAGS=${GCFLAGS} make build

# Use distroless as minimal base image to package the manager binary
# Refer to https://github.com/GoogleContainerTools/distroless for more details
Expand All @@ -30,3 +31,6 @@ COPY --from=builder /workspace/build/manager .
USER 65532:65532

ENTRYPOINT ["/manager"]

LABEL org.opencontainers.image.source=https://github.com/Mellanox/maintenance-operator

62 changes: 61 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ TARGETOS ?= $(shell go env GOOS)
TARGETARCH ?= $(shell go env GOARCH)
GO_BUILD_OPTS ?= CGO_ENABLED=0 GOOS=$(TARGETOS) GOARCH=$(TARGETARCH)
GO_LDFLAGS ?= $(VERSION_LDFLAGS)
GO_GCFLAGS ?=

# PKGs to test
PKGS = $$(go list ./... | grep -v /e2e | grep -v ".*/mocks")
Expand Down Expand Up @@ -162,7 +163,7 @@ generate-mocks: mockery ## generate mock objects

.PHONY: build
build: $(BUILDDIR) ## Build manager binary.
$(GO_BUILD_OPTS) go build -ldflags $(GO_LDFLAGS) -o $(BUILDDIR)/manager cmd/maintenance-manager/main.go
$(GO_BUILD_OPTS) go build -ldflags $(GO_LDFLAGS) -gcflags="$(GO_GCFLAGS)" -o $(BUILDDIR)/manager cmd/maintenance-manager/main.go

.PHONY: run
run: manifests generate fmt vet ## Run a controller from your host.
Expand Down Expand Up @@ -340,3 +341,62 @@ else
OPM = $(shell which opm)
endif
endif

SKAFFOLD_VER := v2.12.0
SKAFFOLD := $(abspath $(LOCALBIN)/skaffold-$(SKAFFOLD_VER))
.PHONY: skaffold
skaffold: $(SKAFFOLD) ## Download skaffold locally if necessary.
$(SKAFFOLD): | $(LOCALBIN)
@{ \
set -e;\
OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
curl -fsSL https://storage.googleapis.com/skaffold/releases/$(SKAFFOLD_VER)/skaffold-$${OS}-$${ARCH} -o $(SKAFFOLD); \
chmod +x $(SKAFFOLD);\
}

# minikube is used to set-up a local kubernetes cluster for dev work.
MINIKUBE_VER := v1.33.1
MINIKUBE := $(abspath $(LOCALBIN)/minikube-$(MINIKUBE_VER))
.PHONY: minikube
minikube: $(MINIKUBE) ## Download minikube locally if necessary.
$(MINIKUBE): | $(LOCALBIN)
@{ \
set -e;\
OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
curl -fsSL https://storage.googleapis.com/minikube/releases/$(MINIKUBE_VER)/minikube-$${OS}-$${ARCH} -o $(MINIKUBE); \
chmod +x $(MINIKUBE);\
}

##@ Dev

TEST_CLUSTER_NAME = mn-op

# run $(MINIKUBE) kubectl -p $(TEST_CLUSTER_NAME) -- port-forward --namespace kube-system svc/registry 5000:80
# to expose in-cluster registry service to the host.
SKAFFOLD_REGISTRY ?= localhost:5000

.PHONY: dev-env
dev-env: $(MINIKUBE) ## Create minikube cluster for dev and tests
CLUSTER_NAME=$(TEST_CLUSTER_NAME) MINIKUBE_BIN=$(MINIKUBE) $(CURDIR)/hack/scripts/setup_minikube.sh

.PHONY: dev-env
dev-env-multinode: $(MINIKUBE) ## Create minikube cluster for dev and tests
CLUSTER_NAME=$(TEST_CLUSTER_NAME) MINIKUBE_BIN=$(MINIKUBE) NUM_NODES=4 USE_MINIKUBE_DOCKER=false $(CURDIR)/hack/scripts/setup_minikube.sh

.PHONY: clean-dev-env
clean-dev-env: $(MINIKUBE) ## Teardown minikube cluster for dev and tests
$(MINIKUBE) delete -p $(TEST_CLUSTER_NAME)

.PHONY: dev-operator
dev-operator: $(MINIKUBE) $(SKAFFOLD) ## Deploy maintenance operator controller to dev cluster using skaffold
{\
eval $$($(MINIKUBE) -p $(TEST_CLUSTER_NAME) docker-env); \
$(SKAFFOLD) dev -p operator --default-repo=$(SKAFFOLD_REGISTRY) --detect-minikube=false --cleanup=false --trigger=manual; \
}

.PHONY: dev-operator-debug
dev-operator-debug: $(MINIKUBE) $(SKAFFOLD) ## Deploy maintenance operator controller to dev cluster using skaffold with remote debug
{\
eval $$($(MINIKUBE) -p $(TEST_CLUSTER_NAME) docker-env); \
$(SKAFFOLD) debug -p operator --default-repo=$(SKAFFOLD_REGISTRY) --detect-minikube=false --cleanup=false; \
}
11 changes: 11 additions & 0 deletions config/debug/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Adds namespace to all resources.
namespace: maintenance-operator-system
namePrefix: maintenance-operator-

resources:
- ../crd
- ../rbac
- ../manager

patches:
- path: manager_debug_patch.yaml
16 changes: 16 additions & 0 deletions config/debug/manager_debug_patch.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
namespace: system
spec:
template:
spec:
securityContext: null
containers:
- name: manager
securityContext: null
livenessProbe: null
readinessProbe: null
resources: null

2 changes: 1 addition & 1 deletion config/manager/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
images:
- name: controller
newName: harbor.mellanox.com/cloud-orchestration-dev/adrianc/maintenance-operator
newName: nvidia.com/maintenance-operator
newTag: latest
94 changes: 94 additions & 0 deletions hack/scripts/setup_minikube.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
#!/bin/bash

# 2024 NVIDIA CORPORATION & AFFILIATES
#
# Licensed under the Apache License, Version 2.0 (the License);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o nounset
set -o pipefail
set -o errexit

if [[ "${TRACE-0}" == "1" ]]; then
set -o xtrace
fi

CLUSTER_NAME="${CLUSTER_NAME:-"mn-op"}"
MINIKUBE_BIN="${MINIKUBE_BIN:-"unknown"}"
# Docker is used as the minikube driver to enable using hollow nodes.
MINIKUBE_DRIVER="${MINIKUBE_DRIVER:-"unknown"}"
USE_MINIKUBE_DOCKER="${USE_MINIKUBE_DOCKER:-"true"}"
NUM_NODES="${NUM_NODES:-"1"}"
NODE_MEMORY="${NODE_MEMORY:-"8g"}"
NODE_CPUS="${NODE_CPUS:-"4"}"
NODE_DISK="${NODE_DISK:-"100g"}"
MINIKUBE_CNI="${MINIKUBE_CNI:-kindnet}"
MINIKUBE_KUBERNETES_VERSION="${MINIKUBE_KUBERNETES_VERSION:-"v1.30.0"}"

## Detect the OS.
OS="unknown"
if [[ "${OSTYPE}" == "linux"* ]]; then
OS="linux"
elif [[ "${OSTYPE}" == "darwin"* ]]; then
OS="darwin"
fi

# Exit if the OS is not supported.
if [[ "$OS" == "unknown" ]]; then
echo "os '$OSTYPE' not supported. Aborting." >&2
exit 1
fi

## Set the driver used for minikube machines. By default this script will select the preferred VM driver for each OS.
## Users can override this using the MINIKUBE_DRIVER env variable.
if [[ "$MINIKUBE_DRIVER" == "unknown" ]]; then
if [[ "${OSTYPE}" == "linux"* ]]; then
MINIKUBE_DRIVER="kvm2"
elif [[ "${OSTYPE}" == "darwin"* ]]; then
MINIKUBE_DRIVER="qemu"
fi
fi

MINIKUBE_ARGS="${MINIKUBE_ARGS:-"\
--driver $MINIKUBE_DRIVER \
--cpus=$NODE_CPUS \
--memory=$NODE_MEMORY \
--disk-size=$NODE_DISK \
--nodes=$NUM_NODES \
--cni $MINIKUBE_CNI \
--kubernetes-version $MINIKUBE_KUBERNETES_VERSION \
--preload=true \
--cache-images=true \
--addons registry"}"

MINIKUBE_EXTRA_ARGS="${MINIKUBE_EXTRA_ARGS:-""}"

echo "Setting up Minikube cluster..."

## Exit early if the cluster already exists.
if [[ $($MINIKUBE_BIN status -p $CLUSTER_NAME -f '{{.Name}}') == $CLUSTER_NAME ]]; then
echo "Minikube cluster '$CLUSTER_NAME' found. Skipping cluster set-up"
## Set environment variables to use the Minikube VM docker build for
if [[ "$USE_MINIKUBE_DOCKER" == "true" ]]; then
echo "Setting environment variables to use $CLUSTER_NAME as docker build environment."
eval $($MINIKUBE_BIN -p $CLUSTER_NAME docker-env)
fi
exit 0
fi

$MINIKUBE_BIN start --profile $CLUSTER_NAME $MINIKUBE_ARGS $MINIKUBE_EXTRA_ARGS

## Set environment variables to use the Minikube VM docker build for
if [[ "$USE_MINIKUBE_DOCKER" == "true" ]]; then
echo "Setting environment variables to use $CLUSTER_NAME as docker build environment."
eval $($MINIKUBE_BIN -p $CLUSTER_NAME docker-env)
fi
31 changes: 31 additions & 0 deletions skaffold.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
apiVersion: skaffold/v4beta11
kind: Config
metadata:
name: maintenance-operator

profiles:
- name: operator
build:
local:
useBuildkit: true
artifacts:
- image: nvidia.com/maintenance-operator
runtimeType: go
docker:
dockerfile: Dockerfile
buildArgs:
# Used for debugging. Disable inlining and optimizations in the go compiler. Passed to go build using `-gcflags`
GCFLAGS: "all=-N -l"
manifests:
## Deploy the manager.
kustomize:
paths:
- config/debug

portForward:
- resourceType: deployment
resourceName: maintenance-operator-controller-manager
namespace: maintenance-operator-system
port: 56268
localPort: 8082
address: 0.0.0.0

0 comments on commit 1fccbbc

Please sign in to comment.