Metal-3121/k8s 1.22 (#89)

* use v1 kubes API versions for v1.22+ * update crd schema for 1.22+, update minikube to v1.22.3 * Fix k3s version string * METAL-3121 add missing (optional) apiEndpoint to CRD definition * METAL-3121 update crd * METAL-3121 fix optional fields, add helm crd in makefile * METAL-3121 fix optional fields * METAL-3121 fix shortnames * METAL-3121 fix typo mistake * METAL-3121 helm files from generated manifests * METAL-3121 fix datatypes for helm, cleanup makefile * METAL-3121 add printcolumns * METAL-3121 rollback generated rbac * METAL-3121 test more k8s versions * METAL-3121 no yaml anchors * METAL-3121 try version matrix * METAL-3121 limit parallel * METAL-3121 split integration test and unit test * METAL-3121 increase retries, looks k8s 1.21 api is slower * METAL-3121 do more parallel * METAL-3121 combine labels in helm helper Co-authored-by: Shawn Napora <[email protected]>
kloeckner-i · Nov 19, 2021 · 70f37af · 70f37af
1 parent a8b3ade
commit 70f37af
Show file tree

Hide file tree

Showing 19 changed files with 751 additions and 398 deletions.
diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml
@@ -21,7 +21,7 @@ jobs:
         # Until we fix up linting completely
         continue-on-error: true
 
-  test:
+  unit-test:
     runs-on: ubuntu-latest
     needs: lint
     steps:
@@ -54,7 +54,42 @@ jobs:
         run: docker-compose up -d
       - name: Run the unit tests
         run: go test -tags tests ./... -v -cover
+
+  test:
+    runs-on: ubuntu-latest
+    needs: unit-test
+    strategy:
+      matrix:
+        k8s_version: ['v1.19.12', 'v1.20.12', 'v1.21.6', 'v1.22.3']
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15
+      # The existing apparmor profile for mysql needs to be removed.
+      # https://github.com/actions/virtual-environments/issues/181
+      # https://github.com/moby/moby/issues/7512#issuecomment-51845976
+      - name: Remove MySQL App Armour Configuration
+        run: |
+          set -x
+          sudo apt-get remove mysql-server --purge
+          sudo apt-get install apparmor-profiles
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
       - name: Create k3d cluster
+        env:
+          K8S_VERSION: ${{ matrix.k8s_version }}
         run: make k3d_install
       - name: Build and import docker images
         run: |
@@ -63,4 +98,4 @@ jobs:
       - name: Install Helm chart
         run: make helm
       - name: Integration test
-        run: ./integration/test.sh
+        run: ./integration/test.sh
diff --git a/Makefile b/Makefile
@@ -1,11 +1,13 @@
-# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
-CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
-
 .PHONY: all deploy build helm
 .ONESHELL: test
 
 SRC = $(shell find . -type f -name '*.go')
 
+ifeq ($(K8S_VERSION),)
+K8S_VERSION := v1.22.3
+endif
+
+
 help:   ## show this help
 	@echo 'usage: make [target] ...'
 	@echo ''
@@ -54,7 +56,7 @@ vet: $(SRC)
 minisetup: miniup miniimage helm
 
 miniup: ## start minikube
-	@minikube start --kubernetes-version=v1.19.12 --cpus 2 --memory 4096
+	@minikube start --kubernetes-version=$(K8S_VERSION) --cpus 2 --memory 4096
 
 minidown: ## stop minikube
 	@minikube stop
@@ -72,23 +74,24 @@ k3d_setup: k3d_install k3d_image helm
 
 k3d_install:
 	@wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
-	@k3d cluster create myk3s -i rancher/k3s:v1.19.16-k3s1
+	@k3d cluster create myk3s -i rancher/k3s:$(K8S_VERSION)-k3s1
 	@kubectl get pod
 
 k3d_image: build
 	@k3d image import my-image.tar -c myk3s
 
 ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 manifests: controller-gen ## generate custom resource definitions
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	$(CONTROLLER_GEN) crd rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	$(CONTROLLER_GEN) crd webhook paths="./..." output:crd:artifacts:config=helm/db-operator/files/gen/crd
 
 ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 generate: controller-gen ## generate supporting code for custom resource types
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1)
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0)
 
 # go-get-tool will 'go get' any package $2 and install it to $1.
 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -102,4 +105,4 @@ echo "Downloading $(2)" ;\
 GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\
 rm -rf $$TMP_DIR ;\
 }
-endef
+endef
diff --git a/README.md b/README.md
@@ -45,20 +45,22 @@ To have kubernetes environment locally, you need to install [minikube](https://k
 #### makefile help
 
 ```
-make miniup: start minikube
-make minidown: stop minikube
-make minidelete: delete minikube
-make minidashboard: open minikube dashboard
-make build: build db-operator docker image
-make helm: install helm if not exist and install local chart using helm upgrade --install command
-make setup: build db-operator image, install helm
-make update: build db-operator image again and delete running pod
-make addexamples: kubectl create -f examples/
-make test: spin up mysql, postgres containers and run go unit test
-make microsetup: install microk8s locally and deploy db-operator (only for linux)
-make k3d_setup: install k3d locally and deploy db-operator
-make manifests: generate custom resource definitions
-make generate: generate supporting code for custom resource types
+addexamples      add examples via kubectl create -f examples/
+build            build db-operator docker image
+controller-gen   Download controller-gen locally if necessary.
+generate         generate supporting code for custom resource types
+helm             install helm if not exist and install local chart using helm upgrade --install command
+helm-lint        lint helm manifests
+help             show this help
+k3d_setup        install microk8s locally and deploy db-operator (only for linux and mac)
+manifests        generate custom resource definitions
+minidashboard    open minikube dashboard
+minidelete       delete minikube
+minidown         stop minikube
+miniup           start minikube
+setup            build db-operator image, install helm
+test             spin up mysql, postgres containers and run go unit test
+update           build db-operator image again and delete running pod
 ```
 
 ### Developing with Minikube
@@ -72,6 +74,11 @@ $ make setup
 
 #### After code changes
 
+rebuild CRD manifests
+```
+$ make manifests
+```
+
 rebuild local docker image
 ```
 $ make build

diff --git a/api/v1alpha1/database_types.go b/api/v1alpha1/database_types.go
@@ -62,6 +62,12 @@ type DatabaseBackup struct {
 
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
+//+kubebuilder:resource:shortName=db
+//+kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`,description="current db phase"
+//+kubebuilder:printcolumn:name="Status",type=boolean,JSONPath=`.status.status`,description="current db status"
+//+kubebuilder:printcolumn:name="Protected",type=boolean,JSONPath=`.spec.deletionProtected`,description="If database is protected to not get deleted."
+//+kubebuilder:printcolumn:name="DBInstance",type=string,JSONPath=`.status.instanceRef.metadata.name`,description="instance reference"
+//+kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`,description="time since creation of resource"
 
 // Database is the Schema for the databases API
 type Database struct {

diff --git a/api/v1alpha1/dbinstance_types.go b/api/v1alpha1/dbinstance_types.go
@@ -29,9 +29,9 @@ type DbInstanceSpec struct {
 	// Important: Run "make generate" to regenerate code after modifying this file
 	Engine           string                  `json:"engine"`
 	AdminUserSecret  NamespacedName          `json:"adminSecretRef"`
-	Backup           DbInstanceBackup        `json:"backup"`
-	Monitoring       DbInstanceMonitoring    `json:"monitoring"`
-	SSLConnection    DbInstanceSSLConnection `json:"sslConnection"`
+	Backup           DbInstanceBackup        `json:"backup,omitempty"`
+	Monitoring       DbInstanceMonitoring    `json:"monitoring,omitempty"`
+	SSLConnection    DbInstanceSSLConnection `json:"sslConnection,omitempty"`
 	DbInstanceSource `json:",inline"`
 }
 
@@ -84,7 +84,7 @@ type GenericInstance struct {
 	// BackupHost address will be used for dumping database for backup
 	// Usually secondary address for primary-secondary setup or cluster lb address
 	// If it's not defined, above Host will be used as backup host address.
-	BackupHost string `json:"backupHost"`
+	BackupHost string `json:"backupHost,omitempty"`
 }
 
 // DbInstanceBackup defines name of google bucket to use for storing database dumps for backup when backup is enabled
@@ -106,7 +106,9 @@ type DbInstanceSSLConnection struct {
 
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
-//+kubebuilder:resource:scope=Cluster
+//+kubebuilder:resource:scope=Cluster,shortName=dbin
+//+kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`,description="current phase"
+//+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.status`,description="health status"
 
 // DbInstance is the Schema for the dbinstances API
 type DbInstance struct {

diff --git a/config/crd/bases/kci.rocks_databases.yaml b/config/crd/bases/kci.rocks_databases.yaml
@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.4.1
+    controller-gen.kubebuilder.io/version: v0.7.0
   creationTimestamp: null
   name: databases.kci.rocks
 spec:
@@ -13,10 +13,33 @@ spec:
     kind: Database
     listKind: DatabaseList
     plural: databases
+    shortNames:
+    - db
     singular: database
   scope: Namespaced
   versions:
-  - name: v1alpha1
+  - additionalPrinterColumns:
+    - description: current db phase
+      jsonPath: .status.phase
+      name: Phase
+      type: string
+    - description: current db status
+      jsonPath: .status.status
+      name: Status
+      type: boolean
+    - description: If database is protected to not get deleted.
+      jsonPath: .spec.deletionProtected
+      name: Protected
+      type: boolean
+    - description: instance reference
+      jsonPath: .status.instanceRef.metadata.name
+      name: DBInstance
+      type: string
+    - description: time since creation of resource
+      jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
     schema:
       openAPIV3Schema:
         description: Database is the Schema for the databases API
@@ -90,8 +113,10 @@ spec:
                     description: DbInstanceSpec defines the desired state of DbInstance
                     properties:
                       adminSecretRef:
-                        description: NamespacedName is required because CRD structs
-                          must have all fields json tagged.
+                        description: NamespacedName is a fork of the kubernetes api
+                          type of the same name. Sadly this is required because CRD
+                          structs must have all fields json tagged and the kubernetes
+                          type is not tagged.
                         properties:
                           Name:
                             type: string
@@ -134,7 +159,6 @@ spec:
                           publicIp:
                             type: string
                         required:
-                        - backupHost
                         - host
                         - port
                         type: object
@@ -143,9 +167,13 @@ spec:
                           Google Cloud SQL and describes necessary informations to
                           use google API to create sql instances
                         properties:
+                          apiEndpoint:
+                            type: string
                           configmapRef:
-                            description: NamespacedName is required because CRD structs
-                              must have all fields json tagged.
+                            description: NamespacedName is a fork of the kubernetes
+                              api type of the same name. Sadly this is required because
+                              CRD structs must have all fields json tagged and the
+                              kubernetes type is not tagged.
                             properties:
                               Name:
                                 type: string
@@ -174,8 +202,10 @@ spec:
                           percona cluster
                         properties:
                           monitorUserSecretRef:
-                            description: NamespacedName is required because CRD structs
-                              must have all fields json tagged.
+                            description: NamespacedName is a fork of the kubernetes
+                              api type of the same name. Sadly this is required because
+                              CRD structs must have all fields json tagged and the
+                              kubernetes type is not tagged.
                             properties:
                               Name:
                                 type: string
@@ -224,10 +254,7 @@ spec:
                         type: object
                     required:
                     - adminSecretRef
-                    - backup
                     - engine
-                    - monitoring
-                    - sslConnection
                     type: object
                   status:
                     description: DbInstanceStatus defines the observed state of DbInstance