support gRPC communication between primary and secondary mantle contr…

…ollers

This commit allows a mantle-controller to serve as one of standalone,
primary, or secondary mode. This commit doesn't have any e2e test's
update and it will land on in the following commits.

Signed-off-by: Ryotaro Banno <[email protected]>

.gitignore

@@ -26,3 +26,4 @@ Dockerfile.cross
 *~
 
 test/e2e/testdata/persistentvolumes.yaml
+include/

Dockerfile

@@ -14,6 +14,7 @@ COPY main.go main.go
 COPY cmd/ cmd/
 COPY api/ api/
 COPY internal/ internal/
+COPY pkg/ pkg/
 
 # Build
 # the GOARCH has not a default value to allow the binary be built according to the host where the command

Makefile

@@ -1,5 +1,22 @@
 include versions.mk
 
+## Location to install dependencies to
+LOCALBIN ?= $(shell pwd)/bin
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+
+## Tool Binaries
+KUBECTL ?= $(LOCALBIN)/kubectl
+KUSTOMIZE ?= $(LOCALBIN)/kustomize
+CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
+ENVTEST ?= $(LOCALBIN)/setup-envtest
+MOCKGEN ?= $(LOCALBIN)/mockgen
+PROTOC ?= $(LOCALBIN)/protoc
+PROTOC_RUN ?= PATH=$(LOCALBIN):$(PATH) $(PROTOC) -I=$(shell pwd)/include:.
+
+# Files to be generated by protoc
+PROTOBUF_GEN = pkg/controller/proto/controller.pb.go pkg/controller/proto/controller_grpc.pb.go docs/controller-protocol.md
+
 # Image URL to use all building/pushing image targets
 IMG ?= controller:latest
 
@@ -52,7 +69,7 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust
 	cat config/rbac/role.yaml | yq '.metadata.name = "mantle-controller"' > charts/mantle-cluster-wide/templates/clusterrole.yaml
 
 .PHONY: generate
-generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+generate: $(PROTOBUF_GEN) controller-gen ## Generate boilerplate code.
 	$(CONTROLLER_GEN) object paths="./..."
 
 .PHONY: fmt
@@ -151,17 +168,29 @@ undeploy: kubectl ## Undeploy controller from the K8s cluster specified in ~/.ku
 
 ##@ Build Dependencies
 
-## Location to install dependencies to
-LOCALBIN ?= $(shell pwd)/bin
-$(LOCALBIN):
-	mkdir -p $(LOCALBIN)
+pkg/controller/proto/controller.pb.go: pkg/controller/proto/controller.proto $(PROTOC)
+	$(PROTOC_RUN) --go_out=module=github.com/cybozu-go/mantle:. $<
 
-## Tool Binaries
-KUBECTL ?= $(LOCALBIN)/kubectl
-KUSTOMIZE ?= $(LOCALBIN)/kustomize
-CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
-ENVTEST ?= $(LOCALBIN)/setup-envtest
-MOCKGEN ?= $(LOCALBIN)/mockgen
+pkg/controller/proto/controller_grpc.pb.go: pkg/controller/proto/controller.proto $(PROTOC)
+	$(PROTOC_RUN) --go-grpc_out=module=github.com/cybozu-go/mantle:. $<
+
+docs/controller-protocol.md: pkg/controller/proto/controller.proto $(PROTOC)
+	$(PROTOC_RUN) --doc_out=./docs --doc_opt=markdown,$@ $<
+
+.PHONY: $(PROTOC)
+$(PROTOC): $(LOCALBIN)
+	[ -f $(PROTOC) ] || { \
+	set -e ;\
+	curl -Lo $(LOCALBIN)/protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-linux-x86_64.zip ;\
+	rm -rf $(LOCALBIN)/protoc.unzip ;\
+	unzip $(LOCALBIN)/protoc.zip -d $(LOCALBIN)/protoc.unzip ;\
+	cp -f $(LOCALBIN)/protoc.unzip/bin/protoc $(PROTOC) ;\
+	cp -r $(LOCALBIN)/protoc.unzip/include . ;\
+	rm -rf $(LOCALBIN)/protoc.unzip $(LOCALBIN)/protoc.zip ;\
+	GOBIN=$(LOCALBIN) go install google.golang.org/protobuf/cmd/protoc-gen-go@v$(PROTOC_GEN_GO_VERSION) ;\
+	GOBIN=$(LOCALBIN) go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v$(PROTOC_GEN_GO_GRPC_VERSION) ;\
+	GOBIN=$(LOCALBIN) go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc@v$(PROTOC_GEN_DOC_VERSION) ;\
+	}
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading.

api/v1/mantlebackup_types.go

@@ -39,7 +39,8 @@ type MantleBackupStatus struct {
 }
 
 const (
-	BackupConditionReadyToUse = "ReadyToUse"
+	BackupConditionReadyToUse     = "ReadyToUse"
+	BackupConditionSyncedToRemote = "SyncedToRemote"
 
 	// Reasons for ConditionReadyToUse
 	BackupReasonNone                 = "NoProblem"

charts/mantle/templates/deployment.yaml

@@ -60,6 +60,10 @@ spec:
           args:
             - controller
             - --leader-elect
+            - --role={{ .Values.controller.role }}
+            {{- with .Values.controller.mantleServiceEndpoint }}
+            - --mantle-service-endpoint={{ . }}
+            {{- end }}
             {{- with .Values.controller.expireOffset }}
             - --expire-offset={{ . }}
             {{- end }}
@@ -75,6 +79,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+          ports:
+            {{- toYaml .Values.controller.ports | nindent 12 }}
         - command:
             - /bin/bash
             - -c

charts/mantle/values.yaml

@@ -22,4 +22,6 @@ nodeSelector: {}
 tolerations: []
 affinity: {}
 
-controller: {}
+controller:
+  role: standalone
+  ports: []