add dataplane webhooks and fix dp finalizer (#75)

Author: bwagner5

operator/Makefile

@@ -9,7 +9,7 @@ WITH_RELEASE_REPO = KO_DOCKER_REPO=$(RELEASE_REPO)
 KIT_NAMESPACE ?= kit
 CONTAINER_IMAGE_REGISTRY ?= 
 ## Extra helm options
-HELM_OPTS ?=
+HELM_OPTS ?= --set serviceAccount.create=false
 
 help: ## Display help
 	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
@@ -47,14 +47,16 @@ licenses: ## Verifies dependency licenses and requires GITHUB_TOKEN to be set
 	golicense hack/license-config.hcl bin/operator
 
 apply: ## Apply the controller into your ~/.kube/config cluster
-	KO_DOCKER_REPO=$(CONTAINER_IMAGE_REGISTRY)/kit ko apply --bare -f config/
-	KO_DOCKER_REPO=$(CONTAINER_IMAGE_REGISTRY)/kit ko apply --bare -f config/controller
-	KO_DOCKER_REPO=$(CONTAINER_IMAGE_REGISTRY)/kit ko apply --bare -f config/webhook
+	KO_DOCKER_REPO=$(CONTAINER_IMAGE_REGISTRY)/kit helm template --include-crds kit charts/kit-operator --namespace $(KIT_NAMESPACE) \
+		$(HELM_OPTS) \
+		--set controller.image=ko://github.com/awslabs/kit/operator/cmd/controller \
+		--set webhook.image=ko://github.com/awslabs/kit/operator/cmd/webhook \
+		| $(WITH_GOFLAGS) ko apply -B -f -
 
 delete: ## Delete the controller from your ~/.kube/config cluster
-	kubectl delete -f config
-	kubectl delete -f config/webhook
-	kubectl delete -f config/controller
+	helm template kit charts/kit-operator --namespace $(KIT_NAMESPACE) \
+		$(HELM_OPTS) \
+		| kubectl delete -f -
 
 codegen: ## Generate code. Must be run if changes are made to ./pkg/apis/...
 	hack/codegen.sh

operator/README.md

@@ -71,7 +71,7 @@ EOF
 2. Get the admin KUBECONFIG for the guest cluster from the substrate cluster
 
 ```bash
-  kubectl get secret example-kube-admin-config -ojsonpath='{.data.config}' | base64 -d > /tmp/kubeconfig
+  kubectl get secret ${GUEST_CLUSTER_NAME}-kube-admin-config -ojsonpath='{.data.config}' | base64 -d > /tmp/kubeconfig
 ```
 > NOTE: It takes about 3-4 minutes for the cluster control plane to be available and healthy
 
@@ -108,4 +108,4 @@ EOF
     --namespace kit \
     --cluster ${SUBSTRATE_CLUSTER_NAME} \
     --region=$AWS_REGION
-```
+```

operator/charts/kit-operator/templates/control-plane-crd.yaml renamed to operator/charts/kit-operator/crds/control-plane-crd.yaml

@@ -1,7 +1,7 @@
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
-  name: defaulting.webhook.controlplane.kit.k8s.sh
+  name: defaulting.webhook.kit.k8s.sh
 webhooks:
 - admissionReviewVersions: ["v1"]
   clientConfig:
@@ -10,7 +10,7 @@ webhooks:
       namespace: {{ .Release.Namespace }}
   failurePolicy: Fail
   sideEffects: None
-  name: defaulting.webhook.controlplane.kit.k8s.sh
+  name: defaulting.webhook.kit.k8s.sh
   rules:
   - apiGroups:
     - kit.k8s.sh
@@ -19,6 +19,8 @@ webhooks:
     resources:
     - controlplanes
       controlplanes/status
+    - dataplanes
+      dataplanes/status
     operations:
     - CREATE
     - UPDATE
@@ -28,7 +30,7 @@ webhooks:
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
-  name: validation.webhook.controlplane.kit.k8s.sh
+  name: validation.webhook.kit.k8s.sh
 webhooks:
 - admissionReviewVersions: ["v1"]
   clientConfig:
@@ -37,7 +39,7 @@ webhooks:
       namespace: {{ .Release.Namespace }}
   failurePolicy: Fail
   sideEffects: None
-  name: validation.webhook.controlplane.kit.k8s.sh
+  name: validation.webhook.kit.k8s.sh
   rules:
   - apiGroups:
     - kit.k8s.sh
@@ -46,6 +48,8 @@ webhooks:
     resources:
     - controlplanes
       controlplanes/status
+    - dataplanes
+      dataplanes/status
     operations:
     - CREATE
     - UPDATE

operator/charts/kit-operator/templates/data-plane-crd.yaml renamed to operator/charts/kit-operator/crds/data-plane-crd.yaml

@@ -18,7 +18,9 @@ import (
 	"context"
 	"flag"
 
-	"github.com/awslabs/kit/operator/pkg/apis/controlplane/v1alpha1"
+	cpv1alpha1 "github.com/awslabs/kit/operator/pkg/apis/controlplane/v1alpha1"
+	dpv1alpha1 "github.com/awslabs/kit/operator/pkg/apis/dataplane/v1alpha1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 
 	"knative.dev/pkg/configmap"
 	"knative.dev/pkg/controller"
@@ -28,12 +30,14 @@ import (
 	"knative.dev/pkg/system"
 	"knative.dev/pkg/webhook"
 	"knative.dev/pkg/webhook/certificates"
+	"knative.dev/pkg/webhook/resourcesemantics"
 	"knative.dev/pkg/webhook/resourcesemantics/defaulting"
 	"knative.dev/pkg/webhook/resourcesemantics/validation"
 )
 
 var (
-	options = Options{}
+	options      = Options{}
+	kitResources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{}
 )
 
 type Options struct {
@@ -46,6 +50,12 @@ func main() {
 
 	config := injection.ParseAndGetRESTConfigOrDie()
 
+	// merge all kit resource handlers
+	kitResources = cpv1alpha1.Resources
+	for gvk, resource := range dpv1alpha1.Resources {
+		kitResources[gvk] = resource
+	}
+
 	// Controllers and webhook
 	sharedmain.MainWithConfig(
 		webhook.WithOptions(injection.WithNamespaceScope(signals.NewContext(), system.Namespace()), webhook.Options{
@@ -63,19 +73,19 @@ func main() {
 
 func NewCRDDefaultingWebhook(ctx context.Context, w configmap.Watcher) *controller.Impl {
 	return defaulting.NewAdmissionController(ctx,
-		"defaulting.webhook.controlplane.kit.k8s.sh",
+		"defaulting.webhook.kit.k8s.sh",
 		"/default-resource",
-		v1alpha1.Resources,
+		kitResources,
 		InjectContext,
 		true,
 	)
 }
 
 func NewCRDValidationWebhook(ctx context.Context, w configmap.Watcher) *controller.Impl {
 	return validation.NewAdmissionController(ctx,
-		"validation.webhook.controlplane.kit.k8s.sh",
+		"validation.webhook.kit.k8s.sh",
 		"/validate-resource",
-		v1alpha1.Resources,
+		kitResources,
 		InjectContext,
 		true,
 	)

operator/charts/kit-operator/templates/webhook/webhook.yaml

@@ -67,6 +67,14 @@ func (c *Controller) Reconcile(ctx context.Context, dataplane *v1alpha1.DataPlan
 }
 
 func (c *Controller) Finalize(ctx context.Context, dataplane *v1alpha1.DataPlane) error {
+	asg, err := c.getAutoScalingGroup(ctx, AutoScalingGroupNameFor(dataplane))
+	if err != nil {
+		return err
+	}
+	// ASG is already gone
+	if asg == nil {
+		return nil
+	}
 	if _, err := c.autoscaling.DeleteAutoScalingGroupWithContext(ctx, &autoscaling.DeleteAutoScalingGroupInput{
 		AutoScalingGroupName: ptr.String(AutoScalingGroupNameFor(dataplane)),
 		ForceDelete:          ptr.Bool(true), // terminate all the nodes in the ASG