Merge pull request #125 from senthilrch/develop

PR for release v0.9.0

Author: senthilrch

Makefile

@@ -43,7 +43,7 @@ ifndef OPERATOR_IMAGE_REPO
 endif
 
 ifndef RELEASE_VERSION
-  RELEASE_VERSION=v0.8.2
+  RELEASE_VERSION=v0.9.0
 endif
 
 ifndef DOCKER_VERSION
@@ -190,16 +190,22 @@ hack:
 deploy-using-yaml:
 	-kubectl apply -f deploy/kubefledged-namespace.yaml
 	kubectl apply -f deploy/kubefledged-crd.yaml
-	kubectl apply -f deploy/kubefledged-serviceaccount.yaml
-	kubectl apply -f deploy/kubefledged-clusterrole.yaml
-	kubectl apply -f deploy/kubefledged-clusterrolebinding.yaml
+	kubectl apply -f deploy/kubefledged-serviceaccount-controller.yaml
+	kubectl apply -f deploy/kubefledged-clusterrole-controller.yaml
+	kubectl apply -f deploy/kubefledged-clusterrolebinding-controller.yaml
+	kubectl apply -f deploy/kubefledged-deployment-controller.yaml
+	kubectl rollout status deployment kubefledged-controller -n kube-fledged --watch
+
+deploy-webhook-server-using-yaml:
 	-kubectl delete validatingwebhookconfigurations -l app=kubefledged
 	kubectl apply -f deploy/kubefledged-validatingwebhook.yaml
+	-kubectl delete deploy -l app=kubefledged,kubefledged=kubefledged-webhook-server
+	kubectl apply -f deploy/kubefledged-serviceaccount-webhook-server.yaml
+	kubectl apply -f deploy/kubefledged-clusterrole-webhook-server.yaml
+	kubectl apply -f deploy/kubefledged-clusterrolebinding-webhook-server.yaml
 	kubectl apply -f deploy/kubefledged-deployment-webhook-server.yaml
 	kubectl apply -f deploy/kubefledged-service-webhook-server.yaml
-	kubectl apply -f deploy/kubefledged-deployment-controller.yaml
 	kubectl rollout status deployment kubefledged-webhook-server -n kube-fledged --watch
-	kubectl rollout status deployment kubefledged-controller -n kube-fledged --watch
 
 deploy-using-operator:
 	# Create the namespace
@@ -218,6 +224,10 @@ deploy-using-operator:
 	kubectl rollout status deployment kubefledged-operator -n ${KUBEFLEDGED_NAMESPACE} --watch
 	kubectl apply -f deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml
 
+deploy-webhook-server-using-operator:
+	sed -i "s|enable: false|enable: true|g" deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml
+	kubectl apply -f deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml
+
 update:
 	kubectl scale deployment kubefledged-controller --replicas=0 -n kube-fledged
 	kubectl scale deployment kubefledged-webhook-server --replicas=0 -n kube-fledged && sleep 1
@@ -227,12 +237,20 @@ update:
 
 remove-kubefledged:
 	-kubectl delete -f deploy/kubefledged-namespace.yaml
-	-kubectl delete -f deploy/kubefledged-clusterrolebinding.yaml
-	-kubectl delete -f deploy/kubefledged-clusterrole.yaml
-	-kubectl delete -f deploy/kubefledged-crd.yaml
-	-kubectl delete -f deploy/kubefledged-validatingwebhook.yaml
+	-kubectl delete clusterrolebinding -l app=kubefledged
+	-kubectl delete clusterrole -l app=kubefledged
+	-kubectl delete crd -l app=kubefledged
+	-kubectl delete validatingwebhookconfigurations -l app=kubefledged
+
+remove-webhook-server:
+	-kubectl delete validatingwebhookconfigurations -l app=kubefledged
+	-kubectl delete deploy -l app=kubefledged,kubefledged=kubefledged-webhook-server -n kube-fledged
+	-kubectl delete service -l app=kubefledged,kubefledged=kubefledged-webhook-server -n kube-fledged
+	-kubectl delete clusterrolebinding -l app=kubefledged,kubefledged=kubefledged-webhook-server
+	-kubectl delete clusterrole -l app=kubefledged,kubefledged=kubefledged-webhook-server
+	-kubectl delete serviceaccount -l app=kubefledged,kubefledged=kubefledged-webhook-server -n kube-fledged
 
-remove-operator-and-kubefledged:
+remove-kubefledged-and-operator:
 	# Remove kubefledged
 	-kubectl delete -f deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml
 	-kubectl delete validatingwebhookconfigurations -l app.kubernetes.io/name=kube-fledged
@@ -247,4 +265,8 @@ remove-operator-and-kubefledged:
 	-git checkout deploy/kubefledged-operator/deploy/operator.yaml
 	-git checkout deploy/kubefledged-operator/deploy/clusterrole_binding.yaml
 	-git checkout deploy/kubefledged-operator/deploy/service_account.yaml
-	-git checkout deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml
+
+remove-webhook-server-using-operator:
+	sed -i "s|enable: true|enable: false|g" deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml
+	kubectl apply -f deploy/kubefledged-operator/deploy/crds/charts.helm.kubefledged.io_v1alpha2_kubefledged_cr.yaml
+

README.md

@@ -83,6 +83,12 @@ These instructions install _kube-fledged_ to a separate namespace called "kube-f
   $ kubectl get imagecaches -n kube-fledged (Output should be: 'No resources found')
   ```
 
+- Optional: Deploy _kube-fledged webhook server_ to the cluster. This component enables validating the ImageCache CR.
+
+  ```
+  $ make deploy-webhook-server-using-yaml
+  ```
+
 ## Quick Install using Helm chart
 
 - Create the namespace where kube-fledged will be installed
@@ -102,6 +108,12 @@ These instructions install _kube-fledged_ to a separate namespace called "kube-f
   $ helm install --verify kube-fledged kubefledged-charts/kube-fledged -n ${KUBEFLEDGED_NAMESPACE} --wait
   ```
 
+- Optional: Verify and install kube-fledged webhook server. This component enables validating the ImageCache CR.
+
+  ```
+  $ helm upgrade --verify kube-fledged kubefledged-charts/kube-fledged -n ${KUBEFLEDGED_NAMESPACE} --set webhookServer.enable=true --wait
+  ```
+
 ## Quick Install using Helm operator
 
 These instructions install _kube-fledged_ to a separate namespace called "kube-fledged", using Helm operator and pre-built images in [Docker Hub.](https://hub.docker.com/u/senthilrch)
@@ -127,6 +139,12 @@ These instructions install _kube-fledged_ to a separate namespace called "kube-f
   $ kubectl get imagecaches -n kube-fledged (Output should be: 'No resources found')
   ```
 
+- Optional: Deploy _kube-fledged webhook server_ to the cluster. This component enables validating the ImageCache CR.
+
+  ```
+  $ make deploy-webhook-server-using-operator
+  ```
+
 ## Helm chart parameters
 
 Parameters of the helm chart are documented [here](docs/helm-parameters.md)
@@ -277,9 +295,18 @@ Run the following command to remove _kube-fledged_ from the cluster.
 
 ```
 $ make remove-kubefledged (if you deployed using YAML manifests)
+$ helm delete kube-fledged -n ${KUBEFLEDGED_NAMESPACE} (if you deployed using Helm chart)
 $ make remove-operator-and-kubefledged (if you deployed using Helm Operator)
 ```
 
+Note: To remove the _kube-fledged webhook server_ alone. 
+
+```
+$ make remove-webhook-server (if you deployed using YAML manifests)
+$ helm upgrade kube-fledged deploy/kubefledged-operator/helm-charts/kubefledged -n ${KUBEFLEDGED_NAMESPACE} --set webhookServer.enable=false --wait --debug (if you deployed using Helm chart)
+$ make remove-webhook-server-using-operator (if you deployed using Helm Operator)
+```
+
 ## How it works
 
 Kubernetes allows developers to extend the kubernetes api via [Custom Resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). _kube-fledged_ defines a custom resource of kind “ImageCache” and implements a custom controller (named _kubefledged-controller_). _kubefledged-controller_ does the heavy-lifting for managing image cache. Users can use kubectl commands for creation and deletion of ImageCache resources.
@@ -297,6 +324,8 @@ For more detailed description, go through _kube-fledged's_ [design proposal](doc
 
 `--image-pull-policy:` Image pull policy for pulling images into and refreshing the cache. Possible values are 'IfNotPresent' and 'Always'. Default value is 'IfNotPresent'. Image with no or ":latest" tag are always pulled.
 
+`--service-account-name:` serviceAccountName used in Jobs created for pulling or deleting images. Optional flag. If not specified the default service account of the namespace is used
+
 `--stderrthreshold:` Log level. set the value of this flag to INFO
 
 ## Supported Container Runtimes

cmd/controller/app/controller.go

@@ -32,6 +32,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	coreinformers "k8s.io/client-go/informers/core/v1"
@@ -94,7 +95,8 @@ func NewController(
 	imagePullDeadlineDuration time.Duration,
 	criClientImage string,
 	busyboxImage string,
-	imagePullPolicy string) *Controller {
+	imagePullPolicy string,
+	serviceAccountName string) *Controller {
 
 	runtime.Must(fledgedscheme.AddToScheme(scheme.Scheme))
 	glog.V(4).Info("Creating event broadcaster")
@@ -117,7 +119,8 @@ func NewController(
 		imageCacheRefreshFrequency: imageCacheRefreshFrequency,
 	}
 
-	imageManager, _ := images.NewImageManager(controller.workqueue, controller.imageworkqueue, controller.kubeclientset, controller.fledgedNameSpace, imagePullDeadlineDuration, criClientImage, busyboxImage, imagePullPolicy)
+	imageManager, _ := images.NewImageManager(controller.workqueue, controller.imageworkqueue, controller.kubeclientset,
+		controller.fledgedNameSpace, imagePullDeadlineDuration, criClientImage, busyboxImage, imagePullPolicy, serviceAccountName)
 	controller.imageManager = imageManager
 
 	glog.Info("Setting up event handlers")
@@ -149,7 +152,14 @@ func (c *Controller) PreFlightChecks() error {
 
 // danglingJobs finds and removes dangling or stuck jobs
 func (c *Controller) danglingJobs() error {
-	joblist, err := c.kubeclientset.BatchV1().Jobs(c.fledgedNameSpace).List(context.TODO(), metav1.ListOptions{})
+	appEqKubefledged, _ := labels.NewRequirement("app", selection.Equals, []string{"kubefledged"})
+	kubefledgedEqImagemanager, _ := labels.NewRequirement("kubefledged", selection.Equals, []string{"kubefledged-image-manager"})
+	labelSelector := labels.NewSelector()
+	labelSelector = labelSelector.Add(*appEqKubefledged, *kubefledgedEqImagemanager)
+
+	joblist, err := c.kubeclientset.BatchV1().Jobs("").List(context.TODO(), metav1.ListOptions{
+		LabelSelector: labelSelector.String(),
+	})
 	if err != nil {
 		glog.Errorf("Error listing jobs: %v", err)
 		return err
@@ -161,7 +171,7 @@ func (c *Controller) danglingJobs() error {
 	}
 	deletePropagation := metav1.DeletePropagationBackground
 	for _, job := range joblist.Items {
-		err := c.kubeclientset.BatchV1().Jobs(c.fledgedNameSpace).
+		err := c.kubeclientset.BatchV1().Jobs(job.Namespace).
 			Delete(context.TODO(), job.Name, metav1.DeleteOptions{PropagationPolicy: &deletePropagation})
 		if err != nil {
 			glog.Errorf("Error deleting job(%s): %v", job.Name, err)
@@ -176,7 +186,7 @@ func (c *Controller) danglingJobs() error {
 // image caches will get refreshed in the next cycle
 func (c *Controller) danglingImageCaches() error {
 	dangling := false
-	imagecachelist, err := c.kubefledgedclientset.KubefledgedV1alpha2().ImageCaches(c.fledgedNameSpace).List(context.TODO(), metav1.ListOptions{})
+	imagecachelist, err := c.kubefledgedclientset.KubefledgedV1alpha2().ImageCaches("").List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
 		glog.Errorf("Error listing imagecaches: %v", err)
 		return err
@@ -385,7 +395,7 @@ func (c *Controller) processNextWorkItem() bool {
 // runRefreshWorker is resposible of refreshing the image cache
 func (c *Controller) runRefreshWorker() {
 	// List the ImageCache resources
-	imageCaches, err := c.imageCachesLister.ImageCaches(c.fledgedNameSpace).List(labels.Everything())
+	imageCaches, err := c.imageCachesLister.ImageCaches("").List(labels.Everything())
 	if err != nil {
 		glog.Errorf("Error in listing image caches: %v", err)
 		return
@@ -506,10 +516,6 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error {
 				}
 			}
 			glog.V(4).Infof("No. of nodes in %+v is %d", i.NodeSelector, len(nodes))
-			if len(nodes) == 0 {
-				glog.Errorf("NodeSelector %+v did not match any nodes.", i.NodeSelector)
-				return fmt.Errorf("NodeSelector %+v did not match any nodes", i.NodeSelector)
-			}
 
 			for _, n := range nodes {
 				for m := range i.Images {
@@ -565,7 +571,9 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error {
 			status.StartTime = imageCache.Status.StartTime
 		}
 
+		status.Status = v1alpha2.ImageCacheActioneNoImagesPulledOrDeleted
 		status.Reason = imageCache.Status.Reason
+		status.Message = v1alpha2.ImageCacheMessageNoImagesPulledOrDeleted
 
 		failures := false
 		for _, v := range *wqKey.Status {
@@ -624,7 +632,7 @@ func (c *Controller) syncHandler(wqKey images.WorkQueueKey) error {
 			}
 		}
 
-		if status.Status == v1alpha2.ImageCacheActionStatusSucceeded {
+		if status.Status == v1alpha2.ImageCacheActionStatusSucceeded || status.Status == v1alpha2.ImageCacheActioneNoImagesPulledOrDeleted {
 			c.recorder.Event(imageCache, corev1.EventTypeNormal, status.Reason, status.Message)
 		}
 

cmd/controller/main.go

@@ -41,7 +41,7 @@ var (
 	busyboxImage               string
 	imagePullPolicy            string
 	fledgedNameSpace           string
-	webhookServerPort          int
+	serviceAccountName         string
 )
 
 func main() {
@@ -71,7 +71,7 @@ func main() {
 	controller := app.NewController(kubeClient, fledgedClient, fledgedNameSpace,
 		kubeInformerFactory.Core().V1().Nodes(),
 		fledgedInformerFactory.Kubefledged().V1alpha2().ImageCaches(),
-		imageCacheRefreshFrequency, imagePullDeadlineDuration, criClientImage, busyboxImage, imagePullPolicy)
+		imageCacheRefreshFrequency, imagePullDeadlineDuration, criClientImage, busyboxImage, imagePullPolicy, serviceAccountName)
 
 	glog.Info("Starting pre-flight checks")
 	if err = controller.PreFlightChecks(); err != nil {
@@ -100,4 +100,5 @@ func init() {
 	if busyboxImage = os.Getenv("BUSYBOX_IMAGE"); busyboxImage == "" {
 		busyboxImage = "busybox:1.29.2"
 	}
+	flag.StringVar(&serviceAccountName, "service-account-name", "", "serviceAccountName used in Jobs created for pulling/deleting images. Optional flag. If not specified the default service account of the namespace is used")
 }

deploy/kubefledged-clusterrole.yaml renamed to deploy/kubefledged-clusterrole-controller.yaml

@@ -4,7 +4,7 @@ metadata:
   name: kubefledged-controller
   labels:
     app: kubefledged
-    component: kubefledged-controller
+    kubefledged: kubefledged-controller
   annotations:
     rbac.authorization.kubernetes.io/autoupdate: "true"
 rules:
@@ -58,21 +58,3 @@ rules:
       - list
       - watch
       - get    
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: kubefledged-webhook-server
-  labels:
-    app: kubefledged
-    component: kubefledged-webhook-server
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-rules:
-  - apiGroups:
-      - "admissionregistration.k8s.io"
-    resources:
-      - validatingwebhookconfigurations
-    verbs:
-      - get
-      - patch

deploy/kubefledged-clusterrole-webhook-server.yaml

@@ -0,0 +1,17 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kubefledged-webhook-server
+  labels:
+    app: kubefledged
+    kubefledged: kubefledged-webhook-server
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+rules:
+  - apiGroups:
+      - "admissionregistration.k8s.io"
+    resources:
+      - validatingwebhookconfigurations
+    verbs:
+      - get
+      - patch

deploy/kubefledged-clusterrolebinding-controller.yaml

@@ -0,0 +1,18 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kubefledged-controller  
+  labels:
+    app: kubefledged
+    kubefledged: kubefledged-controller  
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubefledged-controller
+subjects:
+- kind: ServiceAccount
+  name: kubefledged-controller
+  namespace: kube-fledged
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes

deploy/kubefledged-clusterrolebinding-webhook-server.yaml

@@ -0,0 +1,18 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kubefledged-webhook-server 
+  labels:
+    app: kubefledged
+    kubefledged: kubefledged-webhook-server   
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubefledged-webhook-server
+subjects:
+- kind: ServiceAccount
+  name: kubefledged-webhook-server
+  namespace: kube-fledged
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes