Merge pull request #65 from pacoxu/dev-0.0.5

Dev 0.0.5

Makefile

@@ -1,6 +1,6 @@
 
 # Image URL to use all building/pushing image targets
-IMG ?= daocloud.io/daocloud/kubeadm-operator:v0.0.4-dev
+IMG ?= daocloud.io/daocloud/kubeadm-operator:v0.0.5-dev
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd"
 

api/v1alpha1/command_descriptor_types.go

@@ -43,6 +43,10 @@ type CommandDescriptor struct {
 	// +optional
 	KubectlUncordon *KubectlUncordonCommandSpec `json:"kubectlUncordon,omitempty"`
 
+	// +optional
+	// not implemented yet
+	KubeadmUpgradeKubeProxy *KubeadmUpgradeKubeProxySpec `json:"kubeadmUpgradeKubeProxy,omitempty"`
+
 	// Pass provide a dummy command for testing the kubeadm-operator workflow.
 	// +optional
 	Pass *PassCommandSpec `json:"pass,omitempty"`
@@ -82,12 +86,20 @@ type UpgradeKubeadmCommandSpec struct {
 
 // KubeadmUpgradeApplyCommandSpec provides...
 type KubeadmUpgradeApplyCommandSpec struct {
-	// +optional
 	// KubernetesVersion specifies the target kubernetes version
 	// If the version is empty, we will skip this command.
 	KubernetesVersion string `json:"kubernetesVersion"`
 	// for dry run mode
 	DryRun bool `json:"dryRun,omitempty"`
+	// skip kube-proxy upgrade
+	SkipKubeProxy bool `json:"skipKubePorxy,omitempty"`
+}
+
+// KubeadmUpgradeApplyCommandSpec provides...
+type KubeadmUpgradeKubeProxySpec struct {
+	// KubernetesVersion specifies the target kubernetes version
+	// If the version is empty, we will skip this command.
+	KubernetesVersion string `json:"kubernetesVersion"`
 }
 
 // TODO download the specified version bin and replace it in the node
@@ -123,7 +135,6 @@ type KubectlUncordonCommandSpec struct {
 // TODO download the specified version bin and replace it in the node
 // UpgradeKubeletAndKubeactlCommandSpec provides...
 type UpgradeKubeletAndKubeactlCommandSpec struct {
-	// +optional
 	// KubernetesVersion specifies the target kubernetes version
 	// If the version is empty, we will skip this command.
 	KubernetesVersion string `json:"kubernetesVersion"`

api/v1alpha1/operation_descriptor_types.go

@@ -37,7 +37,17 @@ type UpgradeOperationSpec struct {
 	// KubernetesVersion specifies the target kubernetes version
 	KubernetesVersion string `json:"kubernetesVersion"`
 
+	// Local is used to determine whether to use local binary or download from internet.
+	// +optional
 	Local bool `json:"local,omitempty"`
+
+	// UpgradeKubeProxyAtLast by default is false.
+	// TODO UpgradeKubeProxyAtLast can be true by default if this should be default behavior, needs more disscussions.
+	// If this is true, kube-proxy will not be upgraded at first. See more details in https://github.com/kubernetes/kubeadm/issues/2346
+	// Then kube-proxy will be upgraded after all apiserver are upgraded.
+	// +optional
+	UpgradeKubeProxyAtLast bool `json:"upgradeKubeProxyAtLast,omitempty"`
+
 	// INSERT ADDITIONAL SPEC FIELDS -
 	// Important: Run "make" to regenerate code after modifying this file
 }

commands/factory.go

@@ -59,6 +59,9 @@ func RunCommand(c *operatorv1.CommandDescriptor, log logr.Logger) error {
 	if c.UpgradeKubeletAndKubeactl != nil {
 		return runUpgradeKubectlAndKubelet(c.UpgradeKubeletAndKubeactl, log)
 	}
+	if c.KubeadmUpgradeKubeProxy != nil {
+		return runUpgradeKubeProxy(c.KubeadmUpgradeKubeProxy, log)
+	}
 
 	if c.Pass != nil {
 		return nil

commands/kubeadm_upgrade_apply.go

@@ -28,6 +28,8 @@ import (
 
 // runKubeadmUpgradeApply runs the kubeadm upgrade apply command
 func runKubeadmUpgradeApply(spec *operatorv1.KubeadmUpgradeApplyCommandSpec, log logr.Logger) error {
+	// TODO spec.SkipKubeProxy is not implemented yet
+
 	var cmd *cmd
 	// TODO: add real dry run support
 	cmd = newCmd("kubeadm", "upgrade", "apply", spec.KubernetesVersion, "--yes", "--v=4")

commands/upgrade_kubectlkubelet.go

@@ -64,19 +64,13 @@ func runUpgradeKubectlAndKubelet(spec *operatorv1.UpgradeKubeletAndKubeactlComma
 		return err
 	}
 
-	// TODO stop kubelet and replace it
-	cmd = newCmd("/usr/bin/cp", "-f", "/usr/bin/kubelet-"+spec.KubernetesVersion, "/usr/bin/kubelet")
+	// see https://github.com/pacoxu/kubelet-reloader/ to add a daemon or service on nodes to replace kubelet and restart kubelet.
+	cmd = newCmd("/usr/bin/cp", "-f", "/usr/bin/kubelet-"+spec.KubernetesVersion, "/usr/bin/kubelet-new")
 	start, err = cmd.RunAndCapture()
 	if err != nil {
-		// skip kubelet replacement
-		cmd = newCmd("/usr/bin/cp", "-f", "/usr/bin/kubelet-"+spec.KubernetesVersion, "/usr/bin/kubelet-new")
-		start, err = cmd.RunAndCapture()
-		if err != nil {
-		}
-		// return errors.WithStack(errors.WithMessage(err, strings.Join(start, "\n")))
+		return errors.WithStack(errors.WithMessage(err, strings.Join(start, "\n")))
 	} else {
 		log.Info(fmt.Sprintf("%s", strings.Join(start, "\n")))
 	}
-
 	return nil
 }

commands/upgrade_kubeproxy.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package commands
+
+import (
+	"github.com/go-logr/logr"
+
+	operatorv1 "k8s.io/kubeadm/operator/api/v1alpha1"
+)
+
+// TODO it depends on https://github.com/kubernetes/kubeadm/issues/1318
+// Currently, kubeadm upgrade apply doesn't support phase
+func runUpgradeKubeProxy(spec *operatorv1.KubeadmUpgradeKubeProxySpec, log logr.Logger) error {
+	return nil
+}

config/crd/bases/operator.kubeadm.x-k8s.io_operations.yaml

@@ -248,6 +248,23 @@ spec:
                                                   the version is empty, we will skip
                                                   this command.
                                                 type: string
+                                              skipKubePorxy:
+                                                description: skip kube-proxy upgrade
+                                                type: boolean
+                                            required:
+                                            - kubernetesVersion
+                                            type: object
+                                          kubeadmUpgradeKubeProxy:
+                                            description: not implemented yet
+                                            properties:
+                                              kubernetesVersion:
+                                                description: KubernetesVersion specifies
+                                                  the target kubernetes version If
+                                                  the version is empty, we will skip
+                                                  this command.
+                                                type: string
+                                            required:
+                                            - kubernetesVersion
                                             type: object
                                           kubeadmUpgradeNode:
                                             description: TODO download the specified
@@ -323,6 +340,8 @@ spec:
                                                   download the binary from official
                                                   websites
                                                 type: boolean
+                                            required:
+                                            - kubernetesVersion
                                             type: object
                                           wait:
                                             description: Wait pauses the execution
@@ -435,6 +454,16 @@ spec:
                       version
                     type: string
                   local:
+                    description: Local is used to determine whether to use local binary
+                      or download from internet.
+                    type: boolean
+                  upgradeKubeProxyAtLast:
+                    description: UpgradeKubeProxyAtLast by default is false. TODO
+                      UpgradeKubeProxyAtLast can be true by default if this should
+                      be default behavior, needs more disscussions. If this is true,
+                      kube-proxy will not be upgraded at first. See more details in
+                      https://github.com/kubernetes/kubeadm/issues/2346 Then kube-proxy
+                      will be upgraded after all apiserver are upgraded.
                     type: boolean
                 required:
                 - kubernetesVersion

config/crd/bases/operator.kubeadm.x-k8s.io_runtimetaskgroups.yaml

@@ -202,6 +202,22 @@ spec:
                                     kubernetes version If the version is empty, we
                                     will skip this command.
                                   type: string
+                                skipKubePorxy:
+                                  description: skip kube-proxy upgrade
+                                  type: boolean
+                              required:
+                              - kubernetesVersion
+                              type: object
+                            kubeadmUpgradeKubeProxy:
+                              description: not implemented yet
+                              properties:
+                                kubernetesVersion:
+                                  description: KubernetesVersion specifies the target
+                                    kubernetes version If the version is empty, we
+                                    will skip this command.
+                                  type: string
+                              required:
+                              - kubernetesVersion
                               type: object
                             kubeadmUpgradeNode:
                               description: TODO download the specified version bin
@@ -268,6 +284,8 @@ spec:
                                     operator will download the binary from official
                                     websites
                                   type: boolean
+                              required:
+                              - kubernetesVersion
                               type: object
                             wait:
                               description: Wait pauses the execution on the next command

config/crd/bases/operator.kubeadm.x-k8s.io_runtimetasks.yaml

@@ -83,6 +83,21 @@ spec:
                           description: KubernetesVersion specifies the target kubernetes
                             version If the version is empty, we will skip this command.
                           type: string
+                        skipKubePorxy:
+                          description: skip kube-proxy upgrade
+                          type: boolean
+                      required:
+                      - kubernetesVersion
+                      type: object
+                    kubeadmUpgradeKubeProxy:
+                      description: not implemented yet
+                      properties:
+                        kubernetesVersion:
+                          description: KubernetesVersion specifies the target kubernetes
+                            version If the version is empty, we will skip this command.
+                          type: string
+                      required:
+                      - kubernetesVersion
                       type: object
                     kubeadmUpgradeNode:
                       description: TODO download the specified version bin and replace
@@ -143,6 +158,8 @@ spec:
                             will use the binary in /usr/bin If not, operator will
                             download the binary from official websites
                           type: boolean
+                      required:
+                      - kubernetesVersion
                       type: object
                     wait:
                       description: Wait pauses the execution on the next command for

config/manager/kustomization.yaml

@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: daocloud.io/daocloud/kubeadm-operator
-  newTag: v0.0.4-dev
+  newTag: v0.0.5-dev

config/manager/manager.yaml

@@ -39,11 +39,11 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
-        image: daocloud.io/daocloud/kubeadm-operator:v0.0.4-dev
+        image: daocloud.io/daocloud/kubeadm-operator:v0.0.5-dev
         name: manager
         resources:
           limits:
-            cpu: 100m
+            cpu: 200m
             memory: 30Mi
           requests:
             cpu: 100m

operations/upgrade.go

@@ -89,6 +89,7 @@ func planUpgrade(operation *operatorv1.Operation, spec *operatorv1.UpgradeOperat
 				KubeadmUpgradeApply: &operatorv1.KubeadmUpgradeApplyCommandSpec{
 					DryRun:            dryRun,
 					KubernetesVersion: operation.Spec.Upgrade.KubernetesVersion,
+					SkipKubeProxy:     operation.Spec.Upgrade.UpgradeKubeProxyAtLast,
 				},
 			},
 		)
@@ -128,17 +129,29 @@ func planUpgrade(operation *operatorv1.Operation, spec *operatorv1.UpgradeOperat
 		items = append(items, t2)
 	}
 
+	if operation.Spec.Upgrade.UpgradeKubeProxyAtLast {
+		t3 := createBasicTaskGroup(operation, "03", "upgrade-kube-proxy")
+		t3.Spec.Template.Spec.Commands = append(t3.Spec.Template.Spec.Commands,
+			operatorv1.CommandDescriptor{
+				KubeadmUpgradeKubeProxy: &operatorv1.KubeadmUpgradeKubeProxySpec{
+					KubernetesVersion: operation.Spec.Upgrade.KubernetesVersion,
+				},
+			},
+		)
+		items = append(items, t3)
+	}
+
 	// this can be skipped if there are no worker nodes.
 	// currently it depends on the selector
-	t3 := createBasicTaskGroup(operation, "02", "upgrade-w")
-	setWSelector(&t3)
-	workerNodes, err := listNodesBySelector(c, &t3.Spec.NodeSelector)
+	t4 := createBasicTaskGroup(operation, "04", "upgrade-w")
+	setWSelector(&t4)
+	workerNodes, err := listNodesBySelector(c, &t4.Spec.NodeSelector)
 	if err != nil {
 		fmt.Printf("failed to list nodes: %v", err)
 		return nil
 	}
 	if len(workerNodes.Items) > 0 {
-		t3.Spec.Template.Spec.Commands = append(t3.Spec.Template.Spec.Commands,
+		t4.Spec.Template.Spec.Commands = append(t4.Spec.Template.Spec.Commands,
 			operatorv1.CommandDescriptor{
 				KubectlDrain: &operatorv1.KubectlDrainCommandSpec{},
 			},
@@ -163,7 +176,7 @@ func planUpgrade(operation *operatorv1.Operation, spec *operatorv1.UpgradeOperat
 				KubectlUncordon: &operatorv1.KubectlUncordonCommandSpec{},
 			},
 		)
-		items = append(items, t3)
+		items = append(items, t4)
 	}
 
 	return &operatorv1.RuntimeTaskGroupList{