Merge remote-tracking branch 'origin/master' into release-3.1

.all-contributorsrc

@@ -760,6 +760,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "yzxiu",
+      "name": "yuyu",
+      "avatar_url": "https://avatars.githubusercontent.com/u/13790023?v=4",
+      "profile": "https://github.com/yzxiu",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.github/workflows/build-multiarch.yaml

@@ -21,12 +21,12 @@ jobs:
           fetch-depth: 0
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
         with:
           platforms: all
 
       - name: Set up Docker buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build and push docker images
         env:

.github/workflows/gen-repository-iso.yaml

@@ -35,13 +35,13 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build iso image to local
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: hack/gen-repository-iso
           file: hack/gen-repository-iso/${{ matrix.dockerfile }}

.github/workflows/kubernetes-auto-support.yaml

@@ -56,7 +56,7 @@ jobs:
         if: steps.get_new_version.outputs.UPDATE_VERSION == 'true'
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v4
+        uses: peter-evans/create-pull-request@v5
         with:
           commit-message: Add new kubernetes version
           committer: GitHub <[email protected]>

CONTRIBUTORS.md

@@ -112,6 +112,7 @@ Contributions of any kind are welcome! Thanks goes to these wonderful contributo
       <td align="center" valign="top" width="14.28%"><a href="https://baimeow.cn/"><img src="https://avatars.githubusercontent.com/u/38121125?v=4?s=100" width="100px;" alt="柏喵Sakura"/><br /><sub><b>柏喵Sakura</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=BaiMeow" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://dashen.tech"><img src="https://avatars.githubusercontent.com/u/15921519?v=4?s=100" width="100px;" alt="cui fliter"/><br /><sub><b>cui fliter</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=cuishuang" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/liuxu623"><img src="https://avatars.githubusercontent.com/u/9653438?v=4?s=100" width="100px;" alt="刘旭"/><br /><sub><b>刘旭</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=liuxu623" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/yzxiu"><img src="https://avatars.githubusercontent.com/u/13790023?v=4?s=100" width="100px;" alt="yuyu"/><br /><sub><b>yuyu</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=yzxiu" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

README.md

@@ -396,6 +396,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://baimeow.cn/"><img src="https://avatars.githubusercontent.com/u/38121125?v=4?s=100" width="100px;" alt="柏喵Sakura"/><br /><sub><b>柏喵Sakura</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=BaiMeow" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://dashen.tech"><img src="https://avatars.githubusercontent.com/u/15921519?v=4?s=100" width="100px;" alt="cui fliter"/><br /><sub><b>cui fliter</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=cuishuang" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/liuxu623"><img src="https://avatars.githubusercontent.com/u/9653438?v=4?s=100" width="100px;" alt="刘旭"/><br /><sub><b>刘旭</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=liuxu623" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/yzxiu"><img src="https://avatars.githubusercontent.com/u/13790023?v=4?s=100" width="100px;" alt="yuyu"/><br /><sub><b>yuyu</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=yzxiu" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

README_zh-CN.md

@@ -413,6 +413,7 @@ kubectl completion bash >/etc/bash_completion.d/kubectl
       <td align="center" valign="top" width="14.28%"><a href="https://baimeow.cn/"><img src="https://avatars.githubusercontent.com/u/38121125?v=4?s=100" width="100px;" alt="柏喵Sakura"/><br /><sub><b>柏喵Sakura</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=BaiMeow" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://dashen.tech"><img src="https://avatars.githubusercontent.com/u/15921519?v=4?s=100" width="100px;" alt="cui fliter"/><br /><sub><b>cui fliter</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=cuishuang" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/liuxu623"><img src="https://avatars.githubusercontent.com/u/9653438?v=4?s=100" width="100px;" alt="刘旭"/><br /><sub><b>刘旭</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=liuxu623" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/yzxiu"><img src="https://avatars.githubusercontent.com/u/13790023?v=4?s=100" width="100px;" alt="yuyu"/><br /><sub><b>yuyu</b></sub></a><br /><a href="https://github.com/kubesphere/kubekey/commits?author=yzxiu" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

cmd/kk/apis/kubekey/v1alpha2/default.go

@@ -42,10 +42,10 @@ const (
 	DefaultEtcdVersion             = "v3.5.6"
 	DefaultEtcdPort                = "2379"
 	DefaultDockerVersion           = "24.0.6"
-	DefaultContainerdVersion       = "1.7.8"
-	DefaultRuncVersion             = "v1.1.10"
-	DefaultCrictlVersion           = "v1.24.0"
-	DefaultKubeVersion             = "v1.23.10"
+	DefaultContainerdVersion       = "1.7.12"
+	DefaultRuncVersion             = "v1.1.11"
+	DefaultCrictlVersion           = "v1.29.0"
+	DefaultKubeVersion             = "v1.23.15"
 	DefaultCalicoVersion           = "v3.26.1"
 	DefaultFlannelVersion          = "v0.21.3"
 	DefaultFlannelCniPluginVersion = "v1.1.2"
@@ -54,7 +54,7 @@ const (
 	DefaulthybridnetVersion        = "v0.8.6"
 	DefaultKubeovnVersion          = "v1.10.6"
 	DefalutMultusVersion           = "v3.8"
-	DefaultHelmVersion             = "v3.9.0"
+	DefaultHelmVersion             = "v3.13.3"
 	DefaultDockerComposeVersion    = "v2.2.2"
 	DefaultRegistryVersion         = "2"
 	DefaultHarborVersion           = "v2.5.3"

cmd/kk/pkg/artifact/templates/manifest.go

@@ -66,6 +66,8 @@ spec:
     - type: {{ $v.Type }}
       version: {{ $v.Version }}
     {{- end}}
+    calicoctl:
+      version: {{ .Options.Components.Calicoctl.Version }}
     crictl: 
       version: {{ .Options.Components.Crictl.Version }}
     ## 

cmd/kk/pkg/bootstrap/registry/certs.go

@@ -105,7 +105,7 @@ func (g *GenerateCerts) Execute(runtime connector.Runtime) error {
 
 	var altName cert.AltNames
 
-	dnsList := []string{"localhost", RegistryCertificateBaseName}
+	dnsList := []string{"localhost", g.KubeConf.Cluster.Registry.PrivateRegistry}
 	ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}
 
 	for _, h := range runtime.GetHostsByRole(common.Registry) {

cmd/kk/pkg/bootstrap/registry/tasks.go

@@ -18,11 +18,12 @@ package registry
 
 import (
 	"fmt"
+	"path/filepath"
+	"strings"
+
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry/templates"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
-	"path/filepath"
-	"strings"
 
 	"github.com/pkg/errors"
 
@@ -220,15 +221,21 @@ type GenerateHarborConfig struct {
 }
 
 func (g *GenerateHarborConfig) Execute(runtime connector.Runtime) error {
-	host := runtime.RemoteHost()
+	registryDomain := g.KubeConf.Cluster.Registry.PrivateRegistry
+
+	if g.KubeConf.Cluster.Registry.Type == "harbor-ha" {
+		host := runtime.RemoteHost()
+		registryDomain = host.GetName()
+	}
+
 	templateAction := action.Template{
 		Template: templates.HarborConfigTempl,
 		Dst:      "/opt/harbor/harbor.yml",
 		Data: util.Data{
-			"Domain":      host.GetName(),
-			"Certificate": fmt.Sprintf("%s.pem", RegistryCertificateBaseName),
-			"Key":         fmt.Sprintf("%s-key.pem", RegistryCertificateBaseName),
-			"Password":    templates.Password(g.KubeConf, RegistryCertificateBaseName),
+			"Domain":      registryDomain,
+			"Certificate": fmt.Sprintf("%s.pem", g.KubeConf.Cluster.Registry.PrivateRegistry),
+			"Key":         fmt.Sprintf("%s-key.pem", g.KubeConf.Cluster.Registry.PrivateRegistry),
+			"Password":    templates.Password(g.KubeConf, g.KubeConf.Cluster.Registry.PrivateRegistry),
 		},
 	}
 	templateAction.Init(nil, nil)

cmd/kk/pkg/container/docker.go

@@ -64,6 +64,19 @@ func (s *SyncDockerBinaries) Execute(runtime connector.Runtime) error {
 	return nil
 }
 
+type EnableContainerdForDocker struct {
+	common.KubeAction
+}
+
+func (e *EnableContainerdForDocker) Execute(runtime connector.Runtime) error {
+	if _, err := runtime.GetRunner().SudoCmd(
+		"systemctl daemon-reload && systemctl enable containerd &&  systemctl start containerd",
+		false); err != nil {
+		return errors.Wrap(errors.WithStack(err), fmt.Sprintf("enable and start containerd failed"))
+	}
+	return nil
+}
+
 type EnableDocker struct {
 	common.KubeAction
 }

cmd/kk/pkg/container/module.go

@@ -73,6 +73,21 @@ func InstallDocker(m *InstallContainerModule) []task.Interface {
 		Retry:    2,
 	}
 
+	generateContainerdService := &task.RemoteTask{
+		Name:  "GenerateContainerdService",
+		Desc:  "Generate containerd service",
+		Hosts: m.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&kubernetes.NodeInCluster{Not: true},
+			&ContainerdExist{Not: true},
+		},
+		Action: &action.Template{
+			Template: templates.ContainerdService,
+			Dst:      filepath.Join("/etc/systemd/system", templates.ContainerdService.Name()),
+		},
+		Parallel: true,
+	}
+
 	generateDockerService := &task.RemoteTask{
 		Name:  "GenerateDockerService",
 		Desc:  "Generate docker service",
@@ -109,6 +124,18 @@ func InstallDocker(m *InstallContainerModule) []task.Interface {
 		Parallel: true,
 	}
 
+	enableContainerdForDocker := &task.RemoteTask{
+		Name:  "EnableContainerd",
+		Desc:  "Enable containerd",
+		Hosts: m.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&kubernetes.NodeInCluster{Not: true},
+			&ContainerdExist{Not: true},
+		},
+		Action:   new(EnableContainerdForDocker),
+		Parallel: true,
+	}
+
 	enableDocker := &task.RemoteTask{
 		Name:  "EnableDocker",
 		Desc:  "Enable docker",
@@ -136,8 +163,10 @@ func InstallDocker(m *InstallContainerModule) []task.Interface {
 
 	return []task.Interface{
 		syncBinaries,
+		generateContainerdService,
 		generateDockerService,
 		generateDockerConfig,
+		enableContainerdForDocker,
 		enableDocker,
 		dockerLoginRegistry,
 	}

cmd/kk/pkg/container/prepares.go

@@ -29,7 +29,7 @@ type DockerExist struct {
 }
 
 func (d *DockerExist) PreCheck(runtime connector.Runtime) (bool, error) {
-	output, err := runtime.GetRunner().SudoCmd("if [ -z $(which docker) ] || [ ! -e /var/run/docker.sock ]; "+
+	output, err := runtime.GetRunner().SudoCmd("if [ -z $(command -v docker) ] || [ ! -e /var/run/docker.sock ]; "+
 		"then echo 'not exist'; "+
 		"fi", false)
 	if err != nil {
@@ -48,7 +48,7 @@ type CrictlExist struct {
 
 func (c *CrictlExist) PreCheck(runtime connector.Runtime) (bool, error) {
 	output, err := runtime.GetRunner().SudoCmd(
-		"if [ -z $(which crictl) ]; "+
+		"if [ -z $(command -v crictl) ]; "+
 			"then echo 'not exist'; "+
 			"fi", false)
 	if err != nil {
@@ -68,7 +68,7 @@ type ContainerdExist struct {
 
 func (c *ContainerdExist) PreCheck(runtime connector.Runtime) (bool, error) {
 	output, err := runtime.GetRunner().SudoCmd(
-		"if [ -z $(which containerd) ] || [ ! -e /run/containerd/containerd.sock ]; "+
+		"if [ -z $(command -v containerd) ] || [ ! -e /run/containerd/containerd.sock ]; "+
 			"then echo 'not exist'; "+
 			"fi", false)
 	if err != nil {

cmd/kk/pkg/container/templates/docker_service.go

@@ -34,7 +34,7 @@ Type=notify
 # the default is not to use systemd for cgroups because the delegate issues still
 # exists and systemd currently does not support the cgroup feature set required
 # for containers run by docker
-ExecStart=/usr/bin/dockerd
+ExecStart=/usr/bin/dockerd  --containerd=/run/containerd/containerd.sock
 ExecReload=/bin/kill -s HUP $MAINPID
 TimeoutSec=0
 RestartSec=2

cmd/kk/pkg/core/util/ip.go

@@ -19,6 +19,7 @@ package util
 import (
 	"encoding/binary"
 	"net"
+	"os"
 	"strconv"
 	"strings"
 
@@ -159,10 +160,20 @@ func GetLocalIP() (string, error) {
 	return "", errors.New("valid local IP not found!")
 }
 
-func LocalIP() string {
-	localIp, err := GetLocalIP()
+func isValidIPv4(v string) bool {
+	ip := net.ParseIP(v)
+	return ip != nil && ip.To4() != nil && !ip.IsLoopback() && ip.IsGlobalUnicast()
+}
+
+func LocalIP() (localIp string) {
+	if localIp = os.Getenv("KKLOCALIP"); localIp != "" && isValidIPv4(localIp) {
+		return
+	}
+
+	var err error
+	localIp, err = GetLocalIP()
 	if err != nil {
 		logger.Log.Fatalf("Failed to get Local IP: %v", err)
 	}
-	return localIp
+	return
 }

cmd/kk/pkg/etcd/templates/etcd_env.go

@@ -17,8 +17,9 @@
 package templates
 
 import (
-	"github.com/lithammer/dedent"
 	"text/template"
+
+	"github.com/lithammer/dedent"
 )
 
 // EtcdEnv defines the template of etcd's env.
@@ -91,7 +92,7 @@ ETCD_CLIENT_CERT_AUTH=true
 ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem
 ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/member-{{ .Hostname }}.pem
 ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/member-{{ .Hostname }}-key.pem
-ETCD_PEER_CLIENT_CERT_AUTH=True
+ETCD_PEER_CLIENT_CERT_AUTH=true
 
 # CLI settings
 ETCDCTL_ENDPOINTS=https://127.0.0.1:2379

cmd/kk/pkg/images/tasks.go

@@ -321,6 +321,8 @@ func (c *CopyImagesToRegistry) Execute(runtime connector.Runtime) error {
 		for ; retry < maxRetry; retry++ {
 			if err := o.Copy(); err == nil {
 				break
+			} else {
+				fmt.Println(errors.WithStack(err))
 			}
 		}
 		if retry >= maxRetry {

cmd/kk/pkg/kubernetes/tasks.go

@@ -1131,11 +1131,11 @@ func (k *MasterSecurityEnhancemenAction) Execute(runtime connector.Runtime) erro
 	chmodKubernetesConfigCmd := "chmod 600 -R /etc/kubernetes"
 	chownKubernetesConfigCmd := "chown root:root -R /etc/kubernetes/*"
 
-	chmodKubenretesManifestsDirCmd := "chmod 644 /etc/kubernetes/manifests"
-	chownKubenretesManifestsDirCmd := "chown root:root /etc/kubernetes/manifests"
+	chmodKubernetesManifestsDirCmd := "chmod 644 /etc/kubernetes/manifests"
+	chownKubernetesManifestsDirCmd := "chown root:root /etc/kubernetes/manifests"
 
-	chmodKubenretesCertsDirCmd := "chmod 644 /etc/kubernetes/pki"
-	chownKubenretesCertsDirCmd := "chown root:root /etc/kubernetes/pki"
+	chmodKubernetesCertsDirCmd := "chmod 644 /etc/kubernetes/pki"
+	chownKubernetesCertsDirCmd := "chown root:root /etc/kubernetes/pki"
 
 	// node Security Enhancemen
 	chmodCniConfigDir := "chmod 600 -R /etc/cni/net.d"
@@ -1159,11 +1159,11 @@ func (k *MasterSecurityEnhancemenAction) Execute(runtime connector.Runtime) erro
 	chmodCertsRenew := "chmod 640 /etc/systemd/system/k8s-certs-renew*"
 	chownCertsRenew := "chown root:root /etc/systemd/system/k8s-certs-renew*"
 
-	chmodMasterCmds := []string{chmodKubernetesConfigCmd, chmodKubernetesDirCmd, chmodKubenretesManifestsDirCmd, chmodKubenretesCertsDirCmd}
+	chmodMasterCmds := []string{chmodKubernetesConfigCmd, chmodKubernetesDirCmd, chmodKubernetesManifestsDirCmd, chmodKubernetesCertsDirCmd}
 	if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodMasterCmds, " && "), true); err != nil {
 		return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
 	}
-	chownMasterCmds := []string{chownKubernetesConfigCmd, chownKubernetesDirCmd, chownKubenretesManifestsDirCmd, chownKubenretesCertsDirCmd}
+	chownMasterCmds := []string{chownKubernetesConfigCmd, chownKubernetesDirCmd, chownKubernetesManifestsDirCmd, chownKubernetesCertsDirCmd}
 	if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownMasterCmds, " && "), true); err != nil {
 		return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
 	}
@@ -1193,11 +1193,11 @@ func (n *NodesSecurityEnhancemenAction) Execute(runtime connector.Runtime) error
 	chmodKubernetesConfigCmd := "chmod 600 -R /etc/kubernetes"
 	chownKubernetesConfigCmd := "chown root:root -R /etc/kubernetes"
 
-	chmodKubenretesManifestsDirCmd := "chmod 644 /etc/kubernetes/manifests"
-	chownKubenretesManifestsDirCmd := "chown root:root /etc/kubernetes/manifests"
+	chmodKubernetesManifestsDirCmd := "chmod 644 /etc/kubernetes/manifests"
+	chownKubernetesManifestsDirCmd := "chown root:root /etc/kubernetes/manifests"
 
-	chmodKubenretesCertsDirCmd := "chmod 644 /etc/kubernetes/pki"
-	chownKubenretesCertsDirCmd := "chown root:root /etc/kubernetes/pki"
+	chmodKubernetesCertsDirCmd := "chmod 644 /etc/kubernetes/pki"
+	chownKubernetesCertsDirCmd := "chown root:root /etc/kubernetes/pki"
 
 	// node Security Enhancemen
 	chmodCniConfigDir := "chmod 600 -R /etc/cni/net.d"
@@ -1218,11 +1218,11 @@ func (n *NodesSecurityEnhancemenAction) Execute(runtime connector.Runtime) error
 	chmodKubeletConfig := "chmod 640 /var/lib/kubelet/config.yaml && chmod 640 -R /etc/systemd/system/kubelet.service*"
 	chownKubeletConfig := "chown root:root /var/lib/kubelet/config.yaml && chown root:root -R /etc/systemd/system/kubelet.service*"
 
-	chmodMasterCmds := []string{chmodKubernetesConfigCmd, chmodKubernetesDirCmd, chmodKubenretesManifestsDirCmd, chmodKubenretesCertsDirCmd}
+	chmodMasterCmds := []string{chmodKubernetesConfigCmd, chmodKubernetesDirCmd, chmodKubernetesManifestsDirCmd, chmodKubernetesCertsDirCmd}
 	if _, err := runtime.GetRunner().SudoCmd(strings.Join(chmodMasterCmds, " && "), true); err != nil {
 		return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
 	}
-	chownMasterCmds := []string{chownKubernetesConfigCmd, chownKubernetesDirCmd, chownKubenretesManifestsDirCmd, chownKubenretesCertsDirCmd}
+	chownMasterCmds := []string{chownKubernetesConfigCmd, chownKubernetesDirCmd, chownKubernetesManifestsDirCmd, chownKubernetesCertsDirCmd}
 	if _, err := runtime.GetRunner().SudoCmd(strings.Join(chownMasterCmds, " && "), true); err != nil {
 		return errors.Wrap(errors.WithStack(err), "Updating permissions failed.")
 	}