Merge remote-tracking branch 'origin/master' into release-3.1

.github/workflows/build-multiarch.yaml

@@ -16,7 +16,7 @@ jobs:
         run: echo "RELEASE_TAG=${GITHUB_REF:10}" >> $GITHUB_ENV
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.github/workflows/ci.yml

@@ -20,13 +20,13 @@ jobs:
     steps:
 
       - name: Set up Go 1.19
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version: 1.19
         id: go
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 

.github/workflows/gen-repository-iso.yaml

@@ -32,7 +32,7 @@ jobs:
             dockerfile: dockerfile.ubuntu2204
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -56,7 +56,7 @@ jobs:
           sha256sum *.iso >  ${{ matrix.name }}.iso.sha256sum.txt
 
       - name: Wait for release workflow to finish
-        uses: lewagon/[email protected].1
+        uses: lewagon/[email protected].3
         with:
           ref: ${{ github.ref }}
           check-name: 'create draft release'

.github/workflows/golangci-lint.yml

@@ -17,12 +17,12 @@ jobs:
           - ""
           - test
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: 1.19
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3.6.0
+        uses: golangci/golangci-lint-action@v4.0.0
         with:
           version: v1.50.1
           working-directory: ${{matrix.working-directory}}

.github/workflows/kubernetes-auto-support.yaml

@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository == 'kubesphere/kubekey'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Go 1.19
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version: 1.19
         id: go
@@ -56,7 +56,7 @@ jobs:
         if: steps.get_new_version.outputs.UPDATE_VERSION == 'true'
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v6
         with:
           commit-message: Add new kubernetes version
           committer: GitHub <[email protected]>

.github/workflows/release-drafter.yml

@@ -9,6 +9,6 @@ jobs:
   update_release_draft:
     runs-on: ubuntu-latest
     steps:
-      - uses: release-drafter/release-drafter@v5
+      - uses: release-drafter/release-drafter@v6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -13,11 +13,11 @@ jobs:
       - name: Set env
         run:  echo "RELEASE_TAG=${GITHUB_REF:10}" >> $GITHUB_ENV
       - name: checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Install go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version: '^1.19'
       - name: generate release artifacts

cmd/kk/apis/kubekey/v1alpha2/cluster_types.go

@@ -140,12 +140,19 @@ func (cfg *ClusterSpec) GenerateCertSANs() []string {
 		if host.Address != cfg.ControlPlaneEndpoint.Address {
 			extraCertSANs = append(extraCertSANs, host.Address)
 		}
-		if host.InternalAddress != host.Address && host.InternalAddress != cfg.ControlPlaneEndpoint.Address {
-			extraCertSANs = append(extraCertSANs, host.InternalAddress)
+
+		nodeAddresses := strings.Split(host.InternalAddress, ",")
+		InternalIPv4Address := nodeAddresses[0]
+		if InternalIPv4Address != host.Address && InternalIPv4Address != cfg.ControlPlaneEndpoint.Address {
+			extraCertSANs = append(extraCertSANs, InternalIPv4Address)
+		}
+		if len(nodeAddresses)==2 {
+			InternalIPv6Address := nodeAddresses[1]
+			extraCertSANs = append(extraCertSANs, InternalIPv6Address)
 		}
 	}
 
-	extraCertSANs = append(extraCertSANs, util.ParseIp(cfg.Network.KubeServiceCIDR)[0])
+	extraCertSANs = append(extraCertSANs, util.ParseIp(strings.Split(cfg.Network.KubeServiceCIDR, ",")[0])[0])
 
 	defaultCertSANs = append(defaultCertSANs, extraCertSANs...)
 
@@ -210,12 +217,12 @@ func toHosts(cfg HostCfg) *KubeHost {
 
 // ClusterIP is used to get the kube-apiserver service address inside the cluster.
 func (cfg *ClusterSpec) ClusterIP() string {
-	return util.ParseIp(cfg.Network.KubeServiceCIDR)[0]
+	return util.ParseIp(strings.Split(cfg.Network.KubeServiceCIDR, ",")[0])[0]
 }
 
 // CorednsClusterIP is used to get the coredns service address inside the cluster.
 func (cfg *ClusterSpec) CorednsClusterIP() string {
-	return util.ParseIp(cfg.Network.KubeServiceCIDR)[2]
+	return util.ParseIp(strings.Split(cfg.Network.KubeServiceCIDR, ",")[0])[2]
 }
 
 // ClusterDNS is used to get the dns server address inside the cluster.

cmd/kk/apis/kubekey/v1alpha2/default.go

@@ -22,7 +22,6 @@ import (
 	"strings"
 
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
-	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
 )
 
 const (
@@ -43,12 +42,12 @@ const (
 	DefaultEtcdVersion             = "v3.5.6"
 	DefaultEtcdPort                = "2379"
 	DefaultDockerVersion           = "24.0.9"
-	DefaultCriDockerdVersion       = "0.3.9"
-	DefaultContainerdVersion       = "1.7.12"
-	DefaultRuncVersion             = "v1.1.11"
+	DefaultCriDockerdVersion       = "0.3.10"
+	DefaultContainerdVersion       = "1.7.13"
+	DefaultRuncVersion             = "v1.1.12"
 	DefaultCrictlVersion           = "v1.29.0"
 	DefaultKubeVersion             = "v1.23.15"
-	DefaultCalicoVersion           = "v3.26.1"
+	DefaultCalicoVersion           = "v3.27.2"
 	DefaultFlannelVersion          = "v0.21.3"
 	DefaultFlannelCniPluginVersion = "v1.1.2"
 	DefaultCniVersion              = "v1.2.0"
@@ -203,7 +202,7 @@ func SetDefaultLBCfg(cfg *ClusterSpec, masterGroup []*KubeHost) ControlPlaneEndp
 	}
 
 	if (cfg.ControlPlaneEndpoint.Address == "" && !cfg.ControlPlaneEndpoint.EnableExternalDNS()) || cfg.ControlPlaneEndpoint.Address == "127.0.0.1" {
-		cfg.ControlPlaneEndpoint.Address = masterGroup[0].InternalAddress
+		cfg.ControlPlaneEndpoint.Address = masterGroup[0].GetInternalIPv4Address()
 	}
 	if cfg.ControlPlaneEndpoint.Domain == "" {
 		cfg.ControlPlaneEndpoint.Domain = DefaultLBDomain
@@ -318,7 +317,7 @@ func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes {
 	if cfg.Kubernetes.ContainerRuntimeEndpoint == "" {
 		switch cfg.Kubernetes.ContainerManager {
 		case Docker:
-			if kubernetes.IsAtLeastV124(cfg.Kubernetes.Version) {
+			if cfg.Kubernetes.IsAtLeastV124() {
 				cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultCriDockerdEndpoint
 			} else {
 				cfg.Kubernetes.ContainerRuntimeEndpoint = ""

cmd/kk/apis/kubekey/v1alpha2/kubernetes_types.go

@@ -16,7 +16,10 @@
 
 package v1alpha2
 
-import "k8s.io/apimachinery/pkg/runtime"
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	versionutil "k8s.io/apimachinery/pkg/util/version"
+)
 
 // Kubernetes contains the configuration for the cluster
 type Kubernetes struct {
@@ -103,3 +106,17 @@ func (k *Kubernetes) EnableAudit() bool {
 	}
 	return *k.AutoRenewCerts
 }
+
+// IsAtLeastV124 is used to determine whether the k8s version is greater than v1.24.
+func (k *Kubernetes) IsAtLeastV124() bool {
+	parsedVersion, err := versionutil.ParseGeneric(k.Version)
+	if err != nil {
+		return false
+	}
+
+	if parsedVersion.AtLeast(versionutil.MustParseSemantic("v1.24.0")) {
+		return true
+	}
+
+	return false
+}

cmd/kk/pkg/binaries/kubernetes.go

@@ -26,7 +26,6 @@ import (
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files"
-	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
 	"github.com/pkg/errors"
 )
 
@@ -50,7 +49,7 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string,
 
 	if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Docker {
 		binaries = append(binaries, docker)
-		if kubernetes.IsAtLeastV124(kubeConf.Cluster.Kubernetes.Version) && kubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
+		if kubeConf.Cluster.Kubernetes.IsAtLeastV124() && kubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
 			binaries = append(binaries, criDockerd)
 		}
 	} else if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Containerd {

cmd/kk/pkg/bootstrap/confirm/tasks.go

@@ -28,7 +28,6 @@ import (
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
-	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
 	"github.com/mitchellh/mapstructure"
 	"github.com/modood/table"
 	"github.com/pkg/errors"
@@ -107,17 +106,18 @@ func (i *InstallationConfirm) Execute(runtime connector.Runtime) error {
 	fmt.Println("https://github.com/kubesphere/kubekey#requirements-and-recommendations")
 	fmt.Println("")
 
-	if kubernetes.IsAtLeastV124(i.KubeConf.Cluster.Kubernetes.Version) && i.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker &&
-		i.KubeConf.Cluster.Kubernetes.Type != common.Kubernetes {
+	if i.KubeConf.Cluster.Kubernetes.IsAtLeastV124() && i.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
 		fmt.Println("[Notice]")
-		fmt.Println("Incorrect runtime. Please specify a container runtime other than Docker to install Kubernetes v1.24 or later.")
+		fmt.Println("For Kubernetes v1.24 and later, dockershim has been deprecated.")
+		fmt.Println("Current runtime is set to Docker and `cri-dockerd` will be installed to support Kubernetes v1.24 and later.")
+		fmt.Println("Yoc can also specify a container runtime other than Docker to install Kubernetes v1.24 or later.")
 		fmt.Println("You can set \"spec.kubernetes.containerManager\" in the configuration file to \"containerd\" or add \"--container-manager containerd\" to the \"./kk create cluster\" command.")
 		fmt.Println("For more information, see:")
 		fmt.Println("https://github.com/kubesphere/kubekey/blob/master/docs/commands/kk-create-cluster.md")
 		fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes")
 		fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/")
+		fmt.Println("https://github.com/Mirantis/cri-dockerd")
 		fmt.Println("")
-		stopFlag = true
 	}
 
 	if stopFlag {
@@ -260,17 +260,32 @@ Warning:
 			k8sV124 := versionutil.MustParseSemantic("v1.24.0")
 			if k8sVersion.AtLeast(k8sV124) && versionutil.MustParseSemantic(currentK8sVersion).LessThan(k8sV124) && strings.Contains(cri, "docker") {
 				fmt.Println("[Notice]")
-				fmt.Println("Pre-upgrade check failed. The container runtime of the current cluster is Docker.")
-				fmt.Println("Kubernetes v1.24 and later no longer support dockershim and Docker.")
-				fmt.Println("Make sure you have completed the migration from Docker to other container runtimes that are compatible with the Kubernetes CRI.")
+				fmt.Println("For Kubernetes v1.24 and later, dockershim has been deprecated.")
+				fmt.Println("The container runtime of the current cluster is Docker, `cri-dockerd` will be installed to support Kubernetes v1.24 and later.")
+				fmt.Println("You can also migrate container runtime from Docker to other runtimes that are compatible with the Kubernetes CRI.")
 				fmt.Println("For more information, see:")
 				fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes")
 				fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/")
+				fmt.Println("https://github.com/Mirantis/cri-dockerd")
+				fmt.Println("https://kubernetes.io/docs/tasks/administer-cluster/migrating-from-dockershim/change-runtime-containerd/")
 				fmt.Println("")
 			}
 		}
 	}
 
+	if featureGates, ok := u.PipelineCache.GetMustString(common.ClusterFeatureGates); ok {
+		if featureGates != "" {
+			fmt.Println("[Notice]")
+			fmt.Println("The feature-gates in the cluster is as follow:")
+			fmt.Println("")
+			fmt.Printf("         %s\n", featureGates)
+			fmt.Println("")
+			fmt.Println("Please ensure that there are no deprecated feature-gate in the target version.")
+			fmt.Println("You can modify the feature-gates in `kubeadm-config` and `kubelet-config` configmaps in the kube-system namespace.")
+			fmt.Println("")
+		}
+	}
+
 	reader := bufio.NewReader(os.Stdin)
 	confirmOK := false
 	for !confirmOK {

cmd/kk/pkg/bootstrap/os/tasks.go

@@ -512,12 +512,16 @@ func (n *NodeConfigureNtpServer) Execute(runtime connector.Runtime) error {
 		return errors.Wrapf(err, "delete old servers failed, please check file %s", chronyConfigFile)
 	}
 
+	poolDisableCmd := fmt.Sprintf(`sed -i 's/^pool /#pool /g' %s`, chronyConfigFile)
+	if _, err := runtime.GetRunner().SudoCmd(poolDisableCmd, false); err != nil {
+		return errors.Wrapf(err, "set pool disable failed")
+	}
 	// if NtpServers was configured
 	for _, server := range n.KubeConf.Cluster.System.NtpServers {
 
 		serverAddr := strings.Trim(server, " \"")
 		fmt.Printf("ntpserver: %s, current host: %s\n", serverAddr, currentHost.GetName())
-		if serverAddr == currentHost.GetName() || serverAddr == currentHost.GetInternalAddress() {
+		if serverAddr == currentHost.GetName() || serverAddr == currentHost.GetInternalIPv4Address() {
 			deleteAllowCmd := fmt.Sprintf(`sed -i '/^allow/d' %s`, chronyConfigFile)
 			if _, err := runtime.GetRunner().SudoCmd(deleteAllowCmd, false); err != nil {
 				return errors.Wrapf(err, "delete allow failed, please check file %s", chronyConfigFile)
@@ -539,7 +543,7 @@ func (n *NodeConfigureNtpServer) Execute(runtime connector.Runtime) error {
 		// use internal ip to client chronyd server
 		for _, host := range runtime.GetAllHosts() {
 			if serverAddr == host.GetName() {
-				serverAddr = host.GetInternalAddress()
+				serverAddr = host.GetInternalIPv4Address()
 				break
 			}
 		}

cmd/kk/pkg/bootstrap/os/templates/init_script.go

@@ -18,9 +18,10 @@ package templates
 
 import (
 	"fmt"
-	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry"
 	"text/template"
 
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry"
+
 	"github.com/lithammer/dedent"
 
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
@@ -94,6 +95,12 @@ echo 'kernel.pid_max = 65535' >> /etc/sysctl.conf
 echo 'kernel.watchdog_thresh = 5' >> /etc/sysctl.conf
 echo 'kernel.hung_task_timeout_secs = 5' >> /etc/sysctl.conf
 
+#add for ipv6
+echo 'net.ipv6.conf.all.disable_ipv6 = 0' >> /etc/sysctl.conf
+echo 'net.ipv6.conf.default.disable_ipv6 = 0' >> /etc/sysctl.conf
+echo 'net.ipv6.conf.lo.disable_ipv6 = 0' >> /etc/sysctl.conf
+echo 'net.ipv6.conf.all.forwarding=1' >> /etc/sysctl.conf
+
 #See https://help.aliyun.com/document_detail/118806.html#uicontrol-e50-ddj-w0y
 sed -r -i "s@#{0,}?net.ipv4.tcp_tw_recycle ?= ?(0|1|2)@net.ipv4.tcp_tw_recycle = 0@g" /etc/sysctl.conf
 sed -r -i "s@#{0,}?net.ipv4.tcp_tw_reuse ?= ?(0|1)@net.ipv4.tcp_tw_reuse = 0@g" /etc/sysctl.conf
@@ -221,12 +228,19 @@ func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) [
 
 	if kubeConf.Cluster.ControlPlaneEndpoint.Address != "" {
 		lbHost = fmt.Sprintf("%s  %s", kubeConf.Cluster.ControlPlaneEndpoint.Address, kubeConf.Cluster.ControlPlaneEndpoint.Domain)
+	} else {
+		lbHost = fmt.Sprintf("%s  %s", runtime.GetHostsByRole(common.Master)[0].GetInternalIPv4Address(), kubeConf.Cluster.ControlPlaneEndpoint.Domain)
 	}
 
 	for _, host := range runtime.GetAllHosts() {
 		if host.GetName() != "" {
 			hostsList = append(hostsList, fmt.Sprintf("%s  %s.%s %s",
-				host.GetInternalAddress(),
+				host.GetInternalIPv4Address(),
+				host.GetName(),
+				kubeConf.Cluster.Kubernetes.ClusterName,
+				host.GetName()))
+			hostsList = append(hostsList, fmt.Sprintf("%s  %s.%s %s",
+				host.GetInternalIPv6Address(),
 				host.GetName(),
 				kubeConf.Cluster.Kubernetes.ClusterName,
 				host.GetName()))
@@ -235,9 +249,16 @@ func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) [
 
 	if len(runtime.GetHostsByRole(common.Registry)) > 0 {
 		if kubeConf.Cluster.Registry.PrivateRegistry != "" {
-			hostsList = append(hostsList, fmt.Sprintf("%s  %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalAddress(), kubeConf.Cluster.Registry.PrivateRegistry))
+			hostsList = append(hostsList, fmt.Sprintf("%s  %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv4Address(), kubeConf.Cluster.Registry.PrivateRegistry))
+			if runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv6Address() != "" {
+				hostsList = append(hostsList, fmt.Sprintf("%s  %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv6Address(), kubeConf.Cluster.Registry.PrivateRegistry))
+			}
+
 		} else {
-			hostsList = append(hostsList, fmt.Sprintf("%s  %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalAddress(), registry.RegistryCertificateBaseName))
+			hostsList = append(hostsList, fmt.Sprintf("%s  %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv4Address(), registry.RegistryCertificateBaseName))
+			if runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv6Address() != "" {
+				hostsList = append(hostsList, fmt.Sprintf("%s  %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv6Address(), registry.RegistryCertificateBaseName))
+			}
 		}
 
 	}