Merge remote-tracking branch 'origin/master' into release-3.1

.github/workflows/golangci-lint.yml

@@ -22,7 +22,7 @@ jobs:
         with:
           go-version: 1.19
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3.4.0
+        uses: golangci/golangci-lint-action@v3.6.0
         with:
           version: v1.50.1
           working-directory: ${{matrix.working-directory}}

.github/workflows/kubernetes-auto-support.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository == 'kubesphere/kubekey'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Go 1.19
         uses: actions/setup-go@v3

cmd/kk/apis/kubekey/v1alpha2/addons_types.go

@@ -36,6 +36,7 @@ type Chart struct {
 	Version    string   `yaml:"version" json:"version,omitempty"`
 	ValuesFile string   `yaml:"valuesFile" json:"valuesFile,omitempty"`
 	Values     []string `yaml:"values" json:"values,omitempty"`
+	Wait       bool     `yaml:"wait" json:"wait,omitempty"`
 }
 
 type Yaml struct {

cmd/kk/apis/kubekey/v1alpha2/cluster_types.go

@@ -173,9 +173,6 @@ func (cfg *ClusterSpec) GroupHosts() map[string][]*KubeHost {
 	if len(roleGroups[Etcd]) == 0 && cfg.Etcd.Type == KubeKey {
 		logger.Log.Fatal(errors.New("The number of etcd cannot be 0"))
 	}
-	if len(roleGroups[Registry]) > 1 {
-		logger.Log.Fatal(errors.New("The number of registry node cannot be greater than 1."))
-	}
 
 	for _, host := range roleGroups[ControlPlane] {
 		host.SetRole(Master)

cmd/kk/apis/kubekey/v1alpha2/default.go

@@ -41,7 +41,7 @@ const (
 	DefaultSSHTimeout              = 30
 	DefaultEtcdVersion             = "v3.5.6"
 	DefaultEtcdPort                = "2379"
-	DefaultDockerVersion           = "20.10.8"
+	DefaultDockerVersion           = "24.0.6"
 	DefaultContainerdVersion       = "1.6.4"
 	DefaultRuncVersion             = "v1.1.1"
 	DefaultCrictlVersion           = "v1.24.0"

cmd/kk/apis/kubekey/v1alpha2/network_types.go

@@ -33,6 +33,7 @@ type CalicoCfg struct {
 	VethMTU         int    `yaml:"vethMTU" json:"vethMTU,omitempty"`
 	Ipv4NatOutgoing *bool  `yaml:"ipv4NatOutgoing" json:"ipv4NatOutgoing,omitempty"`
 	DefaultIPPOOL   *bool  `yaml:"defaultIPPOOL" json:"defaultIPPOOL,omitempty"`
+	EnableTypha     *bool  `yaml:"enableTypha" json:"enableTypha,omitempty"`
 }
 
 type FlannelCfg struct {
@@ -183,6 +184,14 @@ func (c *CalicoCfg) EnableDefaultIPPOOL() bool {
 	return *c.DefaultIPPOOL
 }
 
+// Typha is used to determine whether to enable calico Typha
+func (c *CalicoCfg) Typha() bool {
+	if c.EnableTypha == nil {
+		return false
+	}
+	return *c.EnableTypha
+}
+
 // EnableInit is used to determine whether to create default network
 func (h *HybridnetCfg) EnableInit() bool {
 	if h.Init == nil {

cmd/kk/pkg/addons/charts.go

@@ -93,6 +93,7 @@ func InstallChart(kubeConf *common.KubeConf, addon *kubekeyapiv1alpha2.Addon, ku
 	client.Keyring = defaultKeyring()
 	client.RepoURL = addon.Sources.Chart.Repo
 	client.Version = addon.Sources.Chart.Version
+	client.Wait = addon.Sources.Chart.Wait
 	//client.Force = true
 
 	if client.Version == "" && client.Devel {

cmd/kk/pkg/bootstrap/registry/certs.go

@@ -105,9 +105,13 @@ func (g *GenerateCerts) Execute(runtime connector.Runtime) error {
 
 	var altName cert.AltNames
 
-	dnsList := []string{"localhost", g.KubeConf.Cluster.Registry.PrivateRegistry, runtime.GetHostsByRole(common.Registry)[0].GetName()}
-	ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback, netutils.ParseIPSloppy(runtime.GetHostsByRole(common.Registry)[0].GetInternalAddress())}
+	dnsList := []string{"localhost", RegistryCertificateBaseName}
+	ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}
 
+	for _, h := range runtime.GetHostsByRole(common.Registry) {
+		dnsList = append(dnsList, h.GetName())
+		ipList = append(ipList, netutils.ParseIPSloppy(h.GetInternalAddress()))
+	}
 	altName.DNSNames = dnsList
 	altName.IPs = ipList
 

cmd/kk/pkg/bootstrap/registry/module.go

@@ -250,18 +250,10 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface {
 	}
 
 	generateHarborConfig := &task.RemoteTask{
-		Name:  "GenerateHarborConfig",
-		Desc:  "Generate harbor config",
-		Hosts: i.Runtime.GetHostsByRole(common.Registry),
-		Action: &action.Template{
-			Template: templates.HarborConfigTempl,
-			Dst:      "/opt/harbor/harbor.yml",
-			Data: util.Data{
-				"Domain":      i.KubeConf.Cluster.Registry.PrivateRegistry,
-				"Certificate": fmt.Sprintf("%s.pem", i.KubeConf.Cluster.Registry.PrivateRegistry),
-				"Key":         fmt.Sprintf("%s-key.pem", i.KubeConf.Cluster.Registry.PrivateRegistry),
-			},
-		},
+		Name:   "GenerateHarborConfig",
+		Desc:   "Generate harbor config",
+		Hosts:  i.Runtime.GetHostsByRole(common.Registry),
+		Action: new(GenerateHarborConfig),
 		Parallel: true,
 		Retry:    1,
 	}

cmd/kk/pkg/bootstrap/registry/tasks.go

@@ -18,6 +18,9 @@ package registry
 
 import (
 	"fmt"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry/templates"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
 	"path/filepath"
 	"strings"
 
@@ -212,6 +215,29 @@ func (g *SyncHarborPackage) Execute(runtime connector.Runtime) error {
 	return nil
 }
 
+type GenerateHarborConfig struct {
+	common.KubeAction
+}
+
+func (g *GenerateHarborConfig) Execute(runtime connector.Runtime) error {
+	host := runtime.RemoteHost()
+	templateAction := action.Template{
+		Template: templates.HarborConfigTempl,
+		Dst:      "/opt/harbor/harbor.yml",
+		Data: util.Data{
+			"Domain":      host.GetName(),
+			"Certificate": fmt.Sprintf("%s.pem", RegistryCertificateBaseName),
+			"Key":         fmt.Sprintf("%s-key.pem", RegistryCertificateBaseName),
+			"Password":    templates.Password(g.KubeConf, RegistryCertificateBaseName),
+		},
+	}
+	templateAction.Init(nil, nil)
+	if err := templateAction.Execute(runtime); err != nil {
+		return err
+	}
+	return nil
+}
+
 type StartHarbor struct {
 	common.KubeAction
 }

cmd/kk/pkg/bootstrap/registry/templates/harbor.go

@@ -14,6 +14,9 @@ limitations under the License.
 package templates
 
 import (
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry"
+	"strings"
 	"text/template"
 
 	"github.com/lithammer/dedent"
@@ -133,3 +136,14 @@ proxy:
 
     `)))
 )
+
+func Password(kubeConf *common.KubeConf, domain string) string {
+	auths := registry.DockerRegistryAuthEntries(kubeConf.Cluster.Registry.Auths)
+	for repo, entry := range auths {
+		if strings.Contains(repo, domain) {
+			return entry.Password
+		}
+	}
+
+	return "Harbor12345"
+}

cmd/kk/pkg/kubernetes/tasks.go

@@ -45,7 +45,6 @@ import (
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates"
-	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes/templates/v1beta2"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils"
 )
 
@@ -249,11 +248,11 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
 			}
 		}
 
-		_, ApiServerArgs := util.GetArgs(v1beta2.GetApiServerArgs(g.WithSecurityEnhancement, g.KubeConf.Cluster.Kubernetes.EnableAudit()), g.KubeConf.Cluster.Kubernetes.ApiServerArgs)
-		_, ControllerManagerArgs := util.GetArgs(v1beta2.GetControllermanagerArgs(g.KubeConf.Cluster.Kubernetes.Version, g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs)
-		_, SchedulerArgs := util.GetArgs(v1beta2.GetSchedulerArgs(g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.SchedulerArgs)
+		_, ApiServerArgs := util.GetArgs(templates.GetApiServerArgs(g.WithSecurityEnhancement, g.KubeConf.Cluster.Kubernetes.EnableAudit()), g.KubeConf.Cluster.Kubernetes.ApiServerArgs)
+		_, ControllerManagerArgs := util.GetArgs(templates.GetControllermanagerArgs(g.KubeConf.Cluster.Kubernetes.Version, g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.ControllerManagerArgs)
+		_, SchedulerArgs := util.GetArgs(templates.GetSchedulerArgs(g.WithSecurityEnhancement), g.KubeConf.Cluster.Kubernetes.SchedulerArgs)
 
-		checkCgroupDriver, err := v1beta2.GetKubeletCgroupDriver(runtime, g.KubeConf)
+		checkCgroupDriver, err := templates.GetKubeletCgroupDriver(runtime, g.KubeConf)
 		if err != nil {
 			return err
 		}
@@ -273,8 +272,8 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
 		}
 
 		templateAction := action.Template{
-			Template: v1beta2.KubeadmConfig,
-			Dst:      filepath.Join(common.KubeConfigDir, v1beta2.KubeadmConfig.Name()),
+			Template: templates.KubeadmConfig,
+			Dst:      filepath.Join(common.KubeConfigDir, templates.KubeadmConfig.Name()),
 			Data: util.Data{
 				"IsInitCluster":          g.IsInitConfiguration,
 				"ImageRepo":              strings.TrimSuffix(images.GetImage(runtime, g.KubeConf, "kube-apiserver").ImageRepo(), "/kube-apiserver"),
@@ -296,12 +295,13 @@ func (g *GenerateKubeadmConfig) Execute(runtime connector.Runtime) error {
 				"ExternalEtcd":           externalEtcd,
 				"NodeCidrMaskSize":       g.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize,
 				"CriSock":                g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint,
-				"ApiServerArgs":          v1beta2.UpdateFeatureGatesConfiguration(ApiServerArgs, g.KubeConf),
+				"ApiServerArgs":          templates.UpdateFeatureGatesConfiguration(ApiServerArgs, g.KubeConf),
 				"EnableAudit":            g.KubeConf.Cluster.Kubernetes.EnableAudit(),
-				"ControllerManagerArgs":  v1beta2.UpdateFeatureGatesConfiguration(ControllerManagerArgs, g.KubeConf),
-				"SchedulerArgs":          v1beta2.UpdateFeatureGatesConfiguration(SchedulerArgs, g.KubeConf),
-				"KubeletConfiguration":   v1beta2.GetKubeletConfiguration(runtime, g.KubeConf, g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, g.WithSecurityEnhancement),
-				"KubeProxyConfiguration": v1beta2.GetKubeProxyConfiguration(g.KubeConf),
+				"ControllerManagerArgs":  templates.UpdateFeatureGatesConfiguration(ControllerManagerArgs, g.KubeConf),
+				"SchedulerArgs":          templates.UpdateFeatureGatesConfiguration(SchedulerArgs, g.KubeConf),
+				"KubeletConfiguration":   templates.GetKubeletConfiguration(runtime, g.KubeConf, g.KubeConf.Cluster.Kubernetes.ContainerRuntimeEndpoint, g.WithSecurityEnhancement),
+				"KubeProxyConfiguration": templates.GetKubeProxyConfiguration(g.KubeConf),
+				"IsV1beta3":              versionutil.MustParseSemantic(g.KubeConf.Cluster.Kubernetes.Version).AtLeast(versionutil.MustParseSemantic("v1.22.0")),
 				"IsControlPlane":         host.IsRole(common.Master),
 				"CgroupDriver":           checkCgroupDriver,
 				"BootstrapToken":         bootstrapToken,

cmd/kk/pkg/kubernetes/templates/v1beta2/kubeadm_config.go → cmd/kk/pkg/kubernetes/templates/kubeadm_config.go

@@ -14,7 +14,7 @@
  limitations under the License.
 */
 
-package v1beta2
+package templates
 
 import (
 	"fmt"
@@ -38,7 +38,7 @@ var (
 		dedent.Dedent(`
 {{- if .IsInitCluster -}}
 ---
-apiVersion: kubeadm.k8s.io/v1beta2
+apiVersion: kubeadm.k8s.io/{{ if .IsV1beta3 }}v1beta3{{ else }}v1beta2{{ end }}
 kind: ClusterConfiguration
 etcd:
 {{- if .EtcdTypeIsKubeadm }}
@@ -106,7 +106,7 @@ scheduler:
 {{ toYaml .SchedulerArgs | indent 4 }}
 
 ---
-apiVersion: kubeadm.k8s.io/v1beta2
+apiVersion: kubeadm.k8s.io/{{ if .IsV1beta3 }}v1beta3{{ else }}v1beta2{{ end }}
 kind: InitConfiguration
 localAPIEndpoint:
   advertiseAddress: {{ .AdvertiseAddress }}
@@ -128,7 +128,7 @@ kind: KubeletConfiguration
 
 {{- else -}}
 ---
-apiVersion: kubeadm.k8s.io/v1beta2
+apiVersion: kubeadm.k8s.io/{{ if .IsV1beta3 }}v1beta3{{ else }}v1beta2{{ end }}
 kind: JoinConfiguration
 discovery:
   bootstrapToken:
@@ -159,14 +159,10 @@ var (
 	FeatureGatesDefaultConfiguration = map[string]bool{
 		"RotateKubeletServerCertificate": true, //k8s 1.7+
 		"TTLAfterFinished":               true, //k8s 1.12+
-		"ExpandCSIVolumes":               true, //k8s 1.14+
-		"CSIStorageCapacity":             true, //k8s 1.19+
 	}
 	FeatureGatesSecurityDefaultConfiguration = map[string]bool{
 		"RotateKubeletServerCertificate": true, //k8s 1.7+
 		"TTLAfterFinished":               true, //k8s 1.12+
-		"ExpandCSIVolumes":               true, //k8s 1.14+
-		"CSIStorageCapacity":             true, //k8s 1.19+
 		"SeccompDefault":                 true, //kubelet
 	}
 

cmd/kk/pkg/pipelines/init_registry.go

@@ -18,16 +18,17 @@ package pipelines
 
 import (
 	"fmt"
-
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/artifact"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/binaries"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/os"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/precheck"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/module"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/pipeline"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/filesystem"
+	"github.com/pkg/errors"
 )
 
 func NewInitRegistryPipeline(runtime *common.KubeRuntime) error {
@@ -74,6 +75,10 @@ func InitRegistry(args common.Argument, downloadCmd string) error {
 		return err
 	}
 
+	if len(runtime.GetHostsByRole("registry")) <= 0 {
+		logger.Log.Fatal(errors.New("The number of registry must be greater then 0."))
+	}
+
 	if err := NewInitRegistryPipeline(runtime); err != nil {
 		return err
 	}

cmd/kk/pkg/plugins/network/modules.go

@@ -109,7 +109,7 @@ func deployCalico(d *DeployNetworkPluginModule) []task.Interface {
 				"CalicoNodeImage":        images.GetImage(d.Runtime, d.KubeConf, "calico-node").ImageName(),
 				"CalicoFlexvolImage":     images.GetImage(d.Runtime, d.KubeConf, "calico-flexvol").ImageName(),
 				"CalicoControllersImage": images.GetImage(d.Runtime, d.KubeConf, "calico-kube-controllers").ImageName(),
-				"TyphaEnabled":           len(d.Runtime.GetHostsByRole(common.K8s)) > 50,
+				"TyphaEnabled":           len(d.Runtime.GetHostsByRole(common.K8s)) > 50 || d.KubeConf.Cluster.Network.Calico.Typha(),
 				"VethMTU":                d.KubeConf.Cluster.Network.Calico.VethMTU,
 				"NodeCidrMaskSize":       d.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize,
 				"IPIPMode":               d.KubeConf.Cluster.Network.Calico.IPIPMode,
@@ -137,7 +137,7 @@ func deployCalico(d *DeployNetworkPluginModule) []task.Interface {
 				"CalicoFlexvolImage":      images.GetImage(d.Runtime, d.KubeConf, "calico-flexvol").ImageName(),
 				"CalicoControllersImage":  images.GetImage(d.Runtime, d.KubeConf, "calico-kube-controllers").ImageName(),
 				"CalicoTyphaImage":        images.GetImage(d.Runtime, d.KubeConf, "calico-typha").ImageName(),
-				"TyphaEnabled":            len(d.Runtime.GetHostsByRole(common.K8s)) > 50,
+				"TyphaEnabled":            len(d.Runtime.GetHostsByRole(common.K8s)) > 50 || d.KubeConf.Cluster.Network.Calico.Typha(),
 				"VethMTU":                 d.KubeConf.Cluster.Network.Calico.VethMTU,
 				"NodeCidrMaskSize":        d.KubeConf.Cluster.Kubernetes.NodeCidrMaskSize,
 				"IPIPMode":                d.KubeConf.Cluster.Network.Calico.IPIPMode,

cmd/kk/pkg/version/kubernetes/version_enum.go

@@ -36,6 +36,8 @@ const (
 	V124
 	V125
 	V126
+	V127
+	V128
 )
 
 var VersionList = []Version{
@@ -47,6 +49,8 @@ var VersionList = []Version{
 	V124,
 	V125,
 	V126,
+	V127,
+	V128,
 }
 
 func (v Version) String() string {
@@ -67,6 +71,10 @@ func (v Version) String() string {
 		return "v1.25"
 	case V126:
 		return "v1.26"
+	case V127:
+		return "v1.27"
+	case V128:
+		return "v1.28"
 	default:
 		return "invalid option"
 	}