Merge remote-tracking branch 'origin/master' into release-3.1

cmd/kk/apis/kubekey/v1alpha2/default.go

@@ -39,7 +39,7 @@ const (
 	DefaultDNSDomain               = "cluster.local"
 	DefaultArch                    = "amd64"
 	DefaultSSHTimeout              = 30
-	DefaultEtcdVersion             = "v3.5.6"
+	DefaultEtcdVersion             = "v3.5.13"
 	DefaultEtcdPort                = "2379"
 	DefaultDockerVersion           = "24.0.9"
 	DefaultCriDockerdVersion       = "0.3.10"
@@ -51,14 +51,14 @@ const (
 	DefaultFlannelVersion          = "v0.21.3"
 	DefaultFlannelCniPluginVersion = "v1.1.2"
 	DefaultCniVersion              = "v1.2.0"
-	DefaultCiliumVersion           = "v1.11.7"
+	DefaultCiliumVersion           = "v1.15.3"
 	DefaulthybridnetVersion        = "v0.8.6"
-	DefaultKubeovnVersion          = "v1.10.6"
+	DefaultKubeovnVersion          = "v1.10.10"
 	DefalutMultusVersion           = "v3.8"
-	DefaultHelmVersion             = "v3.13.3"
-	DefaultDockerComposeVersion    = "v2.2.2"
+	DefaultHelmVersion             = "v3.14.3"
+	DefaultDockerComposeVersion    = "v2.26.1"
 	DefaultRegistryVersion         = "2"
-	DefaultHarborVersion           = "v2.5.3"
+	DefaultHarborVersion           = "v2.10.1"
 	DefaultMaxPods                 = 110
 	DefaultPodPidsLimit            = 10000
 	DefaultNodeCidrMaskSize        = 24

cmd/kk/cmd/artifact/export.go

@@ -30,10 +30,11 @@ import (
 type ArtifactExportOptions struct {
 	CommonOptions *options.CommonOptions
 
-	ManifestFile string
-	Output       string
-	CriSocket    string
-	DownloadCmd  string
+	ManifestFile       string
+	Output             string
+	CriSocket          string
+	DownloadCmd        string
+	SkipRemoveArtifact bool
 }
 
 func NewArtifactExportOptions() *ArtifactExportOptions {
@@ -76,11 +77,12 @@ func (o *ArtifactExportOptions) Validate(_ []string) error {
 
 func (o *ArtifactExportOptions) Run() error {
 	arg := common.ArtifactArgument{
-		ManifestFile: o.ManifestFile,
-		Output:       o.Output,
-		CriSocket:    o.CriSocket,
-		Debug:        o.CommonOptions.Verbose,
-		IgnoreErr:    o.CommonOptions.IgnoreErr,
+		ManifestFile:       o.ManifestFile,
+		Output:             o.Output,
+		CriSocket:          o.CriSocket,
+		Debug:              o.CommonOptions.Verbose,
+		IgnoreErr:          o.CommonOptions.IgnoreErr,
+		SkipRemoveArtifact: o.SkipRemoveArtifact,
 	}
 
 	return pipelines.ArtifactExport(arg, o.DownloadCmd)
@@ -91,4 +93,6 @@ func (o *ArtifactExportOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&o.Output, "output", "o", "", "Path to a output path")
 	cmd.Flags().StringVarP(&o.DownloadCmd, "download-cmd", "", "curl -L -o %s %s",
 		`The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`)
+	cmd.Flags().BoolVarP(&o.SkipRemoveArtifact, "skip-remove-artifact", "", false, "Skip remove artifact")
+
 }

cmd/kk/cmd/create/manifest.go

@@ -38,6 +38,9 @@ type CreateManifestOptions struct {
 	Name       string
 	KubeConfig string
 	FileName   string
+	Kubernetes string
+	registry   bool
+	Arch       []string
 }
 
 func NewCreateManifestOptions() *CreateManifestOptions {
@@ -86,11 +89,17 @@ func (o *CreateManifestOptions) Run() error {
 		FilePath:   o.FileName,
 		KubeConfig: o.KubeConfig,
 	}
-	return artifact.CreateManifest(arg, o.Name)
+	if o.Kubernetes != "" {
+		return artifact.CreateManifestSpecifyVersion(arg, o.Name, o.Kubernetes, o.registry, o.Arch)
+	}
+	return artifact.CreateManifest(arg, o.Name, o.registry)
 }
 
 func (o *CreateManifestOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&o.Name, "name", "", "sample", "Specify a name of manifest object")
 	cmd.Flags().StringVarP(&o.FileName, "filename", "f", "", "Specify a manifest file path")
 	cmd.Flags().StringVar(&o.KubeConfig, "kubeconfig", "", "Specify a kubeconfig file")
+	cmd.Flags().StringVarP(&o.Kubernetes, "with-kubernetes", "", "", "Specify a supported version of kubernetes")
+	cmd.Flags().BoolVar(&o.registry, "with-registry", false, "Specify a supported registry components")
+	cmd.Flags().StringArrayVar(&o.Arch, "arch", []string{"amd64"}, "Specify a supported arch")
 }

cmd/kk/cmd/upgrade/upgrade.go

@@ -38,6 +38,7 @@ type UpgradeOptions struct {
 	KubeSphere          string
 	SkipPullImages      bool
 	SkipDependencyCheck bool
+	EtcdUpgrade         bool
 	DownloadCmd         string
 	Artifact            string
 }
@@ -90,6 +91,7 @@ func (o *UpgradeOptions) Run() error {
 		SkipConfirmCheck:    o.CommonOptions.SkipConfirmCheck,
 		Artifact:            o.Artifact,
 		SkipDependencyCheck: o.SkipDependencyCheck,
+		EtcdUpgrade:         o.EtcdUpgrade,
 	}
 	return pipelines.UpgradeCluster(arg, o.DownloadCmd)
 }
@@ -103,6 +105,7 @@ func (o *UpgradeOptions) AddFlags(cmd *cobra.Command) {
 		`The user defined command to download the necessary binary files. The first param '%s' is output path, the second param '%s', is the URL`)
 	cmd.Flags().StringVarP(&o.Artifact, "artifact", "a", "", "Path to a KubeKey artifact")
 	cmd.Flags().BoolVarP(&o.SkipDependencyCheck, "skip-dependency-check", "", false, "Skip kubernetes and kubesphere dependency version check")
+	cmd.Flags().BoolVarP(&o.EtcdUpgrade, "with-etcd", "", false, "Upgrade etcd")
 }
 
 func completionSetting(cmd *cobra.Command) (err error) {

cmd/kk/pkg/artifact/manifest.go

@@ -20,6 +20,9 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files"
+	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images"
 	"os"
 	"sort"
 	"strings"
@@ -36,7 +39,7 @@ import (
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
 )
 
-func CreateManifest(arg common.Argument, name string) error {
+func CreateManifest(arg common.Argument, name string, registry bool) error {
 	checkFileExists(arg.FilePath)
 
 	client, err := kubernetes.NewClient(arg.KubeConfig)
@@ -171,6 +174,12 @@ func CreateManifest(arg common.Argument, name string) error {
 		Images: imageArr,
 	}
 
+	if registry {
+		options.Components.DockerRegistry.Version = kubekeyv1alpha2.DefaultRegistryVersion
+		options.Components.DockerCompose.Version = kubekeyv1alpha2.DefaultDockerComposeVersion
+		options.Components.Harbor.Version = kubekeyv1alpha2.DefaultHarborVersion
+	}
+
 	manifestStr, err := templates.RenderManifest(options)
 
 	if err := os.WriteFile(arg.FilePath, []byte(manifestStr), 0644); err != nil {
@@ -204,3 +213,131 @@ func checkFileExists(fileName string) {
 		}
 	}
 }
+
+func CreateManifestSpecifyVersion(arg common.Argument, name, version string, registry bool, arch []string) error {
+	checkFileExists(arg.FilePath)
+
+	var kubernetesDistribution []kubekeyv1alpha2.KubernetesDistribution
+
+	k8sVersion := strings.Split(version, ",")
+
+	imageNames := []string{
+		"pause",
+		"kube-apiserver",
+		"kube-controller-manager",
+		"kube-scheduler",
+		"kube-proxy",
+
+		// network
+		"coredns",
+		"k8s-dns-node-cache",
+		"calico-kube-controllers",
+		"calico-cni",
+		"calico-node",
+		"calico-flexvol",
+		"calico-typha",
+		"flannel",
+		"flannel-cni-plugin",
+		"cilium",
+		"cilium-operator-generic",
+		"hybridnet",
+		"kubeovn",
+		"multus",
+		// storage
+		"provisioner-localpv",
+		"linux-utils",
+		// load balancer
+		"haproxy",
+		"kubevip",
+		// kata-deploy
+		"kata-deploy",
+		// node-feature-discovery
+		"node-feature-discovery",
+	}
+	var imageArr []string
+	for _, v := range k8sVersion {
+		versionutil.MustParseGeneric(v)
+
+		for _, a := range arch {
+			_, ok := files.FileSha256["kubeadm"][a][v]
+			if !ok {
+				return fmt.Errorf("invalid kubernetes version %s", v)
+			}
+		}
+
+		kubernetesDistribution = append(kubernetesDistribution, kubekeyv1alpha2.KubernetesDistribution{
+			Type:    "kubernetes",
+			Version: v,
+		})
+
+		for _, imageName := range imageNames {
+			repo := images.GetImage(&common.KubeRuntime{
+				BaseRuntime: connector.NewBaseRuntime("image list", connector.NewDialer(), arg.Debug, arg.IgnoreErr),
+			}, &common.KubeConf{
+				Cluster: &kubekeyv1alpha2.ClusterSpec{
+					Kubernetes: kubekeyv1alpha2.Kubernetes{Version: v},
+					Registry:   kubekeyv1alpha2.RegistryConfig{PrivateRegistry: "docker.io"},
+				},
+			}, imageName).ImageName()
+			if !imageIsExist(repo, imageArr) {
+				imageArr = append(imageArr, repo)
+			}
+
+		}
+	}
+
+	options := &templates.Options{
+		Name:   name,
+		Arches: arch,
+		OperatingSystems: []kubekeyv1alpha2.OperatingSystem{{
+			Arch:    "amd64",
+			Type:    "linux",
+			Id:      "ubuntu",
+			Version: "20.04",
+			OsImage: "Ubuntu 20.04.6 LTS",
+		}},
+		KubernetesDistributions: kubernetesDistribution,
+		Components: kubekeyv1alpha2.Components{
+			Helm:      kubekeyv1alpha2.Helm{Version: kubekeyv1alpha2.DefaultHelmVersion},
+			CNI:       kubekeyv1alpha2.CNI{Version: kubekeyv1alpha2.DefaultCniVersion},
+			ETCD:      kubekeyv1alpha2.ETCD{Version: kubekeyv1alpha2.DefaultEtcdVersion},
+			Crictl:    kubekeyv1alpha2.Crictl{Version: kubekeyv1alpha2.DefaultCrictlVersion},
+			Calicoctl: kubekeyv1alpha2.Calicoctl{Version: kubekeyv1alpha2.DefaultCalicoVersion},
+			ContainerRuntimes: []kubekeyv1alpha2.ContainerRuntime{
+				{
+					Type:    "docker",
+					Version: kubekeyv1alpha2.DefaultDockerVersion,
+				},
+				{
+					Type:    "containerd",
+					Version: kubekeyv1alpha2.DefaultContainerdVersion,
+				},
+			},
+		},
+		Images: imageArr,
+	}
+
+	if registry {
+		options.Components.DockerRegistry.Version = kubekeyv1alpha2.DefaultRegistryVersion
+		options.Components.DockerCompose.Version = kubekeyv1alpha2.DefaultDockerComposeVersion
+		options.Components.Harbor.Version = kubekeyv1alpha2.DefaultHarborVersion
+	}
+
+	manifestStr, err := templates.RenderManifest(options)
+
+	if err = os.WriteFile(arg.FilePath, []byte(manifestStr), 0644); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("write file %s failed", arg.FilePath))
+	}
+
+	fmt.Println("Generate KubeKey manifest file successfully")
+	return nil
+}
+
+func imageIsExist(s string, arr []string) bool {
+	for _, s2 := range arr {
+		if s2 == s {
+			return true
+		}
+	}
+	return false
+}

cmd/kk/pkg/artifact/tasks.go

@@ -109,6 +109,11 @@ func (a *ArchiveDependencies) Execute(runtime connector.Runtime) error {
 		return errors.Wrapf(errors.WithStack(err), "archive %s failed", src)
 	}
 
+	// skip remove artifact if --skip-remove-artifact
+	if a.Manifest.Arg.SkipRemoveArtifact {
+		return nil
+	}
+
 	// remove the src directory
 	if err := os.RemoveAll(src); err != nil {
 		return errors.Wrapf(errors.WithStack(err), "remove %s failed", src)

cmd/kk/pkg/artifact/templates/manifest.go

@@ -70,13 +70,14 @@ spec:
       version: {{ .Options.Components.Calicoctl.Version }}
     crictl: 
       version: {{ .Options.Components.Crictl.Version }}
-    ## 
-    # docker-registry:
-    #   version: "2"
-    # harbor:
-    #   version: v2.4.1
-    # docker-compose:
-    #   version: v2.2.2
+    {{ if .Options.Components.DockerRegistry.Version -}}
+    docker-registry:
+      version: "{{ .Options.Components.DockerRegistry.Version }}"
+    harbor:
+      version: {{ .Options.Components.Harbor.Version }}
+    docker-compose:
+      version: {{ .Options.Components.DockerCompose.Version }}
+	{{- end }}
   images:
   {{- range .Options.Images }}
   - {{ . }}

cmd/kk/pkg/binaries/kubernetes.go

@@ -127,6 +127,10 @@ func KubernetesComponentBinariesDownload(manifest *common.ArtifactManifest, path
 			containerManagerVersion[c.Type+c.Version] = struct{}{}
 			containerManager := files.NewKubeBinary(c.Type, arch, c.Version, path, manifest.Arg.DownloadCommand)
 			binaries = append(binaries, containerManager)
+			if c.Type == "docker" {
+				criDockerd := files.NewKubeBinary("cri-dockerd", arch, kubekeyapiv1alpha2.DefaultCriDockerdVersion, path, manifest.Arg.DownloadCommand)
+				binaries = append(binaries, criDockerd)
+			}
 			if c.Type == "containerd" {
 				runc := files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, manifest.Arg.DownloadCommand)
 				binaries = append(binaries, runc)

cmd/kk/pkg/bootstrap/registry/tasks.go

@@ -17,9 +17,11 @@
 package registry
 
 import (
+	"embed"
 	"fmt"
 	"path/filepath"
 	"strings"
+	"text/template"
 
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/bootstrap/registry/templates"
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action"
@@ -33,6 +35,10 @@ import (
 	"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils"
 )
 
+//go:embed templates/harbor.yml.tmpl
+
+var f embed.FS
+
 type SyncCertsFile struct {
 	common.KubeAction
 }
@@ -221,6 +227,12 @@ type GenerateHarborConfig struct {
 }
 
 func (g *GenerateHarborConfig) Execute(runtime connector.Runtime) error {
+	harborContent, err := f.ReadFile("templates/harbor.yml.tmpl")
+	if err != nil {
+		return err
+	}
+	harbor := template.Must(template.New("harbor.yml").Parse(string(harborContent)))
+
 	registryDomain := g.KubeConf.Cluster.Registry.GetHost()
 
 	if g.KubeConf.Cluster.Registry.Type == "harbor-ha" {
@@ -229,7 +241,7 @@ func (g *GenerateHarborConfig) Execute(runtime connector.Runtime) error {
 	}
 
 	templateAction := action.Template{
-		Template: templates.HarborConfigTempl,
+		Template: harbor,
 		Dst:      "/opt/harbor/harbor.yml",
 		Data: util.Data{
 			"Domain":      registryDomain,
@@ -250,7 +262,7 @@ type StartHarbor struct {
 }
 
 func (g *StartHarbor) Execute(runtime connector.Runtime) error {
-	startCmd := "cd /opt/harbor && chmod +x install.sh && export PATH=$PATH:/usr/local/bin; ./install.sh --with-notary --with-trivy --with-chartmuseum && systemctl daemon-reload && systemctl enable harbor && systemctl restart harbor"
+	startCmd := "cd /opt/harbor && chmod +x install.sh && export PATH=$PATH:/usr/local/bin; ./install.sh --with-trivy && systemctl daemon-reload && systemctl enable harbor && systemctl restart harbor"
 	if _, err := runtime.GetRunner().SudoCmd(startCmd, false); err != nil {
 		return errors.Wrap(errors.WithStack(err), "start harbor failed")
 	}