Merge remote-tracking branch 'origin/master' into release-3.1

.github/workflows/gen-repository-iso.yaml

@@ -66,7 +66,7 @@ jobs:
 
       - name: Release and upload packages
         if: startsWith(github.ref, 'refs/tags/')
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

.github/workflows/issue_comment_webhook.yml

@@ -0,0 +1,45 @@
+name: Issue Comment WeCom Webhook
+
+on:
+  issues:
+    types: [opened, edited]
+  issue_comment:
+    types: [created, edited]
+
+jobs:
+  send_to_webhook:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20.x"
+      - run: npm install axios
+      - name: Send issue/comment to WeCom webhook
+        uses: actions/github-script@v7
+        env:
+          WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }}
+        with:
+          script: |
+            console.log(context);
+            if (context.payload.sender.login === "ks-ci-bot") return;
+            const axios = require('axios');
+            const issue = context.payload.issue;
+            const comment = context.payload.comment;
+            var subject = {};
+            var action = '';
+            if (comment) {
+              action = "comment";
+              subject = comment;
+            } else {
+              action = "issue";
+              subject = issue;
+            };
+            const payload = {
+              msgtype: 'markdown',
+              markdown: {
+                content: `[${context.payload.sender.login}](${context.payload.sender.html_url}) ${context.payload.action} ${action} [${issue.title}](${subject.html_url})\n${subject.body}`,
+              },
+            };
+            const formattedPayload = JSON.stringify(payload, null, 2);
+            console.log(formattedPayload);
+            await axios.post(process.env.WEBHOOK_URL, payload);

.github/workflows/release.yaml

@@ -24,7 +24,7 @@ jobs:
         run: |
           make release
       - name: Release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           draft: true
           files: out/*

cmd/kk/apis/kubekey/v1alpha2/cluster_types.go

@@ -146,7 +146,7 @@ func (cfg *ClusterSpec) GenerateCertSANs() []string {
 		if InternalIPv4Address != host.Address && InternalIPv4Address != cfg.ControlPlaneEndpoint.Address {
 			extraCertSANs = append(extraCertSANs, InternalIPv4Address)
 		}
-		if len(nodeAddresses)==2 {
+		if len(nodeAddresses) == 2 {
 			InternalIPv6Address := nodeAddresses[1]
 			extraCertSANs = append(extraCertSANs, InternalIPv6Address)
 		}
@@ -310,3 +310,10 @@ func (c *ControlPlaneEndpoint) EnableExternalDNS() bool {
 	}
 	return *c.ExternalDNS
 }
+
+func (r *RegistryConfig) GetHost() string {
+	if r.PrivateRegistry == "" {
+		return ""
+	}
+	return strings.Split(r.PrivateRegistry, "/")[0]
+}

cmd/kk/pkg/binaries/kubernetes.go

@@ -101,18 +101,25 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string,
 	return nil
 }
 
-func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch, k8sVersion string) error {
+func KubernetesComponentBinariesDownload(manifest *common.ArtifactManifest, path, arch string) error {
 	m := manifest.Spec
+	var binaries []*files.KubeBinary
 
-	etcd := files.NewKubeBinary("etcd", arch, m.Components.ETCD.Version, path, manifest.Arg.DownloadCommand)
-	kubeadm := files.NewKubeBinary("kubeadm", arch, k8sVersion, path, manifest.Arg.DownloadCommand)
-	kubelet := files.NewKubeBinary("kubelet", arch, k8sVersion, path, manifest.Arg.DownloadCommand)
-	kubectl := files.NewKubeBinary("kubectl", arch, k8sVersion, path, manifest.Arg.DownloadCommand)
-	kubecni := files.NewKubeBinary("kubecni", arch, m.Components.CNI.Version, path, manifest.Arg.DownloadCommand)
-	helm := files.NewKubeBinary("helm", arch, m.Components.Helm.Version, path, manifest.Arg.DownloadCommand)
-	crictl := files.NewKubeBinary("crictl", arch, m.Components.Crictl.Version, path, manifest.Arg.DownloadCommand)
-	calicoctl := files.NewKubeBinary("calicoctl", arch, m.Components.Calicoctl.Version, path, manifest.Arg.DownloadCommand)
-	binaries := []*files.KubeBinary{kubeadm, kubelet, kubectl, helm, kubecni, etcd, calicoctl}
+	if m.Components.ETCD.Version != "" {
+		binaries = append(binaries, files.NewKubeBinary("etcd", arch, m.Components.ETCD.Version, path, manifest.Arg.DownloadCommand))
+	}
+	if m.Components.CNI.Version != "" {
+		binaries = append(binaries, files.NewKubeBinary("kubecni", arch, m.Components.CNI.Version, path, manifest.Arg.DownloadCommand))
+	}
+	if m.Components.Helm.Version != "" {
+		binaries = append(binaries, files.NewKubeBinary("helm", arch, m.Components.Helm.Version, path, manifest.Arg.DownloadCommand))
+	}
+	if m.Components.Crictl.Version != "" {
+		binaries = append(binaries, files.NewKubeBinary("crictl", arch, m.Components.Crictl.Version, path, manifest.Arg.DownloadCommand))
+	}
+	if m.Components.Calicoctl.Version != "" {
+		binaries = append(binaries, files.NewKubeBinary("calicoctl", arch, m.Components.Calicoctl.Version, path, manifest.Arg.DownloadCommand))
+	}
 
 	containerManagerArr := make([]*files.KubeBinary, 0, 0)
 	containerManagerVersion := make(map[string]struct{})
@@ -128,11 +135,36 @@ func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path,
 		}
 	}
 
-	binaries = append(binaries, containerManagerArr...)
-	if m.Components.Crictl.Version != "" {
-		binaries = append(binaries, crictl)
+	for _, binary := range binaries {
+		if err := binary.CreateBaseDir(); err != nil {
+			return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName)
+		}
+
+		logger.Log.Messagef(common.LocalHost, "downloading %s %s %s ...", arch, binary.ID, binary.Version)
+
+		if util.IsExist(binary.Path()) {
+			// download it again if it's incorrect
+			if err := binary.SHA256Check(); err != nil {
+				_ = exec.Command("/bin/sh", "-c", fmt.Sprintf("rm -f %s", binary.Path())).Run()
+			} else {
+				continue
+			}
+		}
+
+		if err := binary.Download(); err != nil {
+			return fmt.Errorf("Failed to download %s binary: %s error: %w ", binary.ID, binary.GetCmd(), err)
+		}
 	}
 
+	return nil
+}
+
+func KubernetesArtifactBinariesDownload(manifest *common.ArtifactManifest, path, arch, k8sVersion string) error {
+	kubeadm := files.NewKubeBinary("kubeadm", arch, k8sVersion, path, manifest.Arg.DownloadCommand)
+	kubelet := files.NewKubeBinary("kubelet", arch, k8sVersion, path, manifest.Arg.DownloadCommand)
+	kubectl := files.NewKubeBinary("kubectl", arch, k8sVersion, path, manifest.Arg.DownloadCommand)
+	binaries := []*files.KubeBinary{kubeadm, kubelet, kubectl}
+
 	for _, binary := range binaries {
 		if err := binary.CreateBaseDir(); err != nil {
 			return errors.Wrapf(errors.WithStack(err), "create file %s base dir failed", binary.FileName)

cmd/kk/pkg/binaries/tasks.go

@@ -168,6 +168,10 @@ func (a *ArtifactDownload) Execute(runtime connector.Runtime) error {
 			}
 		}
 
+		if err := KubernetesComponentBinariesDownload(a.Manifest, basePath, arch); err != nil {
+			return err
+		}
+
 		if err := RegistryBinariesDownload(a.Manifest, basePath, arch); err != nil {
 			return err
 		}

cmd/kk/pkg/bootstrap/os/templates/init_script.go

@@ -249,9 +249,9 @@ func GenerateHosts(runtime connector.ModuleRuntime, kubeConf *common.KubeConf) [
 
 	if len(runtime.GetHostsByRole(common.Registry)) > 0 {
 		if kubeConf.Cluster.Registry.PrivateRegistry != "" {
-			hostsList = append(hostsList, fmt.Sprintf("%s  %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv4Address(), kubeConf.Cluster.Registry.PrivateRegistry))
+			hostsList = append(hostsList, fmt.Sprintf("%s  %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv4Address(), kubeConf.Cluster.Registry.GetHost()))
 			if runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv6Address() != "" {
-				hostsList = append(hostsList, fmt.Sprintf("%s  %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv6Address(), kubeConf.Cluster.Registry.PrivateRegistry))
+				hostsList = append(hostsList, fmt.Sprintf("%s  %s", runtime.GetHostsByRole(common.Registry)[0].GetInternalIPv6Address(), kubeConf.Cluster.Registry.GetHost()))
 			}
 
 		} else {

cmd/kk/pkg/bootstrap/registry/certs.go

@@ -105,7 +105,7 @@ func (g *GenerateCerts) Execute(runtime connector.Runtime) error {
 
 	var altName cert.AltNames
 
-	dnsList := []string{"localhost", g.KubeConf.Cluster.Registry.PrivateRegistry}
+	dnsList := []string{"localhost", g.KubeConf.Cluster.Registry.GetHost()}
 	ipList := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}
 
 	for _, h := range runtime.GetHostsByRole(common.Registry) {
@@ -115,13 +115,13 @@ func (g *GenerateCerts) Execute(runtime connector.Runtime) error {
 	altName.DNSNames = dnsList
 	altName.IPs = ipList
 
-	files := []string{"ca.pem", "ca-key.pem", fmt.Sprintf("%s.pem", g.KubeConf.Cluster.Registry.PrivateRegistry), fmt.Sprintf("%s-key.pem", g.KubeConf.Cluster.Registry.PrivateRegistry)}
+	files := []string{"ca.pem", "ca-key.pem", fmt.Sprintf("%s.pem", g.KubeConf.Cluster.Registry.GetHost()), fmt.Sprintf("%s-key.pem", g.KubeConf.Cluster.Registry.GetHost())}
 
 	// CA
 	certsList := []*certs.KubekeyCert{KubekeyCertRegistryCA()}
 
 	// Certs
-	certsList = append(certsList, KubekeyCertRegistryServer(g.KubeConf.Cluster.Registry.PrivateRegistry, &altName))
+	certsList = append(certsList, KubekeyCertRegistryServer(g.KubeConf.Cluster.Registry.GetHost(), &altName))
 
 	var lastCACert *certs.KubekeyCert
 	for _, c := range certsList {

cmd/kk/pkg/bootstrap/registry/module.go

@@ -131,8 +131,8 @@ func InstallRegistry(i *InstallRegistryModule) []task.Interface {
 			Template: templates.RegistryConfigTempl,
 			Dst:      "/etc/kubekey/registry/config.yaml",
 			Data: util.Data{
-				"Certificate": fmt.Sprintf("%s.pem", i.KubeConf.Cluster.Registry.PrivateRegistry),
-				"Key":         fmt.Sprintf("%s-key.pem", i.KubeConf.Cluster.Registry.PrivateRegistry),
+				"Certificate": fmt.Sprintf("%s.pem", i.KubeConf.Cluster.Registry.GetHost()),
+				"Key":         fmt.Sprintf("%s-key.pem", i.KubeConf.Cluster.Registry.GetHost()),
 			},
 		},
 		Parallel: true,
@@ -170,6 +170,20 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface {
 		Retry:    2,
 	}
 
+	generateContainerdService := &task.RemoteTask{
+		Name:  "GenerateContainerdService",
+		Desc:  "Generate containerd service",
+		Hosts: i.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&container.ContainerdExist{Not: true},
+		},
+		Action: &action.Template{
+			Template: docker_template.ContainerdService,
+			Dst:      filepath.Join("/etc/systemd/system", docker_template.ContainerdService.Name()),
+		},
+		Parallel: true,
+	}
+
 	generateDockerService := &task.RemoteTask{
 		Name:  "GenerateDockerService",
 		Desc:  "Generate docker service",
@@ -202,6 +216,17 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface {
 		Parallel: true,
 	}
 
+	enableContainerdForDocker := &task.RemoteTask{
+		Name:  "EnableContainerd",
+		Desc:  "Enable containerd",
+		Hosts: i.Runtime.GetHostsByRole(common.K8s),
+		Prepare: &prepare.PrepareCollection{
+			&container.ContainerdExist{Not: true},
+		},
+		Action:   new(container.EnableContainerdForDocker),
+		Parallel: true,
+	}
+
 	enableDocker := &task.RemoteTask{
 		Name:  "EnableDocker",
 		Desc:  "Enable docker",
@@ -250,10 +275,10 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface {
 	}
 
 	generateHarborConfig := &task.RemoteTask{
-		Name:   "GenerateHarborConfig",
-		Desc:   "Generate harbor config",
-		Hosts:  i.Runtime.GetHostsByRole(common.Registry),
-		Action: new(GenerateHarborConfig),
+		Name:     "GenerateHarborConfig",
+		Desc:     "Generate harbor config",
+		Hosts:    i.Runtime.GetHostsByRole(common.Registry),
+		Action:   new(GenerateHarborConfig),
 		Parallel: true,
 		Retry:    1,
 	}
@@ -269,8 +294,10 @@ func InstallHarbor(i *InstallRegistryModule) []task.Interface {
 
 	return []task.Interface{
 		syncBinaries,
+		generateContainerdService,
 		generateDockerService,
 		generateDockerConfig,
+		enableContainerdForDocker,
 		enableDocker,
 		installDockerCompose,
 		syncHarborPackage,

cmd/kk/pkg/bootstrap/registry/tasks.go

@@ -89,7 +89,7 @@ func (s *SyncCertsToAllNodes) Execute(runtime connector.Runtime) error {
 			}
 		}
 
-		if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(filepath.Join("/etc/docker/certs.d", s.KubeConf.Cluster.Registry.PrivateRegistry), dstFileName)); err != nil {
+		if err := runtime.GetRunner().SudoScp(filepath.Join(dir, fileName), filepath.Join(filepath.Join("/etc/docker/certs.d", s.KubeConf.Cluster.Registry.GetHost()), dstFileName)); err != nil {
 			return errors.Wrap(errors.WithStack(err), "scp registry certs file to /etc/docker/certs.d/ failed")
 		}
 
@@ -144,7 +144,7 @@ func (g *StartRegistryService) Execute(runtime connector.Runtime) error {
 	}
 
 	fmt.Println()
-	fmt.Println(fmt.Sprintf("Local image registry created successfully. Address: %s", g.KubeConf.Cluster.Registry.PrivateRegistry))
+	fmt.Println(fmt.Sprintf("Local image registry created successfully. Address: %s", g.KubeConf.Cluster.Registry.GetHost()))
 	fmt.Println()
 
 	return nil
@@ -221,7 +221,7 @@ type GenerateHarborConfig struct {
 }
 
 func (g *GenerateHarborConfig) Execute(runtime connector.Runtime) error {
-	registryDomain := g.KubeConf.Cluster.Registry.PrivateRegistry
+	registryDomain := g.KubeConf.Cluster.Registry.GetHost()
 
 	if g.KubeConf.Cluster.Registry.Type == "harbor-ha" {
 		host := runtime.RemoteHost()
@@ -233,9 +233,9 @@ func (g *GenerateHarborConfig) Execute(runtime connector.Runtime) error {
 		Dst:      "/opt/harbor/harbor.yml",
 		Data: util.Data{
 			"Domain":      registryDomain,
-			"Certificate": fmt.Sprintf("%s.pem", g.KubeConf.Cluster.Registry.PrivateRegistry),
-			"Key":         fmt.Sprintf("%s-key.pem", g.KubeConf.Cluster.Registry.PrivateRegistry),
-			"Password":    templates.Password(g.KubeConf, g.KubeConf.Cluster.Registry.PrivateRegistry),
+			"Certificate": fmt.Sprintf("%s.pem", g.KubeConf.Cluster.Registry.GetHost()),
+			"Key":         fmt.Sprintf("%s-key.pem", g.KubeConf.Cluster.Registry.GetHost()),
+			"Password":    templates.Password(g.KubeConf, g.KubeConf.Cluster.Registry.GetHost()),
 		},
 	}
 	templateAction.Init(nil, nil)
@@ -256,7 +256,7 @@ func (g *StartHarbor) Execute(runtime connector.Runtime) error {
 	}
 
 	fmt.Println()
-	fmt.Println(fmt.Sprintf("Local image registry created successfully. Address: %s", g.KubeConf.Cluster.Registry.PrivateRegistry))
+	fmt.Println(fmt.Sprintf("Local image registry created successfully. Address: %s", g.KubeConf.Cluster.Registry.GetHost()))
 	fmt.Println()
 
 	return nil

cmd/kk/pkg/container/module.go

@@ -281,6 +281,10 @@ type InstallCriDockerdModule struct {
 	Skip bool
 }
 
+func (m *InstallCriDockerdModule) IsSkip() bool {
+	return m.Skip
+}
+
 func (m *InstallCriDockerdModule) Init() {
 	m.Name = "InstallCriDockerdModule"
 	m.Desc = "Install cri-dockerd"

cmd/kk/pkg/images/tasks.go

@@ -289,7 +289,7 @@ func (c *CopyImagesToRegistry) Execute(runtime connector.Runtime) error {
 		}
 
 		auth := new(registry.DockerRegistryEntry)
-		if config, ok := auths[c.KubeConf.Cluster.Registry.PrivateRegistry]; ok {
+		if config, ok := auths[c.KubeConf.Cluster.Registry.GetHost()]; ok {
 			auth = config
 		}
 

cmd/kk/pkg/pipelines/artifact_export.go

@@ -117,7 +117,7 @@ func ArtifactExport(args common.ArtifactArgument, downloadCmd string) error {
 	}
 
 	if len(runtime.Spec.KubernetesDistributions) == 0 {
-		return errors.New("the length of kubernetes distributions can't be 0")
+		return NewArtifactExportPipeline(runtime)
 	}
 
 	pre := runtime.Spec.KubernetesDistributions[0].Type

hack/gen-repository-iso/dockerfile.ubuntu2204

@@ -1,5 +1,6 @@
 FROM ubuntu:22.04 as ubuntu2204
 ARG TARGETARCH
+ARG DISTRO=ubuntu2204
 ARG OS_RELEASE=jammy
 ARG DIR=ubuntu-22.04-${TARGETARCH}-debs
 ARG PKGS=.common[],.debs[],.ubuntu[],.ubuntu2204[]
@@ -15,6 +16,19 @@ RUN apt update -qq \
     && echo "deb [arch=$TARGETARCH] https://download.docker.com/linux/ubuntu ${OS_RELEASE} stable" > /etc/apt/sources.list.d/docker.list\
     && apt update -qq
 
+# install NVIDIA CUDA
+RUN if [ "${TARGETARCH}" = "amd64" ]; then \
+        ARCH=x86_64; \
+    else \
+        ARCH=${TARGETARCH}; \
+    fi \
+    && wget https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/${ARCH}/cuda-archive-keyring.gpg \
+    && mv cuda-archive-keyring.gpg /usr/share/keyrings/cuda-archive-keyring.gpg \
+    && echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/${ARCH}/ /" | tee /etc/apt/sources.list.d/cuda-${DISTRO}-${ARCH}.list \
+    && wget https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/${ARCH}/cuda-${DISTRO}.pin \
+    && mv cuda-${DISTRO}.pin /etc/apt/preferences.d/cuda-repository-pin-600 \
+    && apt-get update
+
 WORKDIR /package
 COPY packages.yaml .
 

hack/gen-repository-iso/packages.yaml

@@ -39,6 +39,9 @@ debs:
   - openssh-server
   - software-properties-common
   - sudo
+  - cuda-toolkit-12-4
+  - nvidia-driver-550-open
+  - cuda-drivers-550
 
 centos:
   - containerd.io