Merge pull request #997 from wzshiming/feat/kind-nerdctl

[kwokctl] Support for all about nerdctl as runtime

hack/requirements.sh

@@ -23,7 +23,7 @@ LOCAL_BIN_DIR="${ROOT_DIR}/bin"
 
 export PATH="${LOCAL_BIN_DIR}:${PATH}"
 
-KIND_VERSION=0.19.0
+KIND_VERSION=0.23.0
 
 KUBE_VERSION=1.29.0
 

pkg/consts/consts.go

@@ -41,7 +41,7 @@ var (
 
 	EtcdBinaryPrefix = "https://github.com/etcd-io/etcd/releases/download"
 
-	KindVersion         = "0.19.0"
+	KindVersion         = "0.23.0"
 	KindBinaryPrefix    = "https://github.com/kubernetes-sigs/kind/releases/download"
 	KindNodeImagePrefix = "docker.io/kindest"
 
@@ -69,12 +69,34 @@ var (
 
 // The following runtime is provided.
 const (
-	RuntimeTypeKind       = "kind"
+	// RuntimeTypeBinary is the binary runtime.
+	RuntimeTypeBinary = "binary"
+
+	// Container runtime type, will create a container for each component.
+
+	// RuntimeTypeDocker is the docker runtime.
+	RuntimeTypeDocker = "docker"
+	// RuntimeTypePodman is the podman runtime.
+	RuntimeTypePodman = "podman"
+	// RuntimeTypeNerdctl is the nerdctl runtime.
+	RuntimeTypeNerdctl = "nerdctl"
+	// RuntimeTypeLima is the lima runtime.
+	RuntimeTypeLima = "lima"
+	// RuntimeTypeFinch is the finch runtime.
+	RuntimeTypeFinch = "finch"
+
+	// Cluster runtime type, creates a cluster and deploys the components in the cluster.
+
+	// RuntimeTypeKind is the kind runtime.
+	RuntimeTypeKind = "kind"
+	// RuntimeTypeKindPodman is the kind runtime with podman.
 	RuntimeTypeKindPodman = RuntimeTypeKind + "-" + RuntimeTypePodman
-	RuntimeTypeDocker     = "docker"
-	RuntimeTypeNerdctl    = "nerdctl"
-	RuntimeTypePodman     = "podman"
-	RuntimeTypeBinary     = "binary"
+	// RuntimeTypeKindNerdctl is the kind runtime with nerdctl.
+	RuntimeTypeKindNerdctl = RuntimeTypeKind + "-" + RuntimeTypeNerdctl
+	// RuntimeTypeKindLima is the kind runtime with lima.
+	RuntimeTypeKindLima = RuntimeTypeKind + "-" + RuntimeTypeLima
+	// RuntimeTypeKindFinch is the kind runtime with finch.
+	RuntimeTypeKindFinch = RuntimeTypeKind + "-" + RuntimeTypeFinch
 )
 
 // The following components is provided.

pkg/kwokctl/cmd/create/cluster/cluster.go

@@ -318,7 +318,10 @@ func runE(ctx context.Context, flags *flagpole) error {
 		}
 
 		if flags.Options.Runtime == consts.RuntimeTypeKind ||
-			flags.Options.Runtime == consts.RuntimeTypeKindPodman {
+			flags.Options.Runtime == consts.RuntimeTypeKindPodman ||
+			flags.Options.Runtime == consts.RuntimeTypeKindNerdctl ||
+			flags.Options.Runtime == consts.RuntimeTypeKindLima ||
+			flags.Options.Runtime == consts.RuntimeTypeKindFinch {
 			// override kubeconfig for kind
 			defer setContext()
 		} else {

pkg/kwokctl/components/utils.go

@@ -76,12 +76,17 @@ const (
 
 var (
 	runtimeTypeMap = map[string]string{
-		consts.RuntimeTypeKind:       RuntimeModeCluster,
-		consts.RuntimeTypeKindPodman: RuntimeModeCluster,
-		consts.RuntimeTypeDocker:     RuntimeModeContainer,
-		consts.RuntimeTypeNerdctl:    RuntimeModeContainer,
-		consts.RuntimeTypePodman:     RuntimeModeContainer,
-		consts.RuntimeTypeBinary:     RuntimeModeNative,
+		consts.RuntimeTypeBinary:      RuntimeModeNative,
+		consts.RuntimeTypeDocker:      RuntimeModeContainer,
+		consts.RuntimeTypePodman:      RuntimeModeContainer,
+		consts.RuntimeTypeNerdctl:     RuntimeModeContainer,
+		consts.RuntimeTypeLima:        RuntimeModeContainer,
+		consts.RuntimeTypeFinch:       RuntimeModeContainer,
+		consts.RuntimeTypeKind:        RuntimeModeCluster,
+		consts.RuntimeTypeKindPodman:  RuntimeModeCluster,
+		consts.RuntimeTypeKindNerdctl: RuntimeModeCluster,
+		consts.RuntimeTypeKindLima:    RuntimeModeCluster,
+		consts.RuntimeTypeKindFinch:   RuntimeModeCluster,
 	}
 )
 

pkg/kwokctl/runtime/cluster.go

@@ -180,6 +180,9 @@ func (c *Cluster) Save(ctx context.Context) error {
 	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.StageKind) &&
 		conf.Options.Runtime != consts.RuntimeTypeKind &&
 		conf.Options.Runtime != consts.RuntimeTypeKindPodman &&
+		conf.Options.Runtime != consts.RuntimeTypeKindNerdctl &&
+		conf.Options.Runtime != consts.RuntimeTypeKindLima &&
+		conf.Options.Runtime != consts.RuntimeTypeKindFinch &&
 		len(config.FilterWithTypeFromContext[*internalversion.Stage](ctx)) == 0 {
 		defaultStages, err := c.getDefaultStages(conf.Options.NodeStatusUpdateFrequencyMilliseconds, conf.Options.NodeLeaseDurationSeconds != 0)
 		if err != nil {
@@ -191,6 +194,9 @@ func (c *Cluster) Save(ctx context.Context) error {
 	if !slices.Contains(conf.Options.EnableCRDs, v1alpha1.StageKind) {
 		if conf.Options.Runtime != consts.RuntimeTypeKind &&
 			conf.Options.Runtime != consts.RuntimeTypeKindPodman &&
+			conf.Options.Runtime != consts.RuntimeTypeKindNerdctl &&
+			conf.Options.Runtime != consts.RuntimeTypeKindLima &&
+			conf.Options.Runtime != consts.RuntimeTypeKindFinch &&
 			len(config.FilterWithTypeFromContext[*internalversion.Stage](ctx)) == 0 {
 			defaultStages, err := c.getDefaultStages(conf.Options.NodeStatusUpdateFrequencyMilliseconds, conf.Options.NodeLeaseDurationSeconds != 0)
 			if err != nil {

pkg/kwokctl/runtime/compose/cluster.go

@@ -49,9 +49,18 @@ type Cluster struct {
 
 	runtime string
 
+	isNerdctl               bool
 	canNerdctlUnlessStopped *bool
 }
 
+// NewDockerCluster creates a new Runtime for docker.
+func NewDockerCluster(name, workdir string) (runtime.Runtime, error) {
+	return &Cluster{
+		Cluster: runtime.NewCluster(name, workdir),
+		runtime: consts.RuntimeTypeDocker,
+	}, nil
+}
+
 // NewPodmanCluster creates a new Runtime for podman.
 func NewPodmanCluster(name, workdir string) (runtime.Runtime, error) {
 	return &Cluster{
@@ -63,16 +72,27 @@ func NewPodmanCluster(name, workdir string) (runtime.Runtime, error) {
 // NewNerdctlCluster creates a new Runtime for nerdctl.
 func NewNerdctlCluster(name, workdir string) (runtime.Runtime, error) {
 	return &Cluster{
-		Cluster: runtime.NewCluster(name, workdir),
-		runtime: consts.RuntimeTypeNerdctl,
+		Cluster:   runtime.NewCluster(name, workdir),
+		runtime:   consts.RuntimeTypeNerdctl,
+		isNerdctl: true,
 	}, nil
 }
 
-// NewDockerCluster creates a new Runtime for docker.
-func NewDockerCluster(name, workdir string) (runtime.Runtime, error) {
+// NewLimaCluster creates a new Runtime for lima.
+func NewLimaCluster(name, workdir string) (runtime.Runtime, error) {
 	return &Cluster{
-		Cluster: runtime.NewCluster(name, workdir),
-		runtime: consts.RuntimeTypeDocker,
+		Cluster:   runtime.NewCluster(name, workdir),
+		runtime:   consts.RuntimeTypeNerdctl + "." + consts.RuntimeTypeLima,
+		isNerdctl: true,
+	}, nil
+}
+
+// NewFinchCluster creates a new Runtime for finch.
+func NewFinchCluster(name, workdir string) (runtime.Runtime, error) {
+	return &Cluster{
+		Cluster:   runtime.NewCluster(name, workdir),
+		runtime:   consts.RuntimeTypeFinch,
+		isNerdctl: true,
 	}, nil
 }
 
@@ -903,7 +923,7 @@ func (c *Cluster) Stop(ctx context.Context) error {
 }
 
 func (c *Cluster) start(ctx context.Context) error {
-	if c.runtime == consts.RuntimeTypeNerdctl {
+	if c.isNerdctl {
 		canNerdctlUnlessStopped, _ := c.isCanNerdctlUnlessStopped(ctx)
 		if !canNerdctlUnlessStopped {
 			// TODO: Remove this, nerdctl stop will restart containers
@@ -925,7 +945,7 @@ func (c *Cluster) start(ctx context.Context) error {
 		return err
 	}
 
-	if c.runtime == consts.RuntimeTypeNerdctl {
+	if c.isNerdctl {
 		canNerdctlUnlessStopped, _ := c.isCanNerdctlUnlessStopped(ctx)
 		if !canNerdctlUnlessStopped {
 			backupFilename := c.GetWorkdirPath("restart.db")
@@ -949,7 +969,7 @@ func (c *Cluster) start(ctx context.Context) error {
 }
 
 func (c *Cluster) stop(ctx context.Context) error {
-	if c.runtime == consts.RuntimeTypeNerdctl {
+	if c.isNerdctl {
 		canNerdctlUnlessStopped, _ := c.isCanNerdctlUnlessStopped(ctx)
 		if !canNerdctlUnlessStopped {
 			err := c.SnapshotSave(ctx, c.GetWorkdirPath("restart.db"))
@@ -968,7 +988,7 @@ func (c *Cluster) stop(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	if c.runtime == consts.RuntimeTypeNerdctl {
+	if c.isNerdctl {
 		canNerdctlUnlessStopped, _ := c.isCanNerdctlUnlessStopped(ctx)
 		if !canNerdctlUnlessStopped {
 			// TODO: Remove this, nerdctl stop will restart containers

pkg/kwokctl/runtime/compose/cluster_snapshot.go

@@ -71,7 +71,7 @@ func (c *Cluster) SnapshotRestore(ctx context.Context, path string) error {
 	}()
 
 	etcdContainerName := c.Name() + "-etcd"
-	if conf.Runtime != consts.RuntimeTypeNerdctl {
+	if !c.isNerdctl {
 		// Restart etcd and kube-apiserver
 		components := []string{
 			consts.ComponentEtcd,

pkg/kwokctl/runtime/compose/init.go

@@ -23,6 +23,8 @@ import (
 
 func init() {
 	runtime.DefaultRegistry.Register(consts.RuntimeTypeDocker, NewDockerCluster)
-	runtime.DefaultRegistry.Register(consts.RuntimeTypeNerdctl, NewNerdctlCluster)
 	runtime.DefaultRegistry.Register(consts.RuntimeTypePodman, NewPodmanCluster)
+	runtime.DefaultRegistry.Register(consts.RuntimeTypeNerdctl, NewNerdctlCluster)
+	runtime.DefaultRegistry.Register(consts.RuntimeTypeLima, NewLimaCluster)
+	runtime.DefaultRegistry.Register(consts.RuntimeTypeFinch, NewFinchCluster)
 }

pkg/kwokctl/runtime/compose/self_compose.go

@@ -72,7 +72,7 @@ func (c *Cluster) deleteNetwork(ctx context.Context) error {
 	logger.Debug("Deleting network")
 	err := c.Exec(ctx, c.runtime, args...)
 	if err != nil {
-		if c.runtime != consts.RuntimeTypeNerdctl {
+		if !c.isNerdctl {
 			return err
 		}
 
@@ -118,7 +118,7 @@ func (c *Cluster) inspectNetwork(ctx context.Context, name string) (exist bool)
 // On Nerdctl, need to check if --restart=unless-stopped is supported
 // https://github.com/containerd/containerd/pull/6744
 func (c *Cluster) isCanNerdctlUnlessStopped(ctx context.Context) (bool, error) {
-	if c.runtime != consts.RuntimeTypeNerdctl {
+	if !c.isNerdctl {
 		return false, fmt.Errorf("canNerdctlUnlessStopped only for nerdctl")
 	}
 
@@ -171,9 +171,11 @@ func (c *Cluster) labelArgs() []string {
 		// https://github.com/containers/podman-compose/blob/f6dbce36181c44d0d08b6f4ca166508542875ce1/podman_compose.py#L729
 		args = append(args, "--label=io.podman.compose.project="+c.Name())
 		args = append(args, "--label=com.docker.compose.project="+c.Name())
-	case consts.RuntimeTypeNerdctl:
-		// https://github.com/containerd/nerdctl/blob/3c9300207f45c4a0422d8381d58c5be06bb49b39/pkg/labels/labels.go#L33
-		args = append(args, "--label=com.docker.compose.project="+c.Name())
+	default:
+		if c.isNerdctl {
+			// https://github.com/containerd/nerdctl/blob/3c9300207f45c4a0422d8381d58c5be06bb49b39/pkg/labels/labels.go#L33
+			args = append(args, "--label=com.docker.compose.project="+c.Name())
+		}
 	}
 	return args
 }
@@ -226,22 +228,24 @@ func (c *Cluster) createComponent(ctx context.Context, componentName string) err
 		for _, link := range component.Links {
 			args = append(args, "--requires="+c.Name()+"-"+link)
 		}
-	case consts.RuntimeTypeNerdctl:
+	default:
 		// Nerdctl does not support --link and --requires
 	}
 
 	switch c.runtime {
 	case consts.RuntimeTypeDocker, consts.RuntimeTypePodman:
 		args = append(args, "--restart=unless-stopped")
-	case consts.RuntimeTypeNerdctl:
-		canNerdctlUnlessStopped, err := c.isCanNerdctlUnlessStopped(ctx)
-		if err != nil {
-			logger.Error("Failed to check unless-stopped support", err)
-		}
-		if canNerdctlUnlessStopped {
-			args = append(args, "--restart=unless-stopped")
-		} else {
-			args = append(args, "--restart=always")
+	default:
+		if c.isNerdctl {
+			canNerdctlUnlessStopped, err := c.isCanNerdctlUnlessStopped(ctx)
+			if err != nil {
+				logger.Error("Failed to check unless-stopped support", err)
+			}
+			if canNerdctlUnlessStopped {
+				args = append(args, "--restart=unless-stopped")
+			} else {
+				args = append(args, "--restart=always")
+			}
 		}
 	}
 
@@ -291,7 +295,7 @@ func (c *Cluster) deleteComponent(ctx context.Context, componentName string) err
 			logger.Debug("Component does not exist")
 			return nil
 		} else if running {
-			if c.runtime == consts.RuntimeTypeNerdctl {
+			if c.isNerdctl {
 				// TODO: Remove this after nerdctl fix
 				// https://github.com/containerd/nerdctl/issues/1980
 				if canNerdctlUnlessStopped, _ := c.isCanNerdctlUnlessStopped(ctx); canNerdctlUnlessStopped {
@@ -402,7 +406,7 @@ func (c *Cluster) startComponent(ctx context.Context, componentName string) erro
 	if err != nil {
 		// TODO: Remove this after nerdctl fix
 		// https://github.com/containerd/nerdctl/issues/2270
-		if c.runtime == consts.RuntimeTypeNerdctl {
+		if c.isNerdctl {
 			errMessage := err.Error()
 			switch {
 			case strings.Contains(errMessage, "already exists"),

pkg/kwokctl/runtime/kind/cluster.go

@@ -73,6 +73,30 @@ func NewPodmanCluster(name, workdir string) (runtime.Runtime, error) {
 	}, nil
 }
 
+// NewNerdctlCluster creates a new Runtime for kind with nerdctl
+func NewNerdctlCluster(name, workdir string) (runtime.Runtime, error) {
+	return &Cluster{
+		Cluster: runtime.NewCluster(name, workdir),
+		runtime: consts.RuntimeTypeNerdctl,
+	}, nil
+}
+
+// NewLimaCluster creates a new Runtime for kind with lima
+func NewLimaCluster(name, workdir string) (runtime.Runtime, error) {
+	return &Cluster{
+		Cluster: runtime.NewCluster(name, workdir),
+		runtime: consts.RuntimeTypeNerdctl + "." + consts.RuntimeTypeLima,
+	}, nil
+}
+
+// NewFinchCluster creates a new Runtime for kind with finch
+func NewFinchCluster(name, workdir string) (runtime.Runtime, error) {
+	return &Cluster{
+		Cluster: runtime.NewCluster(name, workdir),
+		runtime: consts.RuntimeTypeFinch,
+	}, nil
+}
+
 // Available  checks whether the runtime is available.
 func (c *Cluster) Available(ctx context.Context) error {
 	if c.IsDryRun() {

pkg/kwokctl/runtime/kind/init.go

@@ -24,4 +24,7 @@ import (
 func init() {
 	runtime.DefaultRegistry.Register(consts.RuntimeTypeKind, NewDockerCluster)
 	runtime.DefaultRegistry.Register(consts.RuntimeTypeKindPodman, NewPodmanCluster)
+	runtime.DefaultRegistry.Register(consts.RuntimeTypeKindNerdctl, NewNerdctlCluster)
+	runtime.DefaultRegistry.Register(consts.RuntimeTypeKindLima, NewLimaCluster)
+	runtime.DefaultRegistry.Register(consts.RuntimeTypeKindFinch, NewFinchCluster)
 }

site/content/en/_index.md

@@ -55,22 +55,26 @@ and [contribution to KWOK][contributor guide].
 
 Runtime indicates which medium `kwokctl` will use to start the cluster
 
-|    OS/Arch    | binary | docker | podman | nerdctl | kind  | kind-podman |
-|:-------------:|:------:|:------:|:------:|:-------:|:-----:|:-----------:|
-|  linux/amd64  |   🟢   |   🟢   |   🟢   |   🟢    |  🟢   |     🟢      |
-|  linux/arm64  |   🔵   |   🔵   |   🔵   |   🔵    |  🔵   |     🔵      |
-| darwin/amd64  | 🟢/🟠  |   🔵   |   🔵   |  🔵/🟡  |  🔵   |     🔵      |
-| darwin/arm64  | 🟢/🟠  |   🔵   |   🔵   |  🔵/🟡  |  🔵   |     🔵      |
-| windows/amd64 | 🟢/🟠  |   🟣   |   🟤   |  🟤/🟡  |  🟤   |     🟤      |
-| windows/arm64 | 🟣/🟠  |   🟣   |   🔴   |   🔴    |  🔴   |     🔴      |
-
+|              \              | linux/<br/>amd64 | linux/<br/>arm64 | darwin/<br/>amd64 | darwin/<br/>arm64 | windows/<br/>amd64 | windows/<br/>arm64  |
+|:---------------------------:|:----------------:|:----------------:|:-----------------:|:-----------------:|:------------------:|:-------------------:|
+| [binary][binary-runtime] ⭐️ |        🟢        |        🔵        |        🟢         |        🟢         |         🟢         |         🟣          |
+| [docker][docker-runtime] ⭐️ |        🟢        |        🔵        |        🔵         |        🔵         |         🟣         |         🟣          |
+| [podman][podman-runtime] ⭐️ |        🟢        |        🔵        |        🔵         |        🔵         |         🟣         |         🟣          |
+| [nerdctl][nerdctl-runtime]  |        🟢        |        🔵        |        🔴         |        🔴         |         🔴         |         🔴          |
+|   [lima][lima-runtime] ⚠️   |        🟣        |        🟣        |        🟣         |        🟣         |         🔴         |         🔴          |
+|  [finch][finch-runtime] ⚠️  |        🔴        |        🔴        |        🟣         |        🟣         |         🟣         |         🟣          |
+|    [kind][kind-runtime]     |        🟢        |        🔵        |        🔵         |        🔵         |         🟣         |         🟣          |
+|       **kind-podman**       |        🟢        |        🔵        |        🔵         |        🔵         |         🟣         |         🟣          |
+|     **kind-nerdctl** ⚠️     |        🟣        |        🟣        |        🔴         |        🔴         |         🔴         |         🔴          |
+|      **kind-lima** ⚠️       |        🟣        |        🟣        |        🟣         |        🟣         |         🔴         |         🔴          |
+|      **kind-finch** ⚠️      |        🔴        |        🔴        |        🟣         |        🟣         |         🟣         |         🟣          |
+
+- ⭐️ Recommended
+- ⚠️ Work in progress
 - 🟢 Supported and test covered by CI
 - 🔵 Supported and test by manually
-- 🟣 Supported but not test yet (need help to verify)
-- 🟤 Unsupported but should work in theory (need help to verify)
-- 🔴 Unsupported and will not work
-- 🟠 Need to build [platform-specific Kubernetes binaries]
-- 🟡 Need to do some workaround to make it work
+- 🟣 Supported but not fully tested
+- 🔴 Unsupported yet
 
 ## Community
 
@@ -101,3 +105,11 @@ Participation in the Kubernetes community is governed by the [Kubernetes Code of
 [run it using pre-built images]: {{< relref "/docs/user/all-in-one-image" >}}
 [simulate thousands of nodes]: {{< relref "/docs/user/kwok-manage-nodes-and-pods" >}}
 [platform-specific Kubernetes binaries]: {{< relref "/docs/user/kwokctl-platform-specific-binaries" >}}
+
+[binary-runtime]: https://kwok.sigs.k8s.io/docs/user/kwokctl-platform-specific-binaries/
+[docker-runtime]: https://docs.docker.com/get-docker/
+[podman-runtime]: https://podman.io/docs/installation
+[nerdctl-runtime]: https://github.com/containerd/nerdctl/releases
+[lima-runtime]: https://lima-vm.io/docs/installation/
+[finch-runtime]: https://runfinch.com/docs/getting-started/installation/
+[kind-runtime]: https://kind.sigs.k8s.io/docs/user/quick-start/