From 0bec29ae47ae0bcf2323589939ebe1b79a221167 Mon Sep 17 00:00:00 2001 From: Shiming Zhang Date: Tue, 14 Nov 2023 09:55:00 +0800 Subject: [PATCH] Add kwokctl component api --- kustomize/kwokctl/component/dashboard.yaml | 55 +++++ kustomize/kwokctl/component/embed.go | 28 +++ .../kwokctl/component/kustomization.yaml | 4 + kustomize/kwokctl/kustomization.yaml | 1 + .../v1alpha1/kwokctl_component_types.go | 43 ++++ .../config/v1alpha1/zz_generated.deepcopy.go | 31 +++ pkg/apis/internalversion/conversion.go | 22 ++ .../kwokctl_component_types.go | 34 ++++ .../zz_generated.conversion.go | 35 ++++ .../internalversion/zz_generated.deepcopy.go | 22 ++ pkg/config/config.go | 6 + pkg/kwokctl/cmd/create/cluster/cluster.go | 48 ++++- pkg/kwokctl/components/dashboard.go | 117 ----------- pkg/kwokctl/runtime/binary/component.go | 26 +++ pkg/kwokctl/runtime/compose/cluster.go | 67 ++----- pkg/kwokctl/runtime/compose/component.go | 162 +++++++++++++++ pkg/kwokctl/runtime/config.go | 3 + pkg/kwokctl/runtime/kind/cluster.go | 83 ++++---- pkg/kwokctl/runtime/kind/component.go | 188 ++++++++++++++++++ pkg/kwokctl/runtime/kind/kind.go | 3 +- pkg/kwokctl/runtime/kind/kind.yaml.tpl | 12 +- pkg/utils/slices/slices.go | 15 ++ .../testdata/docker/create_cluster.txt | 2 +- .../docker/create_cluster_with_extra.txt | 12 +- .../docker/create_cluster_with_verbosity.txt | 20 +- .../create_cluster_with_verbosity.txt | 136 ++++++------- .../kind/create_cluster_with_verbosity.txt | 133 ++++++------- .../testdata/nerdctl/create_cluster.txt | 2 +- .../nerdctl/create_cluster_with_extra.txt | 12 +- .../nerdctl/create_cluster_with_verbosity.txt | 20 +- .../testdata/podman/create_cluster.txt | 2 +- .../podman/create_cluster_with_extra.txt | 12 +- .../podman/create_cluster_with_verbosity.txt | 20 +- 33 files changed, 965 insertions(+), 411 deletions(-) create mode 100644 kustomize/kwokctl/component/dashboard.yaml create mode 100644 kustomize/kwokctl/component/embed.go create mode 100644 kustomize/kwokctl/component/kustomization.yaml create mode 100644 pkg/apis/config/v1alpha1/kwokctl_component_types.go create mode 100644 pkg/apis/internalversion/kwokctl_component_types.go delete mode 100644 pkg/kwokctl/components/dashboard.go create mode 100644 pkg/kwokctl/runtime/binary/component.go create mode 100644 pkg/kwokctl/runtime/compose/component.go create mode 100644 pkg/kwokctl/runtime/kind/component.go diff --git a/kustomize/kwokctl/component/dashboard.yaml b/kustomize/kwokctl/component/dashboard.yaml new file mode 100644 index 000000000..0d76e6ba1 --- /dev/null +++ b/kustomize/kwokctl/component/dashboard.yaml @@ -0,0 +1,55 @@ +apiVersion: config.kwok.x-k8s.io/v1alpha1 +kind: KwokctlComponent +metadata: + name: dashboard +parameters: + image: "" + binary: "" + version: "v2.7.0" + bindAddress: "0.0.0.0" + port: 8000 +template: |- + {{ $version := + ( or + ( env "KWOK_DASHBOARD_VERSION" ) + .version + ) + }} + {{ $image := + ( or + .image + ( env "KWOK_DASHBOARD_IMAGE" ) + ( join + "" + ( list + ( or + ( env "KWOK_DASHBOARD_IMAGE_PREFIX" ) + "docker.io/kubernetesui" + ) + "/dashboard:" + $version + ) + ) + ) + }} + + image: {{ $image }} + binary: {{ .binary }} + links: + - kube-apiserver + ports: + - name: http + port: {{ .port }} + hostPort: {{ .port }} + protocol: TCP + args: + - --insecure-bind-address={{ .bindAddress }} + - --insecure-port={{ .port }} + - --bind-address=127.0.0.1 + - --port=0 + - --enable-insecure-login + - --enable-skip-login + - --disable-settings-authorizer + - --metrics-provider=none + - --system-banner=Welcome to {{ ClusterName }} + - --kubeconfig={{ Kubeconfig }} diff --git a/kustomize/kwokctl/component/embed.go b/kustomize/kwokctl/component/embed.go new file mode 100644 index 000000000..167cb98ac --- /dev/null +++ b/kustomize/kwokctl/component/embed.go @@ -0,0 +1,28 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package component contains default component for kwokctl. +package component + +import ( + _ "embed" +) + +var ( + // DefaultDashboard is the default dashboard component. + //go:embed dashboard.yaml + DefaultDashboard string +) diff --git a/kustomize/kwokctl/component/kustomization.yaml b/kustomize/kwokctl/component/kustomization.yaml new file mode 100644 index 000000000..7b9ba3236 --- /dev/null +++ b/kustomize/kwokctl/component/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- dashboard.yaml diff --git a/kustomize/kwokctl/kustomization.yaml b/kustomize/kwokctl/kustomization.yaml index 2948fa173..c10c9ac13 100644 --- a/kustomize/kwokctl/kustomization.yaml +++ b/kustomize/kwokctl/kustomization.yaml @@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./resource +- ./component diff --git a/pkg/apis/config/v1alpha1/kwokctl_component_types.go b/pkg/apis/config/v1alpha1/kwokctl_component_types.go new file mode 100644 index 000000000..c73e2e4b2 --- /dev/null +++ b/pkg/apis/config/v1alpha1/kwokctl_component_types.go @@ -0,0 +1,43 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "encoding/json" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // KwokctlComponentKind is the kind of the kwokctl component. + KwokctlComponentKind = "KwokctlComponent" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// KwokctlComponent holds information about the kwokctl component. +type KwokctlComponent struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata,omitempty"` + // Parameters is the parameters for the kwokctl component configuration. + Parameters json.RawMessage `json:"parameters,omitempty"` + // Template is the template for the kwokctl component configuration. + Template string `json:"template,omitempty"` +} diff --git a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index 443fc8af2..b901b241c 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -213,6 +213,37 @@ func (in *KwokConfigurationOptions) DeepCopy() *KwokConfigurationOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KwokctlComponent) DeepCopyInto(out *KwokctlComponent) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(json.RawMessage, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KwokctlComponent. +func (in *KwokctlComponent) DeepCopy() *KwokctlComponent { + if in == nil { + return nil + } + out := new(KwokctlComponent) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KwokctlComponent) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KwokctlConfiguration) DeepCopyInto(out *KwokctlConfiguration) { *out = *in diff --git a/pkg/apis/internalversion/conversion.go b/pkg/apis/internalversion/conversion.go index c4b5db997..739ca552e 100644 --- a/pkg/apis/internalversion/conversion.go +++ b/pkg/apis/internalversion/conversion.go @@ -66,6 +66,28 @@ func ConvertToInternalKwokctlResource(in *configv1alpha1.KwokctlResource) (*Kwok return &out, nil } +// ConvertToV1alpha1KwokctlComponent converts an internal version KwokctlComponent to a v1alpha1.KwokctlComponent. +func ConvertToV1alpha1KwokctlComponent(in *KwokctlComponent) (*configv1alpha1.KwokctlComponent, error) { + var out configv1alpha1.KwokctlComponent + out.APIVersion = configv1alpha1.GroupVersion.String() + out.Kind = configv1alpha1.KwokctlComponentKind + err := Convert_internalversion_KwokctlComponent_To_v1alpha1_KwokctlComponent(in, &out, nil) + if err != nil { + return nil, err + } + return &out, nil +} + +// ConvertToInternalKwokctlComponent converts a v1alpha1.KwokctlComponent to an internal version. +func ConvertToInternalKwokctlComponent(in *configv1alpha1.KwokctlComponent) (*KwokctlComponent, error) { + var out KwokctlComponent + err := Convert_v1alpha1_KwokctlComponent_To_internalversion_KwokctlComponent(in, &out, nil) + if err != nil { + return nil, err + } + return &out, nil +} + // ConvertToV1alpha1KwokConfiguration converts an internal version KwokConfiguration to a v1alpha1.KwokConfiguration. func ConvertToV1alpha1KwokConfiguration(in *KwokConfiguration) (*configv1alpha1.KwokConfiguration, error) { var out configv1alpha1.KwokConfiguration diff --git a/pkg/apis/internalversion/kwokctl_component_types.go b/pkg/apis/internalversion/kwokctl_component_types.go new file mode 100644 index 000000000..ee8a3b540 --- /dev/null +++ b/pkg/apis/internalversion/kwokctl_component_types.go @@ -0,0 +1,34 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internalversion + +import ( + "encoding/json" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// KwokctlComponent provides component definition for kwokctl. +type KwokctlComponent struct { + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta + // Parameters is the parameters for the kwokctl component configuration. + Parameters json.RawMessage + // Template is the template for the kwokctl component configuration. + Template string +} diff --git a/pkg/apis/internalversion/zz_generated.conversion.go b/pkg/apis/internalversion/zz_generated.conversion.go index 31cf53b27..786bc3fbf 100644 --- a/pkg/apis/internalversion/zz_generated.conversion.go +++ b/pkg/apis/internalversion/zz_generated.conversion.go @@ -299,6 +299,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*KwokctlComponent)(nil), (*configv1alpha1.KwokctlComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_internalversion_KwokctlComponent_To_v1alpha1_KwokctlComponent(a.(*KwokctlComponent), b.(*configv1alpha1.KwokctlComponent), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*configv1alpha1.KwokctlComponent)(nil), (*KwokctlComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_KwokctlComponent_To_internalversion_KwokctlComponent(a.(*configv1alpha1.KwokctlComponent), b.(*KwokctlComponent), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*KwokctlConfiguration)(nil), (*configv1alpha1.KwokctlConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_internalversion_KwokctlConfiguration_To_v1alpha1_KwokctlConfiguration(a.(*KwokctlConfiguration), b.(*configv1alpha1.KwokctlConfiguration), scope) }); err != nil { @@ -1369,6 +1379,31 @@ func Convert_v1alpha1_KwokConfigurationOptions_To_internalversion_KwokConfigurat return autoConvert_v1alpha1_KwokConfigurationOptions_To_internalversion_KwokConfigurationOptions(in, out, s) } +func autoConvert_internalversion_KwokctlComponent_To_v1alpha1_KwokctlComponent(in *KwokctlComponent, out *configv1alpha1.KwokctlComponent, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + out.Parameters = *(*json.RawMessage)(unsafe.Pointer(&in.Parameters)) + out.Template = in.Template + return nil +} + +// Convert_internalversion_KwokctlComponent_To_v1alpha1_KwokctlComponent is an autogenerated conversion function. +func Convert_internalversion_KwokctlComponent_To_v1alpha1_KwokctlComponent(in *KwokctlComponent, out *configv1alpha1.KwokctlComponent, s conversion.Scope) error { + return autoConvert_internalversion_KwokctlComponent_To_v1alpha1_KwokctlComponent(in, out, s) +} + +func autoConvert_v1alpha1_KwokctlComponent_To_internalversion_KwokctlComponent(in *configv1alpha1.KwokctlComponent, out *KwokctlComponent, s conversion.Scope) error { + // INFO: in.TypeMeta opted out of conversion generation + out.ObjectMeta = in.ObjectMeta + out.Parameters = *(*json.RawMessage)(unsafe.Pointer(&in.Parameters)) + out.Template = in.Template + return nil +} + +// Convert_v1alpha1_KwokctlComponent_To_internalversion_KwokctlComponent is an autogenerated conversion function. +func Convert_v1alpha1_KwokctlComponent_To_internalversion_KwokctlComponent(in *configv1alpha1.KwokctlComponent, out *KwokctlComponent, s conversion.Scope) error { + return autoConvert_v1alpha1_KwokctlComponent_To_internalversion_KwokctlComponent(in, out, s) +} + func autoConvert_internalversion_KwokctlConfiguration_To_v1alpha1_KwokctlConfiguration(in *KwokctlConfiguration, out *configv1alpha1.KwokctlConfiguration, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_internalversion_KwokctlConfigurationOptions_To_v1alpha1_KwokctlConfigurationOptions(&in.Options, &out.Options, s); err != nil { diff --git a/pkg/apis/internalversion/zz_generated.deepcopy.go b/pkg/apis/internalversion/zz_generated.deepcopy.go index 2c03bf13f..ec6bc260c 100644 --- a/pkg/apis/internalversion/zz_generated.deepcopy.go +++ b/pkg/apis/internalversion/zz_generated.deepcopy.go @@ -607,6 +607,28 @@ func (in *KwokConfigurationOptions) DeepCopy() *KwokConfigurationOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KwokctlComponent) DeepCopyInto(out *KwokctlComponent) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(json.RawMessage, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KwokctlComponent. +func (in *KwokctlComponent) DeepCopy() *KwokctlComponent { + if in == nil { + return nil + } + out := new(KwokctlComponent) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KwokctlConfiguration) DeepCopyInto(out *KwokctlConfiguration) { *out = *in diff --git a/pkg/config/config.go b/pkg/config/config.go index 835a177b8..e5c80fdbb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -107,6 +107,12 @@ var configHandlers = map[string]configHandler{ MutateToInternal: mutateToInternalConfig(internalversion.ConvertToInternalKwokctlResource), MutateToVersiond: mutateToVersiondConfig(internalversion.ConvertToV1alpha1KwokctlResource), }, + configv1alpha1.KwokctlComponentKind: { + Unmarshal: unmarshalConfig[*configv1alpha1.KwokctlComponent], + Marshal: marshalConfig, + MutateToInternal: mutateToInternalConfig(internalversion.ConvertToInternalKwokctlComponent), + MutateToVersiond: mutateToVersiondConfig(internalversion.ConvertToV1alpha1KwokctlComponent), + }, v1alpha1.StageKind: { Unmarshal: unmarshalConfig[*v1alpha1.Stage], Marshal: marshalConfig, diff --git a/pkg/kwokctl/cmd/create/cluster/cluster.go b/pkg/kwokctl/cmd/create/cluster/cluster.go index b5e462e08..91fa3c240 100644 --- a/pkg/kwokctl/cmd/create/cluster/cluster.go +++ b/pkg/kwokctl/cmd/create/cluster/cluster.go @@ -21,6 +21,7 @@ import ( "context" "fmt" "os" + "strconv" "strings" "time" @@ -31,8 +32,10 @@ import ( "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/kwokctl/runtime" "sigs.k8s.io/kwok/pkg/log" + "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/kubeconfig" "sigs.k8s.io/kwok/pkg/utils/path" + "sigs.k8s.io/kwok/pkg/utils/slices" ) type flagpole struct { @@ -41,6 +44,9 @@ type flagpole struct { Wait time.Duration Kubeconfig string + Param []string + EnableComponents []string + *internalversion.KwokctlConfiguration } @@ -97,9 +103,11 @@ func NewCommand(ctx context.Context) *cobra.Command { '${KWOK_KIND_NODE_IMAGE_PREFIX}/node:${KWOK_KUBE_VERSION}' `) cmd.Flags().Uint32Var(&flags.Options.DashboardPort, "dashboard-port", flags.Options.DashboardPort, `Port of dashboard given to the host`) + _ = cmd.Flags().MarkDeprecated("dashboard-port", "will be removed in a future release, use --enable-components=dashboard --param=.dashboard.port= instead") cmd.Flags().StringVar(&flags.Options.DashboardImage, "dashboard-image", flags.Options.DashboardImage, `Image of dashboard, only for docker/podman/nerdctl/kind/kind-podman runtime '${KWOK_DASHBOARD_IMAGE_PREFIX}/dashboard:${KWOK_DASHBOARD_VERSION}' `) + _ = cmd.Flags().MarkDeprecated("dashboard-image", "will be removed in a future release, use --param=.dashboard.image= instead") cmd.Flags().StringVar(&flags.Options.KubeApiserverBinary, "kube-apiserver-binary", flags.Options.KubeApiserverBinary, `Binary of kube-apiserver, only for binary runtime `) cmd.Flags().StringVar(&flags.Options.KubeControllerManagerBinary, "kube-controller-manager-binary", flags.Options.KubeControllerManagerBinary, `Binary of kube-controller-manager, only for binary runtime @@ -137,11 +145,32 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().StringVar(&flags.Kubeconfig, "kubeconfig", flags.Kubeconfig, "The path to the kubeconfig file will be added to the newly created cluster and set to current-context") cmd.Flags().BoolVar(&flags.Options.DisableQPSLimits, "disable-qps-limits", flags.Options.DisableQPSLimits, "Disable QPS limits for components") cmd.Flags().StringSliceVar(&flags.Options.EnableCRDs, "enable-crds", flags.Options.EnableCRDs, "List of CRDs to enable") + cmd.Flags().StringArrayVar(&flags.Param, "param", flags.Param, "Set parameters for the cluster, format: --param=.key1=value1 --param=.key2=value2") + cmd.Flags().StringArrayVar(&flags.EnableComponents, "enable-components", flags.EnableComponents, "Enable components for the cluster") return cmd } +func mutation(flags *flagpole) error { + if flags.Options.DashboardPort != 0 { + flags.EnableComponents = append(flags.EnableComponents, "dashboard") + flags.Param = append(flags.Param, + ".dashboard.port="+format.String(flags.Options.DashboardPort), + ".dashboard.image="+strconv.Quote(flags.Options.DashboardImage), + ) + + flags.Options.DashboardPort = 0 + } + + return nil +} + func runE(ctx context.Context, flags *flagpole) error { + err := mutation(flags) + if err != nil { + return err + } + name := config.ClusterName(flags.Name) workdir := path.Join(config.ClustersDir, flags.Name) @@ -149,7 +178,6 @@ func runE(ctx context.Context, flags *flagpole) error { logger = logger.With("cluster", flags.Name) ctx = log.NewContext(ctx, logger) - var err error if flags.Kubeconfig != "" { flags.Kubeconfig, err = path.Expand(flags.Kubeconfig) if err != nil { @@ -228,6 +256,24 @@ func runE(ctx context.Context, flags *flagpole) error { cleanUp() return err } + + // Enable components + for _, componentName := range flags.EnableComponents { + componentPrefix := fmt.Sprintf(".%s.", componentName) + params := slices.FilterAndMap(flags.Param, func(s string) (string, bool) { + if !strings.HasPrefix(s, componentPrefix) { + return "", false + } + return s[len(componentPrefix)-1:], true + }) + err := rt.SetComponents(ctx, componentName, params...) + if err != nil { + logger.Error("Failed to set components", err, "component", componentName) + cleanUp() + return err + } + } + err = rt.Save(ctx) if err != nil { logger.Error("Failed to save config", err) diff --git a/pkg/kwokctl/components/dashboard.go b/pkg/kwokctl/components/dashboard.go deleted file mode 100644 index a5aa78ce7..000000000 --- a/pkg/kwokctl/components/dashboard.go +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package components - -import ( - "sigs.k8s.io/kwok/pkg/apis/internalversion" - "sigs.k8s.io/kwok/pkg/consts" - "sigs.k8s.io/kwok/pkg/utils/format" - "sigs.k8s.io/kwok/pkg/utils/version" -) - -// BuildDashboardComponentConfig is the configuration for building the dashboard component. -type BuildDashboardComponentConfig struct { - Binary string - Image string - Version version.Version - Workdir string - BindAddress string - Port uint32 - - Banner string - - CaCertPath string - AdminCertPath string - AdminKeyPath string - KubeconfigPath string -} - -// BuildDashboardComponent builds the dashboard component. -func BuildDashboardComponent(conf BuildDashboardComponentConfig) (component internalversion.Component, err error) { - dashboardArgs := []string{ - "--insecure-bind-address=" + conf.BindAddress, - "--bind-address=127.0.0.1", - "--port=0", - "--enable-insecure-login", - "--enable-skip-login", - "--disable-settings-authorizer", - "--metrics-provider=none", - } - if conf.Banner != "" { - dashboardArgs = append(dashboardArgs, "--system-banner="+conf.Banner) - } - - inContainer := conf.Image != "" - user := "" - var volumes []internalversion.Volume - var ports []internalversion.Port - if inContainer { - dashboardArgs = append(dashboardArgs, - "--kubeconfig=/root/.kube/config", - "--insecure-port=8000", - ) - volumes = append(volumes, - internalversion.Volume{ - HostPath: conf.KubeconfigPath, - MountPath: "/root/.kube/config", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: conf.CaCertPath, - MountPath: "/etc/kubernetes/pki/ca.crt", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: conf.AdminCertPath, - MountPath: "/etc/kubernetes/pki/admin.crt", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: conf.AdminKeyPath, - MountPath: "/etc/kubernetes/pki/admin.key", - ReadOnly: true, - }, - ) - ports = append(ports, - internalversion.Port{ - Name: "http", - Port: 8000, - HostPort: conf.Port, - Protocol: internalversion.ProtocolTCP, - }, - ) - } else { - dashboardArgs = append(dashboardArgs, - "--kubeconfig="+conf.KubeconfigPath, - "--insecure-port="+format.String(conf.Port), - ) - } - - component = internalversion.Component{ - Name: consts.ComponentDashboard, - Image: conf.Image, - Links: []string{ - consts.ComponentKubeApiserver, - }, - WorkDir: conf.Workdir, - Ports: ports, - Volumes: volumes, - Args: dashboardArgs, - User: user, - } - return component, nil -} diff --git a/pkg/kwokctl/runtime/binary/component.go b/pkg/kwokctl/runtime/binary/component.go new file mode 100644 index 000000000..8cf93ee8a --- /dev/null +++ b/pkg/kwokctl/runtime/binary/component.go @@ -0,0 +1,26 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package binary + +import ( + "context" +) + +// SetComponents returns the components of cluster +func (c *Cluster) SetComponents(ctx context.Context, name string, args ...string) error { + return nil +} diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index eb91d76bd..6e501e5c1 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -40,6 +40,7 @@ import ( "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/path" "sigs.k8s.io/kwok/pkg/utils/sets" + "sigs.k8s.io/kwok/pkg/utils/slices" "sigs.k8s.io/kwok/pkg/utils/wait" "sigs.k8s.io/kwok/pkg/utils/yaml" ) @@ -134,26 +135,17 @@ func (c *Cluster) Available(ctx context.Context) error { func (c *Cluster) pullAllImages(ctx context.Context, env *env) error { conf := &env.kwokctlConfig.Options - images := []string{ - conf.EtcdImage, - conf.KubeApiserverImage, - conf.KwokControllerImage, - } - if !conf.DisableKubeControllerManager { - images = append(images, conf.KubeControllerManagerImage) - } - if !conf.DisableKubeScheduler { - images = append(images, conf.KubeSchedulerImage) - } - if conf.DashboardPort != 0 { - images = append(images, conf.DashboardImage) - } - if conf.PrometheusPort != 0 { - images = append(images, conf.PrometheusImage) - } - if conf.JaegerPort != 0 { - images = append(images, conf.JaegerImage) - } + + images := slices.FilterAndMap(env.kwokctlConfig.Components, func(component internalversion.Component) (string, bool) { + if component.Image == "" { + return "", false + } + return component.Image, true + }) + + images = slices.Unique(images) + images = slices.Sort(images) + err := c.PullImages(ctx, c.runtime, images, conf.QuietPull) if err != nil { return err @@ -339,11 +331,6 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - err = c.pullAllImages(ctx, env) - if err != nil { - return err - } - err = c.addEtcd(ctx, env) if err != nil { return err @@ -379,7 +366,7 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - err = c.addDashboard(ctx, env) + err = c.pullAllImages(ctx, env) if err != nil { return err } @@ -680,34 +667,6 @@ func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { return nil } -func (c *Cluster) addDashboard(_ context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - if conf.DashboardPort != 0 { - dashboardComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentDashboard) - dashboardComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(dashboardComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for dashboard component: %w", err) - } - dashboardComponent, err := components.BuildDashboardComponent(components.BuildDashboardComponentConfig{ - Workdir: env.workdir, - Image: conf.DashboardImage, - BindAddress: net.PublicAddress, - KubeconfigPath: env.inClusterOnHostKubeconfigPath, - CaCertPath: env.caCertPath, - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - Port: conf.DashboardPort, - Banner: fmt.Sprintf("Welcome to %s", c.Name()), - }) - if err != nil { - return err - } - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, dashboardComponent) - } - return nil -} - func (c *Cluster) addJaeger(ctx context.Context, env *env) error { conf := &env.kwokctlConfig.Options diff --git a/pkg/kwokctl/runtime/compose/component.go b/pkg/kwokctl/runtime/compose/component.go new file mode 100644 index 000000000..746c9273c --- /dev/null +++ b/pkg/kwokctl/runtime/compose/component.go @@ -0,0 +1,162 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package compose + +import ( + "context" + "encoding/json" + "fmt" + + "sigs.k8s.io/kwok/kustomize/kwokctl/component" + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/config" + "sigs.k8s.io/kwok/pkg/consts" + "sigs.k8s.io/kwok/pkg/kwokctl/runtime" + "sigs.k8s.io/kwok/pkg/kwokctl/scale" + "sigs.k8s.io/kwok/pkg/utils/gotpl" + "sigs.k8s.io/kwok/pkg/utils/path" + "sigs.k8s.io/kwok/pkg/utils/slices" +) + +type envKubeconfig struct { + inClusterOnHostKubeconfigPath string + inClusterKubeconfig string + pkiPath string + caCertPath string + adminKeyPath string + adminCertPath string + inClusterPkiPath string + inClusterCaCertPath string + inClusterAdminKeyPath string + inClusterAdminCertPath string +} + +func (c *Cluster) envKubeconfig(ctx context.Context) envKubeconfig { + inClusterOnHostKubeconfigPath := c.GetWorkdirPath(runtime.InClusterKubeconfigName) + inClusterKubeconfig := "/root/.kube/config" + pkiPath := c.GetWorkdirPath(runtime.PkiName) + caCertPath := path.Join(pkiPath, "ca.crt") + adminKeyPath := path.Join(pkiPath, "admin.key") + adminCertPath := path.Join(pkiPath, "admin.crt") + inClusterPkiPath := "/etc/kubernetes/pki/" + inClusterCaCertPath := path.Join(inClusterPkiPath, "ca.crt") + inClusterAdminKeyPath := path.Join(inClusterPkiPath, "admin.key") + inClusterAdminCertPath := path.Join(inClusterPkiPath, "admin.crt") + + return envKubeconfig{ + inClusterOnHostKubeconfigPath: inClusterOnHostKubeconfigPath, + inClusterKubeconfig: inClusterKubeconfig, + pkiPath: pkiPath, + caCertPath: caCertPath, + adminKeyPath: adminKeyPath, + adminCertPath: adminCertPath, + inClusterPkiPath: inClusterPkiPath, + inClusterCaCertPath: inClusterCaCertPath, + inClusterAdminKeyPath: inClusterAdminKeyPath, + inClusterAdminCertPath: inClusterAdminCertPath, + } +} + +// SetComponents returns the components of cluster +func (c *Cluster) SetComponents(ctx context.Context, name string, args ...string) error { + conf, err := c.Config(ctx) + if err != nil { + return err + } + _, ok := slices.Find(conf.Components, func(component internalversion.Component) bool { + return component.Name == name + }) + if ok { + return fmt.Errorf("component %s is already exists", name) + } + + env := c.envKubeconfig(ctx) + + var useKubeconfig bool + + kcp := config.FilterWithTypeFromContext[*internalversion.KwokctlComponent](ctx) + renderer := gotpl.NewRenderer(gotpl.FuncMap{ + "ClusterName": c.Name, + "Kubeconfig": func() string { + useKubeconfig = true + return env.inClusterKubeconfig + }, + }) + + krc, ok := slices.Find(kcp, func(krc *internalversion.KwokctlComponent) bool { + return krc.Name == name + }) + if !ok { + var componentData string + switch name { + default: + return fmt.Errorf("component %s is not exists", name) + case consts.ComponentDashboard: + componentData = component.DefaultDashboard + } + krc, err = config.UnmarshalWithType[*internalversion.KwokctlComponent](componentData) + if err != nil { + return err + } + } + + param, err := scale.NewParameters(ctx, krc.Parameters, args) + if err != nil { + return err + } + + componentData, err := renderer.ToJSON(krc.Template, param) + if err != nil { + return err + } + var component internalversion.Component + err = json.Unmarshal(componentData, &component) + if err != nil { + return err + } + component.Name = name + + if useKubeconfig { + volumes := []internalversion.Volume{ + { + HostPath: env.inClusterOnHostKubeconfigPath, + MountPath: env.inClusterKubeconfig, + ReadOnly: true, + }, + { + HostPath: env.caCertPath, + MountPath: env.inClusterCaCertPath, + ReadOnly: true, + }, + { + HostPath: env.adminCertPath, + MountPath: env.inClusterAdminCertPath, + ReadOnly: true, + }, + { + HostPath: env.adminKeyPath, + MountPath: env.inClusterAdminKeyPath, + ReadOnly: true, + }, + } + component.Volumes = append(component.Volumes, volumes...) + } + + conf.Components = append(conf.Components, component) + + return c.SetConfig(ctx, conf) +} diff --git a/pkg/kwokctl/runtime/config.go b/pkg/kwokctl/runtime/config.go index 3c3b72d0d..812593e93 100644 --- a/pkg/kwokctl/runtime/config.go +++ b/pkg/kwokctl/runtime/config.go @@ -65,6 +65,9 @@ type Runtime interface { // GetComponent return the component if it exists GetComponent(ctx context.Context, name string) (internalversion.Component, error) + // SetComponents returns the components of cluster + SetComponents(ctx context.Context, name string, args ...string) error + // Ready check the cluster is ready Ready(ctx context.Context) (bool, error) diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 9405eb387..e06b1d618 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -206,11 +206,6 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - err = c.addDashboard(ctx, env) - if err != nil { - return err - } - err = c.addPrometheus(ctx, env) if err != nil { return err @@ -337,12 +332,17 @@ func (c *Cluster) addKind(ctx context.Context, env *env) (err error) { len(kubeControllerManagerComponentPatches.ExtraEnvs) > 0 { logger.Warn("extraEnvs config in etcd, kube-apiserver, kube-scheduler or kube-controller-manager is not supported in kind") } + + ports := []internalversion.Port{} + for _, component := range env.kwokctlConfig.Components { + ports = append(ports, component.Ports...) + } + kindYaml, err := BuildKind(BuildKindConfig{ BindAddress: conf.BindAddress, KubeApiserverPort: conf.KubeApiserverPort, EtcdPort: conf.EtcdPort, JaegerPort: conf.JaegerPort, - DashboardPort: conf.DashboardPort, PrometheusPort: conf.PrometheusPort, KwokControllerPort: conf.KwokControllerPort, FeatureGates: featureGates, @@ -365,6 +365,7 @@ func (c *Cluster) addKind(ctx context.Context, env *env) (err error) { PrometheusExtraVolumes: prometheusPatches.ExtraVolumes, DisableQPSLimits: conf.DisableQPSLimits, KubeVersion: kubeVersion, + Ports: ports, }) if err != nil { return err @@ -445,44 +446,6 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { return nil } -func (c *Cluster) addDashboard(ctx context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - if conf.DashboardPort != 0 { - dashboardVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.DashboardImage, "") - if err != nil { - return err - } - - dashboardComponent, err := components.BuildDashboardComponent(components.BuildDashboardComponentConfig{ - Workdir: env.workdir, - Image: conf.DashboardImage, - Version: dashboardVersion, - BindAddress: net.PublicAddress, - KubeconfigPath: env.inClusterOnHostKubeconfigPath, - CaCertPath: env.caCertPath, - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - Port: 8000, - Banner: fmt.Sprintf("Welcome to %s", c.Name()), - }) - if err != nil { - return fmt.Errorf("failed to build dashboard component: %w", err) - } - - dashboardPod, err := yaml.Marshal(components.ConvertToPod(dashboardComponent)) - if err != nil { - return fmt.Errorf("failed to marshal dashboard pod: %w", err) - } - err = c.WriteFile(path.Join(c.GetWorkdirPath(runtime.ManifestsName), consts.ComponentDashboard+".yaml"), dashboardPod) - if err != nil { - return fmt.Errorf("failed to write: %w", err) - } - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, dashboardComponent) - } - return nil -} - func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { conf := &env.kwokctlConfig.Options @@ -741,6 +704,38 @@ func (c *Cluster) listAllImages(ctx context.Context) ([]string, error) { return images, nil } +func (c *Cluster) loadAllImages(ctx context.Context) error { + kindPath, err := c.preDownloadKind(ctx) + if err != nil { + return err + } + + config, err := c.Config(ctx) + if err != nil { + return err + } + + images := slices.FilterAndMap(config.Components, func(component internalversion.Component) (string, bool) { + if component.Image == "" { + return "", false + } + return component.Image, true + }) + + images = slices.Unique(images) + images = slices.Sort(images) + + if c.runtime == consts.RuntimeTypeDocker { + err = c.loadDockerImages(ctx, kindPath, c.Name(), images) + } else { + err = c.loadArchiveImages(ctx, kindPath, c.Name(), images, c.runtime, config.Options.CacheDir) + } + if err != nil { + return err + } + return nil +} + // loadDockerImages loads docker images into the cluster. // `kind load docker-image` func (c *Cluster) loadDockerImages(ctx context.Context, command string, kindCluster string, images []string) error { diff --git a/pkg/kwokctl/runtime/kind/component.go b/pkg/kwokctl/runtime/kind/component.go new file mode 100644 index 000000000..bf9bc7c02 --- /dev/null +++ b/pkg/kwokctl/runtime/kind/component.go @@ -0,0 +1,188 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kind + +import ( + "context" + "encoding/json" + "fmt" + + "sigs.k8s.io/kwok/kustomize/kwokctl/component" + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/config" + "sigs.k8s.io/kwok/pkg/consts" + "sigs.k8s.io/kwok/pkg/kwokctl/components" + "sigs.k8s.io/kwok/pkg/kwokctl/runtime" + "sigs.k8s.io/kwok/pkg/kwokctl/scale" + "sigs.k8s.io/kwok/pkg/utils/gotpl" + "sigs.k8s.io/kwok/pkg/utils/path" + "sigs.k8s.io/kwok/pkg/utils/slices" + "sigs.k8s.io/kwok/pkg/utils/yaml" +) + +type envKubeconfig struct { + inClusterOnHostKubeconfigPath string + inClusterKubeconfig string + pkiPath string + caCertPath string + adminKeyPath string + adminCertPath string + inClusterPkiPath string + inClusterCaCertPath string + inClusterAdminKeyPath string + inClusterAdminCertPath string + + manifestsPath string +} + +func (c *Cluster) envKubeconfig(ctx context.Context) envKubeconfig { + inClusterOnHostKubeconfigPath := "/etc/kubernetes/admin.conf" + inClusterKubeconfig := "/root/.kube/config" + + pkiPath := "/etc/kubernetes/pki" + + caCertPath := path.Join(pkiPath, "ca.crt") + adminKeyPath := path.Join(pkiPath, "admin.key") + adminCertPath := path.Join(pkiPath, "admin.crt") + + inClusterPkiPath := pkiPath + inClusterCaCertPath := caCertPath + inClusterAdminKeyPath := adminKeyPath + inClusterAdminCertPath := adminCertPath + + manifestsPath := c.GetWorkdirPath(runtime.ManifestsName) + + return envKubeconfig{ + inClusterOnHostKubeconfigPath: inClusterOnHostKubeconfigPath, + inClusterKubeconfig: inClusterKubeconfig, + pkiPath: pkiPath, + caCertPath: caCertPath, + adminKeyPath: adminKeyPath, + adminCertPath: adminCertPath, + inClusterPkiPath: inClusterPkiPath, + inClusterCaCertPath: inClusterCaCertPath, + inClusterAdminKeyPath: inClusterAdminKeyPath, + inClusterAdminCertPath: inClusterAdminCertPath, + + manifestsPath: manifestsPath, + } +} + +// SetComponents returns the components of cluster +func (c *Cluster) SetComponents(ctx context.Context, name string, args ...string) error { + conf, err := c.Config(ctx) + if err != nil { + return err + } + _, ok := slices.Find(conf.Components, func(component internalversion.Component) bool { + return component.Name == name + }) + if ok { + return fmt.Errorf("component %s is already exists", name) + } + + env := c.envKubeconfig(ctx) + + var useKubeconfig bool + + kcp := config.FilterWithTypeFromContext[*internalversion.KwokctlComponent](ctx) + renderer := gotpl.NewRenderer(gotpl.FuncMap{ + "ClusterName": c.Name, + "Kubeconfig": func() string { + useKubeconfig = true + return env.inClusterKubeconfig + }, + }) + + krc, ok := slices.Find(kcp, func(krc *internalversion.KwokctlComponent) bool { + return krc.Name == name + }) + if !ok { + var componentData string + switch name { + default: + return fmt.Errorf("component %s is not exists", name) + case consts.ComponentDashboard: + componentData = component.DefaultDashboard + } + krc, err = config.UnmarshalWithType[*internalversion.KwokctlComponent](componentData) + if err != nil { + return err + } + } + + param, err := scale.NewParameters(ctx, krc.Parameters, args) + if err != nil { + return err + } + + componentData, err := renderer.ToJSON(krc.Template, param) + if err != nil { + return err + } + var component internalversion.Component + err = json.Unmarshal(componentData, &component) + if err != nil { + return err + } + component.Name = name + + if useKubeconfig { + volumes := []internalversion.Volume{ + { + HostPath: env.inClusterOnHostKubeconfigPath, + MountPath: env.inClusterKubeconfig, + ReadOnly: true, + }, + { + HostPath: env.caCertPath, + MountPath: env.inClusterCaCertPath, + ReadOnly: true, + }, + { + HostPath: env.adminCertPath, + MountPath: env.inClusterAdminCertPath, + ReadOnly: true, + }, + { + HostPath: env.adminKeyPath, + MountPath: env.inClusterAdminKeyPath, + ReadOnly: true, + }, + } + component.Volumes = append(component.Volumes, volumes...) + } + + conf.Components = append(conf.Components, component) + + componentPod, err := yaml.Marshal(components.ConvertToPod(component)) + if err != nil { + return fmt.Errorf("failed to marshal pod: %w", err) + } + + err = c.MkdirAll(env.manifestsPath) + if err != nil { + return err + } + podFile := name + ".yaml" + err = c.WriteFile(path.Join(env.manifestsPath, podFile), componentPod) + if err != nil { + return fmt.Errorf("failed to write %s: %w", podFile, err) + } + + return c.SetConfig(ctx, conf) +} diff --git a/pkg/kwokctl/runtime/kind/kind.go b/pkg/kwokctl/runtime/kind/kind.go index edb083f2f..c2b9ca7c3 100644 --- a/pkg/kwokctl/runtime/kind/kind.go +++ b/pkg/kwokctl/runtime/kind/kind.go @@ -250,7 +250,6 @@ func expandHostVolumePaths(conf BuildKindConfig) (BuildKindConfig, error) { type BuildKindConfig struct { KubeApiserverPort uint32 EtcdPort uint32 - DashboardPort uint32 PrometheusPort uint32 JaegerPort uint32 KwokControllerPort uint32 @@ -281,4 +280,6 @@ type BuildKindConfig struct { BindAddress string DisableQPSLimits bool KubeVersion version.Version + + Ports []internalversion.Port } diff --git a/pkg/kwokctl/runtime/kind/kind.yaml.tpl b/pkg/kwokctl/runtime/kind/kind.yaml.tpl index eff962f40..4225d61b1 100644 --- a/pkg/kwokctl/runtime/kind/kind.yaml.tpl +++ b/pkg/kwokctl/runtime/kind/kind.yaml.tpl @@ -8,13 +8,8 @@ networking: nodes: - role: control-plane - {{ if or .DashboardPort .PrometheusPort .KwokControllerPort .EtcdPort .JaegerPort}} + {{ if or .PrometheusPort .KwokControllerPort .EtcdPort .JaegerPort .Ports }} extraPortMappings: - {{ if .DashboardPort }} - - containerPort: 8000 - hostPort: {{ .DashboardPort }} - protocol: TCP - {{ end }} {{ if .PrometheusPort }} - containerPort: 9090 hostPort: {{ .PrometheusPort }} @@ -35,6 +30,11 @@ nodes: hostPort: {{ .EtcdPort }} protocol: TCP {{ end }} + {{ range .Ports }} + - containerPort: {{ .Port }} + hostPort: {{ .HostPort }} + protocol: {{ .Protocol }} + {{ end }} {{ end }} kubeadmConfigPatches: diff --git a/pkg/utils/slices/slices.go b/pkg/utils/slices/slices.go index d69f36502..6edcb914a 100644 --- a/pkg/utils/slices/slices.go +++ b/pkg/utils/slices/slices.go @@ -16,6 +16,11 @@ limitations under the License. package slices +import ( + "cmp" + "sort" +) + // Map returns a new slice containing the results of applying the given function func Map[S ~[]T, T any, O any](s S, f func(T) O) []O { out := make([]O, len(s)) @@ -119,3 +124,13 @@ func Reverse[S ~[]T, T any](s S) []T { } return out } + +// Sort returns a new slice containing the elements of the slice in sorted order. +func Sort[S ~[]T, T cmp.Ordered](s S) []T { + out := make([]T, len(s)) + copy(out, s) + sort.Slice(out, func(i, j int) bool { + return cmp.Less(out[i], out[j]) + }) + return out +} diff --git a/test/kwokctl/testdata/docker/create_cluster.txt b/test/kwokctl/testdata/docker/create_cluster.txt index 08b98cb9c..5021e34aa 100644 --- a/test/kwokctl/testdata/docker/create_cluster.txt +++ b/test/kwokctl/testdata/docker/create_cluster.txt @@ -3,9 +3,9 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd +docker pull localhost/kwok:test docker pull registry.k8s.io/etcd:3.5.9-0 docker pull registry.k8s.io/kube-apiserver:v1.28.0 -docker pull localhost/kwok:test docker pull registry.k8s.io/kube-controller-manager:v1.28.0 docker pull registry.k8s.io/kube-scheduler:v1.28.0 cat </workdir/clusters//kubeconfig.yaml diff --git a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt index 3b5342dea..5635bea6e 100644 --- a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt @@ -3,12 +3,6 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd -docker pull registry.k8s.io/etcd:3.5.9-0 -docker pull registry.k8s.io/kube-apiserver:v1.28.0 -docker pull localhost/kwok:test -docker pull registry.k8s.io/kube-controller-manager:v1.28.0 -docker pull registry.k8s.io/kube-scheduler:v1.28.0 -docker pull docker.io/prom/prometheus:v2.44.0 cat </workdir/clusters//prometheus.yaml global: scrape_interval: 15s @@ -95,6 +89,12 @@ scrape_configs: - "kwok--kube-scheduler:10259" EOF chmod 0644 /workdir/clusters//prometheus.yaml +docker pull docker.io/prom/prometheus:v2.44.0 +docker pull localhost/kwok:test +docker pull registry.k8s.io/etcd:3.5.9-0 +docker pull registry.k8s.io/kube-apiserver:v1.28.0 +docker pull registry.k8s.io/kube-controller-manager:v1.28.0 +docker pull registry.k8s.io/kube-scheduler:v1.28.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: diff --git a/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt index 23391014d..bea5bd400 100644 --- a/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt @@ -6,14 +6,6 @@ mkdir -p /workdir/clusters//logs touch /workdir/clusters//logs/audit.log cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml mkdir -p /workdir/clusters//etcd -docker pull registry.k8s.io/etcd:3.5.9-0 -docker pull registry.k8s.io/kube-apiserver:v1.28.0 -docker pull localhost/kwok:test -docker pull registry.k8s.io/kube-controller-manager:v1.28.0 -docker pull registry.k8s.io/kube-scheduler:v1.28.0 -docker pull docker.io/kubernetesui/dashboard:v2.7.0 -docker pull docker.io/prom/prometheus:v2.44.0 -docker pull docker.io/jaegertracing/all-in-one:1.45.0 cat </workdir/clusters//apiserver-tracing-config.yaml apiVersion: apiserver.config.k8s.io/v1alpha1 kind: TracingConfiguration @@ -111,6 +103,14 @@ scrape_configs: - "kwok--kube-scheduler:10259" EOF chmod 0644 /workdir/clusters//prometheus.yaml +docker pull docker.io/jaegertracing/all-in-one:1.45.0 +docker pull docker.io/kubernetesui/dashboard:v2.7.0 +docker pull docker.io/prom/prometheus:v2.44.0 +docker pull localhost/kwok:test +docker pull registry.k8s.io/etcd:3.5.9-0 +docker pull registry.k8s.io/kube-apiserver:v1.28.0 +docker pull registry.k8s.io/kube-controller-manager:v1.28.0 +docker pull registry.k8s.io/kube-scheduler:v1.28.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: @@ -158,17 +158,17 @@ docker network create kwok- --label=com.docker.compose.project=kwo docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug docker create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.45.0 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --log-level=debug docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --link=kwok--jaeger --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 +docker create --name=kwok--dashboard --pull=never --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --insecure-port=8000 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.28.0 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 --v=DEBUG -docker create --name=kwok--dashboard --pull=never --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config --insecure-port=8000 docker create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --link=kwok--etcd --link=kwok--kube-apiserver --link=kwok--kube-controller-manager --link=kwok--kube-scheduler --link=kwok--kwok-controller --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug # Add context kwok- to ~/.kube/config docker start kwok--etcd docker start kwok--jaeger docker start kwok--kube-apiserver +docker start kwok--dashboard docker start kwok--kube-controller-manager docker start kwok--kube-scheduler docker start kwok--kwok-controller -docker start kwok--dashboard docker start kwok--prometheus diff --git a/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt index cd86793c9..30443716d 100644 --- a/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt @@ -1,3 +1,66 @@ +mkdir -p /workdir/clusters//manifests +cat </workdir/clusters//manifests/dashboard.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: dashboard + namespace: kube-system +spec: + containers: + - args: + - --insecure-bind-address=0.0.0.0 + - --insecure-port=8000 + - --bind-address=127.0.0.1 + - --port=0 + - --enable-insecure-login + - --enable-skip-login + - --disable-settings-authorizer + - --metrics-provider=none + - --system-banner=Welcome to kwok- + - --kubeconfig=~/.kube/config + image: docker.io/kubernetesui/dashboard:v2.7.0 + imagePullPolicy: Never + name: dashboard + ports: + - containerPort: 8000 + hostPort: 8000 + name: http + protocol: TCP + resources: {} + volumeMounts: + - mountPath: ~/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 +status: {} +EOF # Save cluster config to /workdir/clusters//kwok.yaml mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki @@ -115,15 +178,15 @@ networking: nodes: - role: control-plane extraPortMappings: - - containerPort: 8000 - hostPort: 8000 - protocol: TCP - containerPort: 9090 hostPort: 9090 protocol: TCP - containerPort: 16686 hostPort: 16686 protocol: TCP + - containerPort: 8000 + hostPort: 8000 + protocol: TCP kubeadmConfigPatches: - | kind: ClusterConfiguration @@ -275,68 +338,6 @@ spec: name: volume-4 status: {} EOF -cat </workdir/clusters//manifests/dashboard.yaml -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: dashboard - namespace: kube-system -spec: - containers: - - args: - - --insecure-bind-address=0.0.0.0 - - --bind-address=127.0.0.1 - - --port=0 - - --enable-insecure-login - - --enable-skip-login - - --disable-settings-authorizer - - --metrics-provider=none - - --system-banner=Welcome to kwok- - - --kubeconfig=~/.kube/config - - --insecure-port=8000 - image: docker.io/kubernetesui/dashboard:v2.7.0 - imagePullPolicy: Never - name: dashboard - ports: - - containerPort: 8000 - hostPort: 8000 - name: http - protocol: TCP - resources: {} - volumeMounts: - - mountPath: ~/.kube/config - name: volume-0 - readOnly: true - - mountPath: /etc/kubernetes/pki/ca.crt - name: volume-1 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-2 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-3 - readOnly: true - hostNetwork: true - restartPolicy: Always - securityContext: - runAsGroup: 0 - runAsUser: 0 - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - name: volume-0 - - hostPath: - path: /etc/kubernetes/pki/ca.crt - name: volume-1 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-2 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-3 -status: {} -EOF cat </workdir/clusters//manifests/prometheus.yaml apiVersion: v1 kind: Pod @@ -427,7 +428,6 @@ status: {} EOF podman pull docker.io/kindest/node:v1.28.0 podman pull localhost/kwok:test -podman pull docker.io/kubernetesui/dashboard:v2.7.0 podman pull docker.io/prom/prometheus:v2.44.0 podman pull docker.io/jaegertracing/all-in-one:1.45.0 # Save cluster config to /workdir/clusters//kwok.yaml @@ -436,10 +436,6 @@ mkdir -p /workdir/cache/image-archive/localhost/kwok podman save localhost/kwok:test -o /workdir/cache/image-archive/localhost/kwok/test.tar KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/localhost/kwok/test.tar --name kwok- rm /workdir/cache/image-archive/localhost/kwok/test.tar -mkdir -p /workdir/cache/image-archive/docker.io/kubernetesui/dashboard -podman save docker.io/kubernetesui/dashboard:v2.7.0 -o /workdir/cache/image-archive/docker.io/kubernetesui/dashboard/v2.7.0.tar -KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/kubernetesui/dashboard/v2.7.0.tar --name kwok- -rm /workdir/cache/image-archive/docker.io/kubernetesui/dashboard/v2.7.0.tar mkdir -p /workdir/cache/image-archive/docker.io/prom/prometheus podman save docker.io/prom/prometheus:v2.44.0 -o /workdir/cache/image-archive/docker.io/prom/prometheus/v2.44.0.tar KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/prom/prometheus/v2.44.0.tar --name kwok- diff --git a/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt index 98fb31d32..ac1f2e256 100644 --- a/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt @@ -1,3 +1,66 @@ +mkdir -p /workdir/clusters//manifests +cat </workdir/clusters//manifests/dashboard.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: dashboard + namespace: kube-system +spec: + containers: + - args: + - --insecure-bind-address=0.0.0.0 + - --insecure-port=8000 + - --bind-address=127.0.0.1 + - --port=0 + - --enable-insecure-login + - --enable-skip-login + - --disable-settings-authorizer + - --metrics-provider=none + - --system-banner=Welcome to kwok- + - --kubeconfig=~/.kube/config + image: docker.io/kubernetesui/dashboard:v2.7.0 + imagePullPolicy: Never + name: dashboard + ports: + - containerPort: 8000 + hostPort: 8000 + name: http + protocol: TCP + resources: {} + volumeMounts: + - mountPath: ~/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 +status: {} +EOF # Save cluster config to /workdir/clusters//kwok.yaml mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki @@ -115,15 +178,15 @@ networking: nodes: - role: control-plane extraPortMappings: - - containerPort: 8000 - hostPort: 8000 - protocol: TCP - containerPort: 9090 hostPort: 9090 protocol: TCP - containerPort: 16686 hostPort: 16686 protocol: TCP + - containerPort: 8000 + hostPort: 8000 + protocol: TCP kubeadmConfigPatches: - | kind: ClusterConfiguration @@ -275,68 +338,6 @@ spec: name: volume-4 status: {} EOF -cat </workdir/clusters//manifests/dashboard.yaml -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - name: dashboard - namespace: kube-system -spec: - containers: - - args: - - --insecure-bind-address=0.0.0.0 - - --bind-address=127.0.0.1 - - --port=0 - - --enable-insecure-login - - --enable-skip-login - - --disable-settings-authorizer - - --metrics-provider=none - - --system-banner=Welcome to kwok- - - --kubeconfig=~/.kube/config - - --insecure-port=8000 - image: docker.io/kubernetesui/dashboard:v2.7.0 - imagePullPolicy: Never - name: dashboard - ports: - - containerPort: 8000 - hostPort: 8000 - name: http - protocol: TCP - resources: {} - volumeMounts: - - mountPath: ~/.kube/config - name: volume-0 - readOnly: true - - mountPath: /etc/kubernetes/pki/ca.crt - name: volume-1 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-2 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-3 - readOnly: true - hostNetwork: true - restartPolicy: Always - securityContext: - runAsGroup: 0 - runAsUser: 0 - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - name: volume-0 - - hostPath: - path: /etc/kubernetes/pki/ca.crt - name: volume-1 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-2 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-3 -status: {} -EOF cat </workdir/clusters//manifests/prometheus.yaml apiVersion: v1 kind: Pod @@ -427,13 +428,11 @@ status: {} EOF docker pull docker.io/kindest/node:v1.28.0 docker pull localhost/kwok:test -docker pull docker.io/kubernetesui/dashboard:v2.7.0 docker pull docker.io/prom/prometheus:v2.44.0 docker pull docker.io/jaegertracing/all-in-one:1.45.0 # Save cluster config to /workdir/clusters//kwok.yaml KIND_EXPERIMENTAL_PROVIDER=docker kind create cluster --config /workdir/clusters//kind.yaml --name kwok- --image docker.io/kindest/node:v1.28.0 --wait 29m KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image localhost/kwok:test --name kwok- -KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/kubernetesui/dashboard:v2.7.0 --name kwok- KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/prom/prometheus:v2.44.0 --name kwok- KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/jaegertracing/all-in-one:1.45.0 --name kwok- kubectl config view --minify=true --raw=true diff --git a/test/kwokctl/testdata/nerdctl/create_cluster.txt b/test/kwokctl/testdata/nerdctl/create_cluster.txt index cacd5beb4..af02180b1 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster.txt @@ -3,9 +3,9 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd +nerdctl pull localhost/kwok:test nerdctl pull registry.k8s.io/etcd:3.5.9-0 nerdctl pull registry.k8s.io/kube-apiserver:v1.28.0 -nerdctl pull localhost/kwok:test nerdctl pull registry.k8s.io/kube-controller-manager:v1.28.0 nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 cat </workdir/clusters//kubeconfig.yaml diff --git a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt index 76f94cf56..78f29f1a2 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt @@ -3,12 +3,6 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd -nerdctl pull registry.k8s.io/etcd:3.5.9-0 -nerdctl pull registry.k8s.io/kube-apiserver:v1.28.0 -nerdctl pull localhost/kwok:test -nerdctl pull registry.k8s.io/kube-controller-manager:v1.28.0 -nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 -nerdctl pull docker.io/prom/prometheus:v2.44.0 cat </workdir/clusters//prometheus.yaml global: scrape_interval: 15s @@ -95,6 +89,12 @@ scrape_configs: - "kwok--kube-scheduler:10259" EOF chmod 0644 /workdir/clusters//prometheus.yaml +nerdctl pull docker.io/prom/prometheus:v2.44.0 +nerdctl pull localhost/kwok:test +nerdctl pull registry.k8s.io/etcd:3.5.9-0 +nerdctl pull registry.k8s.io/kube-apiserver:v1.28.0 +nerdctl pull registry.k8s.io/kube-controller-manager:v1.28.0 +nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: diff --git a/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt index 837920ff8..061923c97 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt @@ -6,14 +6,6 @@ mkdir -p /workdir/clusters//logs touch /workdir/clusters//logs/audit.log cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml mkdir -p /workdir/clusters//etcd -nerdctl pull registry.k8s.io/etcd:3.5.9-0 -nerdctl pull registry.k8s.io/kube-apiserver:v1.28.0 -nerdctl pull localhost/kwok:test -nerdctl pull registry.k8s.io/kube-controller-manager:v1.28.0 -nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 -nerdctl pull docker.io/kubernetesui/dashboard:v2.7.0 -nerdctl pull docker.io/prom/prometheus:v2.44.0 -nerdctl pull docker.io/jaegertracing/all-in-one:1.45.0 cat </workdir/clusters//apiserver-tracing-config.yaml apiVersion: apiserver.config.k8s.io/v1alpha1 kind: TracingConfiguration @@ -111,6 +103,14 @@ scrape_configs: - "kwok--kube-scheduler:10259" EOF chmod 0644 /workdir/clusters//prometheus.yaml +nerdctl pull docker.io/jaegertracing/all-in-one:1.45.0 +nerdctl pull docker.io/kubernetesui/dashboard:v2.7.0 +nerdctl pull docker.io/prom/prometheus:v2.44.0 +nerdctl pull localhost/kwok:test +nerdctl pull registry.k8s.io/etcd:3.5.9-0 +nerdctl pull registry.k8s.io/kube-apiserver:v1.28.0 +nerdctl pull registry.k8s.io/kube-controller-manager:v1.28.0 +nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: @@ -158,17 +158,17 @@ nerdctl network create kwok- --label=com.docker.compose.project=kw nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug nerdctl create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.45.0 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --log-level=debug nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 +nerdctl create --name=kwok--dashboard --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --insecure-port=8000 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.28.0 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 --v=DEBUG -nerdctl create --name=kwok--dashboard --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config --insecure-port=8000 nerdctl create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug # Add context kwok- to ~/.kube/config nerdctl start kwok--etcd nerdctl start kwok--jaeger nerdctl start kwok--kube-apiserver +nerdctl start kwok--dashboard nerdctl start kwok--kube-controller-manager nerdctl start kwok--kube-scheduler nerdctl start kwok--kwok-controller -nerdctl start kwok--dashboard nerdctl start kwok--prometheus diff --git a/test/kwokctl/testdata/podman/create_cluster.txt b/test/kwokctl/testdata/podman/create_cluster.txt index 24705c293..1c2eaf3e8 100644 --- a/test/kwokctl/testdata/podman/create_cluster.txt +++ b/test/kwokctl/testdata/podman/create_cluster.txt @@ -3,9 +3,9 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd +podman pull localhost/kwok:test podman pull registry.k8s.io/etcd:3.5.9-0 podman pull registry.k8s.io/kube-apiserver:v1.28.0 -podman pull localhost/kwok:test podman pull registry.k8s.io/kube-controller-manager:v1.28.0 podman pull registry.k8s.io/kube-scheduler:v1.28.0 cat </workdir/clusters//kubeconfig.yaml diff --git a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt index 23ddd4c3d..836cbc75e 100644 --- a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt @@ -3,12 +3,6 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd -podman pull registry.k8s.io/etcd:3.5.9-0 -podman pull registry.k8s.io/kube-apiserver:v1.28.0 -podman pull localhost/kwok:test -podman pull registry.k8s.io/kube-controller-manager:v1.28.0 -podman pull registry.k8s.io/kube-scheduler:v1.28.0 -podman pull docker.io/prom/prometheus:v2.44.0 cat </workdir/clusters//prometheus.yaml global: scrape_interval: 15s @@ -95,6 +89,12 @@ scrape_configs: - "kwok--kube-scheduler:10259" EOF chmod 0644 /workdir/clusters//prometheus.yaml +podman pull docker.io/prom/prometheus:v2.44.0 +podman pull localhost/kwok:test +podman pull registry.k8s.io/etcd:3.5.9-0 +podman pull registry.k8s.io/kube-apiserver:v1.28.0 +podman pull registry.k8s.io/kube-controller-manager:v1.28.0 +podman pull registry.k8s.io/kube-scheduler:v1.28.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: diff --git a/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt index 4c48f92a8..9c2e36aaf 100644 --- a/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt @@ -6,14 +6,6 @@ mkdir -p /workdir/clusters//logs touch /workdir/clusters//logs/audit.log cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml mkdir -p /workdir/clusters//etcd -podman pull registry.k8s.io/etcd:3.5.9-0 -podman pull registry.k8s.io/kube-apiserver:v1.28.0 -podman pull localhost/kwok:test -podman pull registry.k8s.io/kube-controller-manager:v1.28.0 -podman pull registry.k8s.io/kube-scheduler:v1.28.0 -podman pull docker.io/kubernetesui/dashboard:v2.7.0 -podman pull docker.io/prom/prometheus:v2.44.0 -podman pull docker.io/jaegertracing/all-in-one:1.45.0 cat </workdir/clusters//apiserver-tracing-config.yaml apiVersion: apiserver.config.k8s.io/v1alpha1 kind: TracingConfiguration @@ -111,6 +103,14 @@ scrape_configs: - "kwok--kube-scheduler:10259" EOF chmod 0644 /workdir/clusters//prometheus.yaml +podman pull docker.io/jaegertracing/all-in-one:1.45.0 +podman pull docker.io/kubernetesui/dashboard:v2.7.0 +podman pull docker.io/prom/prometheus:v2.44.0 +podman pull localhost/kwok:test +podman pull registry.k8s.io/etcd:3.5.9-0 +podman pull registry.k8s.io/kube-apiserver:v1.28.0 +podman pull registry.k8s.io/kube-controller-manager:v1.28.0 +podman pull registry.k8s.io/kube-scheduler:v1.28.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: @@ -158,17 +158,17 @@ podman network create kwok- --label=io.podman.compose.project=kwok podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug podman create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.45.0 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --log-level=debug podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --requires=kwok--jaeger --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 +podman create --name=kwok--dashboard --pull=never --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --insecure-port=8000 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.28.0 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 --v=DEBUG -podman create --name=kwok--dashboard --pull=never --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config --insecure-port=8000 podman create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --requires=kwok--etcd --requires=kwok--kube-apiserver --requires=kwok--kube-controller-manager --requires=kwok--kube-scheduler --requires=kwok--kwok-controller --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug # Add context kwok- to ~/.kube/config podman start kwok--etcd podman start kwok--jaeger podman start kwok--kube-apiserver +podman start kwok--dashboard podman start kwok--kube-controller-manager podman start kwok--kube-scheduler podman start kwok--kwok-controller -podman start kwok--dashboard podman start kwok--prometheus