From abc2c3609ddb7121b529a8c646ce93693a5f8c43 Mon Sep 17 00:00:00 2001 From: Shiming Zhang Date: Fri, 24 Nov 2023 22:15:56 +0800 Subject: [PATCH] Add kwokctl component api --- kustomize/kwokctl/component/dashboard.yaml | 55 ++ kustomize/kwokctl/component/embed.go | 44 ++ .../component/kube-controller-manager.yaml | 65 +++ .../kwokctl/component/kube-scheduler.yaml | 57 ++ .../kwokctl/component/kustomization.yaml | 8 + .../kwokctl/component/kwok-controller.yaml | 87 +++ kustomize/kwokctl/component/prometheus.yaml | 109 ++++ kustomize/kwokctl/kustomization.yaml | 1 + .../v1alpha1/kwokctl_component_types.go | 43 ++ .../v1alpha1/kwokctl_configuration_types.go | 20 + .../config/v1alpha1/zz_generated.deepcopy.go | 52 ++ pkg/apis/internalversion/conversion.go | 22 + .../kwokctl_component_types.go | 34 ++ .../kwokctl_configuration_types.go | 18 + .../zz_generated.conversion.go | 75 +++ .../internalversion/zz_generated.deepcopy.go | 43 ++ pkg/config/config.go | 6 + pkg/kwokctl/cmd/create/cluster/cluster.go | 66 ++- pkg/kwokctl/components/dashboard.go | 117 ---- pkg/kwokctl/components/etcd.go | 2 +- pkg/kwokctl/components/jaeger.go | 2 +- pkg/kwokctl/components/kube_apiserver.go | 10 +- .../components/kube_controller_manager.go | 8 +- pkg/kwokctl/components/kube_scheduler.go | 6 +- pkg/kwokctl/components/kwok_controller.go | 193 ------- pkg/kwokctl/components/prometheus.go | 152 ----- pkg/kwokctl/components/prometheus.yaml.tpl | 47 -- pkg/kwokctl/components/utils.go | 47 +- pkg/kwokctl/dryrun/dryrun.go | 2 +- pkg/kwokctl/runtime/binary/cluster.go | 348 ++---------- pkg/kwokctl/runtime/binary/component.go | 127 +++++ pkg/kwokctl/runtime/binary/file.go | 64 +++ pkg/kwokctl/runtime/cluster.go | 1 - pkg/kwokctl/runtime/component.go | 31 + pkg/kwokctl/runtime/compose/cluster.go | 294 +--------- pkg/kwokctl/runtime/compose/component.go | 146 +++++ pkg/kwokctl/runtime/compose/file.go | 74 +++ pkg/kwokctl/runtime/config.go | 3 + pkg/kwokctl/runtime/kind/cluster.go | 315 ++-------- pkg/kwokctl/runtime/kind/component.go | 125 ++++ pkg/kwokctl/runtime/kind/file.go | 148 +++++ pkg/kwokctl/runtime/kind/kind.go | 7 +- pkg/kwokctl/runtime/kind/kind.yaml.tpl | 51 +- pkg/utils/gotpl/funcs.go | 15 + pkg/utils/gotpl/renderer.go | 6 +- pkg/utils/slices/slices.go | 15 + site/content/en/docs/generated/apis.md | 167 ++++++ .../docs/generated/kwokctl_create_cluster.md | 11 +- .../testdata/binary/create_cluster.txt | 20 +- .../binary/create_cluster_with_extra.txt | 66 +-- .../binary/create_cluster_with_verbosity.txt | 80 ++- .../testdata/docker/create_cluster.txt | 10 +- .../docker/create_cluster_with_extra.txt | 57 +- .../docker/create_cluster_with_verbosity.txt | 68 +-- .../testdata/kind-podman/create_cluster.txt | 108 ++-- .../kind-podman/create_cluster_with_extra.txt | 442 ++++++-------- .../create_cluster_with_verbosity.txt | 537 +++++++++--------- test/kwokctl/testdata/kind/create_cluster.txt | 108 ++-- .../kind/create_cluster_with_extra.txt | 436 ++++++-------- .../kind/create_cluster_with_verbosity.txt | 525 ++++++++--------- .../testdata/nerdctl/create_cluster.txt | 10 +- .../nerdctl/create_cluster_with_extra.txt | 57 +- .../nerdctl/create_cluster_with_verbosity.txt | 68 +-- .../testdata/podman/create_cluster.txt | 10 +- .../podman/create_cluster_with_extra.txt | 57 +- .../podman/create_cluster_with_verbosity.txt | 68 +-- 66 files changed, 3117 insertions(+), 2949 deletions(-) create mode 100644 kustomize/kwokctl/component/dashboard.yaml create mode 100644 kustomize/kwokctl/component/embed.go create mode 100644 kustomize/kwokctl/component/kube-controller-manager.yaml create mode 100644 kustomize/kwokctl/component/kube-scheduler.yaml create mode 100644 kustomize/kwokctl/component/kustomization.yaml create mode 100644 kustomize/kwokctl/component/kwok-controller.yaml create mode 100644 kustomize/kwokctl/component/prometheus.yaml create mode 100644 pkg/apis/config/v1alpha1/kwokctl_component_types.go create mode 100644 pkg/apis/internalversion/kwokctl_component_types.go delete mode 100644 pkg/kwokctl/components/dashboard.go delete mode 100644 pkg/kwokctl/components/kwok_controller.go delete mode 100644 pkg/kwokctl/components/prometheus.go delete mode 100644 pkg/kwokctl/components/prometheus.yaml.tpl create mode 100644 pkg/kwokctl/runtime/binary/component.go create mode 100644 pkg/kwokctl/runtime/binary/file.go create mode 100644 pkg/kwokctl/runtime/component.go create mode 100644 pkg/kwokctl/runtime/compose/component.go create mode 100644 pkg/kwokctl/runtime/compose/file.go create mode 100644 pkg/kwokctl/runtime/kind/component.go create mode 100644 pkg/kwokctl/runtime/kind/file.go diff --git a/kustomize/kwokctl/component/dashboard.yaml b/kustomize/kwokctl/component/dashboard.yaml new file mode 100644 index 0000000000..b3994412ca --- /dev/null +++ b/kustomize/kwokctl/component/dashboard.yaml @@ -0,0 +1,55 @@ +apiVersion: config.kwok.x-k8s.io/v1alpha1 +kind: KwokctlComponent +metadata: + name: dashboard +parameters: + image: "" + binary: "" + version: "2.7.0" + bindAddress: "0.0.0.0" + port: 8000 +template: |- + {{ $version := + ( or + ( env "KWOK_DASHBOARD_VERSION" ) + .version + ) + }} + {{ $image := + ( or + .image + ( env "KWOK_DASHBOARD_IMAGE" ) + ( join + "" + ( list + ( or + ( env "KWOK_DASHBOARD_IMAGE_PREFIX" ) + "docker.io/kubernetesui" + ) + "/dashboard:v" + $version + ) + ) + ) + }} + + image: {{ $image }} + binary: {{ .binary }} + links: + - kube-apiserver + ports: + - name: http + port: {{ .port }} + hostPort: {{ .port }} + protocol: TCP + args: + - --insecure-bind-address={{ .bindAddress }} + - --insecure-port={{ .port }} + - --bind-address=127.0.0.1 + - --port=0 + - --enable-insecure-login + - --enable-skip-login + - --disable-settings-authorizer + - --metrics-provider=none + - --system-banner=Welcome to {{ ClusterName }} + - --kubeconfig={{ Kubeconfig }} diff --git a/kustomize/kwokctl/component/embed.go b/kustomize/kwokctl/component/embed.go new file mode 100644 index 0000000000..b75a927e74 --- /dev/null +++ b/kustomize/kwokctl/component/embed.go @@ -0,0 +1,44 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package component contains default component for kwokctl. +package component + +import ( + _ "embed" +) + +var ( + // DefaultDashboard is the default dashboard component. + //go:embed dashboard.yaml + DefaultDashboard string + + // DefaultPrometheus is the default prometheus component. + //go:embed prometheus.yaml + DefaultPrometheus string + + // DefaultKwokController is the default kwok controller component. + //go:embed kwok-controller.yaml + DefaultKwokController string + + // DefaultKubeControllerManager is the default kube-controller-manager component. + //go:embed kube-controller-manager.yaml + DefaultKubeControllerManager string + + // DefaultKubeScheduler is the default kube-scheduler component. + //go:embed kube-scheduler.yaml + DefaultKubeScheduler string +) diff --git a/kustomize/kwokctl/component/kube-controller-manager.yaml b/kustomize/kwokctl/component/kube-controller-manager.yaml new file mode 100644 index 0000000000..c43ab22064 --- /dev/null +++ b/kustomize/kwokctl/component/kube-controller-manager.yaml @@ -0,0 +1,65 @@ +apiVersion: config.kwok.x-k8s.io/v1alpha1 +kind: KwokctlComponent +metadata: + name: kube-controller-manager +parameters: + image: "" + binary: "" + bindAddress: "0.0.0.0" + port: 10257 + expose: false +template: |- + {{ $image := + ( or + .image + Config.Options.KubeControllerManagerImage + ) + }} + {{ $binary := + ( or + .binary + Config.Options.KubeControllerManagerBinary + ) + }} + + image: {{ $image }} + binary: {{ $binary }} + links: + - kube-apiserver + workdir: {{ Workdir }} + {{ if .expose }} + ports: + - name: http + port: {{ .port }} + hostPort: {{ .port }} + protocol: TCP + {{ end }} + command: + - kube-controller-manager + args: + {{ with Config.Options.KubeFeatureGates }} + - --feature-gates={{ . }} + {{ end }} + {{ with Config.Options.KubeControllerManagerNodeMonitorPeriodMilliseconds }} + - --node-monitor-period={{ . }}ms + {{ end }} + {{ with Config.Options.KubeControllerManagerNodeMonitorGracePeriodMilliseconds }} + - --node-monitor-grace-period={{ . }}ms + {{ end }} + - --kubeconfig={{ Kubeconfig }} + - --bind-address={{ .bindAddress }} + - --secure-port={{ .port }} + - --root-ca-file={{ PkiDir }}/ca.crt + - --service-account-private-key-file={{ PkiDir }}/admin.key + {{ with Config.Options.DisableQPSLimits }} + - --kube-api-qps=5000 + - --kube-api-burst=1000 + {{ end }} + - --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics + metric: + scheme: https + host: {{ Address }}:{{ .port }} + path: /metrics + certPath: {{ PkiDir }}/admin.crt + keyPath: {{ PkiDir }}/admin.key + insecureSkipVerify: true diff --git a/kustomize/kwokctl/component/kube-scheduler.yaml b/kustomize/kwokctl/component/kube-scheduler.yaml new file mode 100644 index 0000000000..fa0abc5415 --- /dev/null +++ b/kustomize/kwokctl/component/kube-scheduler.yaml @@ -0,0 +1,57 @@ +apiVersion: config.kwok.x-k8s.io/v1alpha1 +kind: KwokctlComponent +metadata: + name: kube-scheduler +parameters: + image: "" + binary: "" + bindAddress: "0.0.0.0" + port: 10259 + expose: false +template: |- + {{ $image := + ( or + .image + Config.Options.KubeSchedulerImage + ) + }} + {{ $binary := + ( or + .binary + Config.Options.KubeSchedulerBinary + ) + }} + + image: {{ $image }} + binary: {{ $binary }} + links: + - kube-apiserver + workdir: {{ Workdir }} + {{ if .expose }} + ports: + - name: http + port: {{ .port }} + hostPort: {{ .port }} + protocol: TCP + {{ end }} + command: + - kube-scheduler + args: + {{ with Config.Options.KubeFeatureGates }} + - --feature-gates=. + {{ end }} + - --kubeconfig={{ Kubeconfig }} + - --bind-address={{ .bindAddress }} + - --secure-port={{ .port }} + {{ with Config.Options.DisableQPSLimits }} + - --kube-api-qps=5000 + - --kube-api-burst=1000 + {{ end }} + - --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics + metric: + scheme: https + host: {{ Address }}:{{ .port }} + path: /metrics + certPath: {{ PkiDir }}/admin.crt + keyPath: {{ PkiDir }}/admin.key + insecureSkipVerify: true diff --git a/kustomize/kwokctl/component/kustomization.yaml b/kustomize/kwokctl/component/kustomization.yaml new file mode 100644 index 0000000000..2c5b1c9d2c --- /dev/null +++ b/kustomize/kwokctl/component/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- dashboard.yaml +- prometheus.yaml +- kwok-controller.yaml +- kube-controller-manager.yaml +- kube-scheduler.yaml diff --git a/kustomize/kwokctl/component/kwok-controller.yaml b/kustomize/kwokctl/component/kwok-controller.yaml new file mode 100644 index 0000000000..12e5215ba4 --- /dev/null +++ b/kustomize/kwokctl/component/kwok-controller.yaml @@ -0,0 +1,87 @@ +apiVersion: config.kwok.x-k8s.io/v1alpha1 +kind: KwokctlComponent +metadata: + name: kwok-controller +parameters: + image: "" + binary: "" + version: "0.4.0" + bindAddress: "0.0.0.0" + port: 10247 + expose: false +template: |- + {{ $version := + ( or + ( env "KWOK_CONTROLLER_VERSION" ) + .version + ) + }} + {{ $image := + ( or + .image + ( env "KWOK_CONTROLLER_IMAGE" ) + ( printf + "%s/kwok:v%s" + ( or + ( env "KWOK_CONTROLLER_IMAGE_PREFIX" ) + "registry.k8s.io/kwok" + ) + $version + ) + ) + }} + {{ $binary := + ( or + .binary + ( env "KWOK_CONTROLLER_BINARY" ) + ( printf + "%s/kwok-%s-%s" + ( or + ( env "KWOK_CONTROLLER_BINARY_PREFIX" ) + ( printf + "https://github.com/kubernetes-sigs/kwok/releases/download/v%s/" + $version + ) + ) + ( goos ) + ( goarch ) + ) + ) + }} + + image: {{ $image }} + binary: {{ $binary }} + links: + - kube-apiserver + workdir: {{ Workdir }} + {{ if .expose }} + ports: + - name: http + port: {{ .port }} + hostPort: {{ .port }} + protocol: TCP + {{ end }} + args: + {{ if eq Mode "cluster" }} + - --manage-all-nodes=false + - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake + {{ else }} + - --manage-all-nodes=true + {{ end }} + - --kubeconfig={{ Kubeconfig }} + - --config=~/.kwok/kwok.yaml + - --tls-cert-file={{ PkiDir }}/admin.crt + - --tls-private-key-file={{ PkiDir }}/admin.key + {{ if eq ( Mode ) "cluster" }} + - --node-ip=$(POD_IP) + - --node-name=kwok-controller.kube-system.svc + {{ else }} + - --node-name={{ Address }} + {{ end }} + - --node-port={{ .port }} + - --server-address=0.0.0.0:{{ .port }} + - --node-lease-duration-seconds={{ Config.Options.NodeLeaseDurationSeconds }} + metric: + scheme: http + host: {{ Address }}:{{ .port }} + path: /metrics diff --git a/kustomize/kwokctl/component/prometheus.yaml b/kustomize/kwokctl/component/prometheus.yaml new file mode 100644 index 0000000000..7fee9d53f3 --- /dev/null +++ b/kustomize/kwokctl/component/prometheus.yaml @@ -0,0 +1,109 @@ +apiVersion: config.kwok.x-k8s.io/v1alpha1 +kind: KwokctlComponent +metadata: + name: prometheus +parameters: + image: "" + binary: "" + version: "2.44.0" + bindAddress: "0.0.0.0" + port: 9090 +template: |- + {{ $version := + ( or + ( env "KWOK_PROMETHEUS_VERSION" ) + .version + ) + }} + {{ $image := + ( or + .image + ( env "KWOK_PROMETHEUS_IMAGE" ) + ( printf + "%s/prometheus:v%s" + ( or + ( env "KWOK_PROMETHEUS_IMAGE_PREFIX" ) + "docker.io/prom" + ) + $version + ) + ) + }} + {{ $binary := + ( or + .binary + ( env "KWOK_PROMETHEUS_BINARY" ) + ( printf + "%s/prometheus-%s.%s-%s.tar.gz#prometheus" + ( or + ( env "KWOK_PROMETHEUS_BINARY_PREFIX" ) + ( printf + "https://github.com/prometheus/prometheus/releases/download/v%s/" + $version + ) + ) + $version + ( goos ) + ( goarch ) + ) + ) + }} + + image: {{ $image }} + binary: {{ $binary }} + links: + - kube-apiserver + workdir: {{ Workdir }} + ports: + - name: http + port: {{ .port }} + hostPort: {{ .port }} + protocol: TCP + args: + - --config.file={{ FilesDir }}/etc/prometheus/prometheus.yaml + - --web.listen-address={{ .bindAddress }}:{{ .port }} + metric: + scheme: http + host: 127.0.0.1:{{ .port }} + path: /metrics + files: + - path: /etc/prometheus/prometheus.yaml + template: |- + global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s + alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] + scrape_configs: + {{ "{{ range .Components }}" }} + {{ "{{ range .MetricsDiscoveries }}" }} + - job_name: {{ "{{ .Name | quote }}" }} + http_sd_configs: + - url: {{ "{{ .Scheme }}://{{ .Host }}{{ .Path }}" }} + {{ "{{ end }}" }} + {{ "{{ if .Metric }}" }} + - job_name: {{ "{{ or .Metric.Name .Name | quote }}" }} + scheme: {{ "{{ .Metric.Scheme }}" }} + honor_timestamps: true + metrics_path: {{ "{{ .Metric.Path }}" }} + follow_redirects: true + enable_http2: true + {{ "{{ if eq .Metric.Scheme \"https\" }}" }} + tls_config: + cert_file: {{ "{{ .Metric.CertPath | quote }}" }} + key_file: {{ "{{ .Metric.KeyPath | quote }}" }} + insecure_skip_verify: {{ "{{ .Metric.InsecureSkipVerify }}" }} + {{ "{{ end }}" }} + static_configs: + - targets: + - {{ "{{ .Metric.Host }}" }} + {{ "{{ end }}" }} + {{ "{{ end }}" }} diff --git a/kustomize/kwokctl/kustomization.yaml b/kustomize/kwokctl/kustomization.yaml index 2948fa1738..c10c9ac139 100644 --- a/kustomize/kwokctl/kustomization.yaml +++ b/kustomize/kwokctl/kustomization.yaml @@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./resource +- ./component diff --git a/pkg/apis/config/v1alpha1/kwokctl_component_types.go b/pkg/apis/config/v1alpha1/kwokctl_component_types.go new file mode 100644 index 0000000000..c73e2e4b2a --- /dev/null +++ b/pkg/apis/config/v1alpha1/kwokctl_component_types.go @@ -0,0 +1,43 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "encoding/json" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // KwokctlComponentKind is the kind of the kwokctl component. + KwokctlComponentKind = "KwokctlComponent" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// KwokctlComponent holds information about the kwokctl component. +type KwokctlComponent struct { + //+k8s:conversion-gen=false + metav1.TypeMeta `json:",inline"` + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta `json:"metadata,omitempty"` + // Parameters is the parameters for the kwokctl component configuration. + Parameters json.RawMessage `json:"parameters,omitempty"` + // Template is the template for the kwokctl component configuration. + Template string `json:"template,omitempty"` +} diff --git a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go index 1d48f6ce2a..d7d89e54e9 100644 --- a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go +++ b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go @@ -460,12 +460,20 @@ type Component struct { // +optional Volumes []Volume `json:"volumes,omitempty"` + // Files is a list of files that can be mounted by containers belonging to the component. + // +optional + Files []File `json:"files,omitempty"` + // Metric is the metric of the component. Metric *ComponentMetric `json:"metric,omitempty"` // MetricsDiscoveries is the metrics discovery of the component. MetricsDiscoveries []ComponentMetric `json:"metricsDiscoveries,omitempty"` + // Address is the address of the component. + // +optional + Address string `json:"address,omitempty"` + // Version is the version of the component. // +optional Version string `json:"version,omitempty"` @@ -548,6 +556,18 @@ type Volume struct { PathType HostPathType `json:"pathType,omitempty"` } +// File represents a file that is accessible to the containers running in a component. +type File struct { + // Path is the path of the file. + Path string `json:"path"` + // Data is the content of the file. + Data string `json:"data,omitempty"` + // Template is the template of the file. + Template string `json:"template,omitempty"` + // Mode is the mode of the file. + Mode string `json:"mode,omitempty"` +} + // HostPathType represents the type of storage used for HostPath volumes. // +enum type HostPathType string diff --git a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index 74a2f23339..9a19f98738 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -62,6 +62,11 @@ func (in *Component) DeepCopyInto(out *Component) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Files != nil { + in, out := &in.Files, &out.Files + *out = make([]File, len(*in)) + copy(*out, *in) + } if in.Metric != nil { in, out := &in.Metric, &out.Metric *out = new(ComponentMetric) @@ -166,6 +171,22 @@ func (in *ExtraArgs) DeepCopy() *ExtraArgs { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *File) DeepCopyInto(out *File) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new File. +func (in *File) DeepCopy() *File { + if in == nil { + return nil + } + out := new(File) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KwokConfiguration) DeepCopyInto(out *KwokConfiguration) { *out = *in @@ -239,6 +260,37 @@ func (in *KwokConfigurationOptions) DeepCopy() *KwokConfigurationOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KwokctlComponent) DeepCopyInto(out *KwokctlComponent) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(json.RawMessage, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KwokctlComponent. +func (in *KwokctlComponent) DeepCopy() *KwokctlComponent { + if in == nil { + return nil + } + out := new(KwokctlComponent) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KwokctlComponent) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KwokctlConfiguration) DeepCopyInto(out *KwokctlConfiguration) { *out = *in diff --git a/pkg/apis/internalversion/conversion.go b/pkg/apis/internalversion/conversion.go index c4b5db9977..739ca552e2 100644 --- a/pkg/apis/internalversion/conversion.go +++ b/pkg/apis/internalversion/conversion.go @@ -66,6 +66,28 @@ func ConvertToInternalKwokctlResource(in *configv1alpha1.KwokctlResource) (*Kwok return &out, nil } +// ConvertToV1alpha1KwokctlComponent converts an internal version KwokctlComponent to a v1alpha1.KwokctlComponent. +func ConvertToV1alpha1KwokctlComponent(in *KwokctlComponent) (*configv1alpha1.KwokctlComponent, error) { + var out configv1alpha1.KwokctlComponent + out.APIVersion = configv1alpha1.GroupVersion.String() + out.Kind = configv1alpha1.KwokctlComponentKind + err := Convert_internalversion_KwokctlComponent_To_v1alpha1_KwokctlComponent(in, &out, nil) + if err != nil { + return nil, err + } + return &out, nil +} + +// ConvertToInternalKwokctlComponent converts a v1alpha1.KwokctlComponent to an internal version. +func ConvertToInternalKwokctlComponent(in *configv1alpha1.KwokctlComponent) (*KwokctlComponent, error) { + var out KwokctlComponent + err := Convert_v1alpha1_KwokctlComponent_To_internalversion_KwokctlComponent(in, &out, nil) + if err != nil { + return nil, err + } + return &out, nil +} + // ConvertToV1alpha1KwokConfiguration converts an internal version KwokConfiguration to a v1alpha1.KwokConfiguration. func ConvertToV1alpha1KwokConfiguration(in *KwokConfiguration) (*configv1alpha1.KwokConfiguration, error) { var out configv1alpha1.KwokConfiguration diff --git a/pkg/apis/internalversion/kwokctl_component_types.go b/pkg/apis/internalversion/kwokctl_component_types.go new file mode 100644 index 0000000000..ee8a3b540e --- /dev/null +++ b/pkg/apis/internalversion/kwokctl_component_types.go @@ -0,0 +1,34 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internalversion + +import ( + "encoding/json" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// KwokctlComponent provides component definition for kwokctl. +type KwokctlComponent struct { + // Standard list metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + metav1.ObjectMeta + // Parameters is the parameters for the kwokctl component configuration. + Parameters json.RawMessage + // Template is the template for the kwokctl component configuration. + Template string +} diff --git a/pkg/apis/internalversion/kwokctl_configuration_types.go b/pkg/apis/internalversion/kwokctl_configuration_types.go index 4cf78a2eca..74cf5bebeb 100644 --- a/pkg/apis/internalversion/kwokctl_configuration_types.go +++ b/pkg/apis/internalversion/kwokctl_configuration_types.go @@ -304,12 +304,18 @@ type Component struct { // Volumes is a list of named volumes that can be mounted by containers belonging to the component. Volumes []Volume + // Files is a list of files that can be mounted by containers belonging to the component. + Files []File + // Metric is the metric of the component. Metric *ComponentMetric // MetricsDiscoveries is the metrics discovery of the component. MetricsDiscoveries []ComponentMetric + // Address is the address of the component. + Address string + // Version is the version of the component. Version string } @@ -382,6 +388,18 @@ type Volume struct { PathType HostPathType } +// File represents a file that is accessible to the containers running in a component. +type File struct { + // Path is the path of the file. + Path string + // Data is the content of the file. + Data string + // Template is the template of the file. + Template string + // Mode is the mode of the file. + Mode string +} + // HostPathType represents the type of storage used for HostPath volumes. type HostPathType string diff --git a/pkg/apis/internalversion/zz_generated.conversion.go b/pkg/apis/internalversion/zz_generated.conversion.go index b71eb0f361..45186725ee 100644 --- a/pkg/apis/internalversion/zz_generated.conversion.go +++ b/pkg/apis/internalversion/zz_generated.conversion.go @@ -259,6 +259,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*File)(nil), (*configv1alpha1.File)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_internalversion_File_To_v1alpha1_File(a.(*File), b.(*configv1alpha1.File), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*configv1alpha1.File)(nil), (*File)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_File_To_internalversion_File(a.(*configv1alpha1.File), b.(*File), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*FinalizerItem)(nil), (*v1alpha1.FinalizerItem)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_internalversion_FinalizerItem_To_v1alpha1_FinalizerItem(a.(*FinalizerItem), b.(*v1alpha1.FinalizerItem), scope) }); err != nil { @@ -309,6 +319,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*KwokctlComponent)(nil), (*configv1alpha1.KwokctlComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_internalversion_KwokctlComponent_To_v1alpha1_KwokctlComponent(a.(*KwokctlComponent), b.(*configv1alpha1.KwokctlComponent), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*configv1alpha1.KwokctlComponent)(nil), (*KwokctlComponent)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_KwokctlComponent_To_internalversion_KwokctlComponent(a.(*configv1alpha1.KwokctlComponent), b.(*KwokctlComponent), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*KwokctlConfiguration)(nil), (*configv1alpha1.KwokctlConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_internalversion_KwokctlConfiguration_To_v1alpha1_KwokctlConfiguration(a.(*KwokctlConfiguration), b.(*configv1alpha1.KwokctlConfiguration), scope) }); err != nil { @@ -938,8 +958,10 @@ func autoConvert_internalversion_Component_To_v1alpha1_Component(in *Component, } else { out.Volumes = nil } + out.Files = *(*[]configv1alpha1.File)(unsafe.Pointer(&in.Files)) out.Metric = (*configv1alpha1.ComponentMetric)(unsafe.Pointer(in.Metric)) out.MetricsDiscoveries = *(*[]configv1alpha1.ComponentMetric)(unsafe.Pointer(&in.MetricsDiscoveries)) + out.Address = in.Address out.Version = in.Version return nil } @@ -971,8 +993,10 @@ func autoConvert_v1alpha1_Component_To_internalversion_Component(in *configv1alp } else { out.Volumes = nil } + out.Files = *(*[]File)(unsafe.Pointer(&in.Files)) out.Metric = (*ComponentMetric)(unsafe.Pointer(in.Metric)) out.MetricsDiscoveries = *(*[]ComponentMetric)(unsafe.Pointer(&in.MetricsDiscoveries)) + out.Address = in.Address out.Version = in.Version return nil } @@ -1240,6 +1264,32 @@ func Convert_v1alpha1_ExtraArgs_To_internalversion_ExtraArgs(in *configv1alpha1. return autoConvert_v1alpha1_ExtraArgs_To_internalversion_ExtraArgs(in, out, s) } +func autoConvert_internalversion_File_To_v1alpha1_File(in *File, out *configv1alpha1.File, s conversion.Scope) error { + out.Path = in.Path + out.Data = in.Data + out.Template = in.Template + out.Mode = in.Mode + return nil +} + +// Convert_internalversion_File_To_v1alpha1_File is an autogenerated conversion function. +func Convert_internalversion_File_To_v1alpha1_File(in *File, out *configv1alpha1.File, s conversion.Scope) error { + return autoConvert_internalversion_File_To_v1alpha1_File(in, out, s) +} + +func autoConvert_v1alpha1_File_To_internalversion_File(in *configv1alpha1.File, out *File, s conversion.Scope) error { + out.Path = in.Path + out.Data = in.Data + out.Template = in.Template + out.Mode = in.Mode + return nil +} + +// Convert_v1alpha1_File_To_internalversion_File is an autogenerated conversion function. +func Convert_v1alpha1_File_To_internalversion_File(in *configv1alpha1.File, out *File, s conversion.Scope) error { + return autoConvert_v1alpha1_File_To_internalversion_File(in, out, s) +} + func autoConvert_internalversion_FinalizerItem_To_v1alpha1_FinalizerItem(in *FinalizerItem, out *v1alpha1.FinalizerItem, s conversion.Scope) error { out.Value = in.Value return nil @@ -1415,6 +1465,31 @@ func Convert_v1alpha1_KwokConfigurationOptions_To_internalversion_KwokConfigurat return autoConvert_v1alpha1_KwokConfigurationOptions_To_internalversion_KwokConfigurationOptions(in, out, s) } +func autoConvert_internalversion_KwokctlComponent_To_v1alpha1_KwokctlComponent(in *KwokctlComponent, out *configv1alpha1.KwokctlComponent, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + out.Parameters = *(*json.RawMessage)(unsafe.Pointer(&in.Parameters)) + out.Template = in.Template + return nil +} + +// Convert_internalversion_KwokctlComponent_To_v1alpha1_KwokctlComponent is an autogenerated conversion function. +func Convert_internalversion_KwokctlComponent_To_v1alpha1_KwokctlComponent(in *KwokctlComponent, out *configv1alpha1.KwokctlComponent, s conversion.Scope) error { + return autoConvert_internalversion_KwokctlComponent_To_v1alpha1_KwokctlComponent(in, out, s) +} + +func autoConvert_v1alpha1_KwokctlComponent_To_internalversion_KwokctlComponent(in *configv1alpha1.KwokctlComponent, out *KwokctlComponent, s conversion.Scope) error { + // INFO: in.TypeMeta opted out of conversion generation + out.ObjectMeta = in.ObjectMeta + out.Parameters = *(*json.RawMessage)(unsafe.Pointer(&in.Parameters)) + out.Template = in.Template + return nil +} + +// Convert_v1alpha1_KwokctlComponent_To_internalversion_KwokctlComponent is an autogenerated conversion function. +func Convert_v1alpha1_KwokctlComponent_To_internalversion_KwokctlComponent(in *configv1alpha1.KwokctlComponent, out *KwokctlComponent, s conversion.Scope) error { + return autoConvert_v1alpha1_KwokctlComponent_To_internalversion_KwokctlComponent(in, out, s) +} + func autoConvert_internalversion_KwokctlConfiguration_To_v1alpha1_KwokctlConfiguration(in *KwokctlConfiguration, out *configv1alpha1.KwokctlConfiguration, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_internalversion_KwokctlConfigurationOptions_To_v1alpha1_KwokctlConfigurationOptions(&in.Options, &out.Options, s); err != nil { diff --git a/pkg/apis/internalversion/zz_generated.deepcopy.go b/pkg/apis/internalversion/zz_generated.deepcopy.go index 50de2d47f0..a2a12dd30f 100644 --- a/pkg/apis/internalversion/zz_generated.deepcopy.go +++ b/pkg/apis/internalversion/zz_generated.deepcopy.go @@ -304,6 +304,11 @@ func (in *Component) DeepCopyInto(out *Component) { *out = make([]Volume, len(*in)) copy(*out, *in) } + if in.Files != nil { + in, out := &in.Files, &out.Files + *out = make([]File, len(*in)) + copy(*out, *in) + } if in.Metric != nil { in, out := &in.Metric, &out.Metric *out = new(ComponentMetric) @@ -531,6 +536,22 @@ func (in *ExtraArgs) DeepCopy() *ExtraArgs { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *File) DeepCopyInto(out *File) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new File. +func (in *File) DeepCopy() *File { + if in == nil { + return nil + } + out := new(File) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FinalizerItem) DeepCopyInto(out *FinalizerItem) { *out = *in @@ -633,6 +654,28 @@ func (in *KwokConfigurationOptions) DeepCopy() *KwokConfigurationOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KwokctlComponent) DeepCopyInto(out *KwokctlComponent) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(json.RawMessage, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KwokctlComponent. +func (in *KwokctlComponent) DeepCopy() *KwokctlComponent { + if in == nil { + return nil + } + out := new(KwokctlComponent) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KwokctlConfiguration) DeepCopyInto(out *KwokctlConfiguration) { *out = *in diff --git a/pkg/config/config.go b/pkg/config/config.go index 835a177b8e..e5c80fdbb3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -107,6 +107,12 @@ var configHandlers = map[string]configHandler{ MutateToInternal: mutateToInternalConfig(internalversion.ConvertToInternalKwokctlResource), MutateToVersiond: mutateToVersiondConfig(internalversion.ConvertToV1alpha1KwokctlResource), }, + configv1alpha1.KwokctlComponentKind: { + Unmarshal: unmarshalConfig[*configv1alpha1.KwokctlComponent], + Marshal: marshalConfig, + MutateToInternal: mutateToInternalConfig(internalversion.ConvertToInternalKwokctlComponent), + MutateToVersiond: mutateToVersiondConfig(internalversion.ConvertToV1alpha1KwokctlComponent), + }, v1alpha1.StageKind: { Unmarshal: unmarshalConfig[*v1alpha1.Stage], Marshal: marshalConfig, diff --git a/pkg/kwokctl/cmd/create/cluster/cluster.go b/pkg/kwokctl/cmd/create/cluster/cluster.go index b5e462e088..c3c0f745fb 100644 --- a/pkg/kwokctl/cmd/create/cluster/cluster.go +++ b/pkg/kwokctl/cmd/create/cluster/cluster.go @@ -21,6 +21,7 @@ import ( "context" "fmt" "os" + "strconv" "strings" "time" @@ -31,8 +32,10 @@ import ( "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/kwokctl/runtime" "sigs.k8s.io/kwok/pkg/log" + "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/kubeconfig" "sigs.k8s.io/kwok/pkg/utils/path" + "sigs.k8s.io/kwok/pkg/utils/slices" ) type flagpole struct { @@ -41,6 +44,9 @@ type flagpole struct { Wait time.Duration Kubeconfig string + Param []string + EnableComponents []string + *internalversion.KwokctlConfiguration } @@ -62,6 +68,7 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().Uint32Var(&flags.Options.KubeApiserverPort, "kube-apiserver-port", flags.Options.KubeApiserverPort, `Port of the apiserver (default random)`) cmd.Flags().Uint32Var(&flags.Options.PrometheusPort, "prometheus-port", flags.Options.PrometheusPort, `Port to expose Prometheus metrics`) + _ = cmd.Flags().MarkDeprecated("prometheus-port", "will be removed in a future release, use --enable-components=prometheus --param=.prometheus.port= instead") cmd.Flags().Uint32Var(&flags.Options.JaegerPort, "jaeger-port", flags.Options.JaegerPort, `Port to expose Jaeger UI`) cmd.Flags().BoolVar(&flags.Options.SecurePort, "secure-port", flags.Options.SecurePort, `The apiserver port on which to serve HTTPS with authentication and authorization, is not available before Kubernetes 1.13.0`) cmd.Flags().BoolVar(&flags.Options.QuietPull, "quiet-pull", flags.Options.QuietPull, `Pull without printing progress information`) @@ -89,6 +96,7 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().StringVar(&flags.Options.PrometheusImage, "prometheus-image", flags.Options.PrometheusImage, `Image of Prometheus, only for docker/podman/nerdctl/kind/kind-podman runtime '${KWOK_PROMETHEUS_IMAGE_PREFIX}/prometheus:${KWOK_PROMETHEUS_VERSION}' `) + _ = cmd.Flags().MarkDeprecated("prometheus-image", "will be removed in a future release, use --enable-components=prometheus --param=.prometheus.image= instead") cmd.Flags().StringVar(&flags.Options.JaegerImage, "jaeger-image", flags.Options.JaegerImage, `Image of Jaeger, only for docker/podman/nerdctl/kind/kind-podman runtime '${KWOK_JAEGER_IMAGE_PREFIX}/all-in-one:${KWOK_JAEGER_VERSION}' `) @@ -97,9 +105,11 @@ func NewCommand(ctx context.Context) *cobra.Command { '${KWOK_KIND_NODE_IMAGE_PREFIX}/node:${KWOK_KUBE_VERSION}' `) cmd.Flags().Uint32Var(&flags.Options.DashboardPort, "dashboard-port", flags.Options.DashboardPort, `Port of dashboard given to the host`) + _ = cmd.Flags().MarkDeprecated("dashboard-port", "will be removed in a future release, use --enable-components=dashboard --param=.dashboard.port= instead") cmd.Flags().StringVar(&flags.Options.DashboardImage, "dashboard-image", flags.Options.DashboardImage, `Image of dashboard, only for docker/podman/nerdctl/kind/kind-podman runtime '${KWOK_DASHBOARD_IMAGE_PREFIX}/dashboard:${KWOK_DASHBOARD_VERSION}' `) + _ = cmd.Flags().MarkDeprecated("dashboard-image", "will be removed in a future release, use --param=.dashboard.image= instead") cmd.Flags().StringVar(&flags.Options.KubeApiserverBinary, "kube-apiserver-binary", flags.Options.KubeApiserverBinary, `Binary of kube-apiserver, only for binary runtime `) cmd.Flags().StringVar(&flags.Options.KubeControllerManagerBinary, "kube-controller-manager-binary", flags.Options.KubeControllerManagerBinary, `Binary of kube-controller-manager, only for binary runtime @@ -113,9 +123,10 @@ func NewCommand(ctx context.Context) *cobra.Command { `) _ = cmd.Flags().MarkDeprecated("etcd-binary-tar", "--etcd-binary-tar will be removed in a future release, please use --etcd-binary instead") cmd.Flags().StringVar(&flags.Options.PrometheusBinary, "prometheus-binary", flags.Options.PrometheusBinary, `Binary of Prometheus, only for binary runtime`) + _ = cmd.Flags().MarkDeprecated("prometheus-binary", "--prometheus-binary will be removed in a future release, please use --enable-components=prometheus --param=.prometheus.binary= instead") cmd.Flags().StringVar(&flags.Options.PrometheusBinaryTar, "prometheus-binary-tar", flags.Options.PrometheusBinaryTar, `Tar of Prometheus, if --prometheus-binary is set, this is ignored, only for binary runtime `) - _ = cmd.Flags().MarkDeprecated("prometheus-binary-tar", "--prometheus-binary-tar will be removed in a future release, please use --prometheus-binary instead") + _ = cmd.Flags().MarkDeprecated("prometheus-binary-tar", "--prometheus-binary-tar will be removed in a future release, please use --enable-components=prometheus --param=.prometheus.binary= instead") cmd.Flags().StringVar(&flags.Options.JaegerBinary, "jaeger-binary", flags.Options.JaegerBinary, `Binary of Jaeger, only for binary runtime`) cmd.Flags().StringVar(&flags.Options.JaegerBinaryTar, "jaeger-binary-tar", flags.Options.JaegerBinaryTar, `Tar of Jaeger, if --jaeger-binary is set, this is ignored, only for binary runtime `) @@ -137,11 +148,37 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().StringVar(&flags.Kubeconfig, "kubeconfig", flags.Kubeconfig, "The path to the kubeconfig file will be added to the newly created cluster and set to current-context") cmd.Flags().BoolVar(&flags.Options.DisableQPSLimits, "disable-qps-limits", flags.Options.DisableQPSLimits, "Disable QPS limits for components") cmd.Flags().StringSliceVar(&flags.Options.EnableCRDs, "enable-crds", flags.Options.EnableCRDs, "List of CRDs to enable") + cmd.Flags().StringArrayVar(&flags.Param, "param", flags.Param, "Set parameters for the cluster, format: --param=.key1=value1 --param=.key2=value2") + cmd.Flags().StringArrayVar(&flags.EnableComponents, "enable-components", flags.EnableComponents, "Enable components for the cluster") return cmd } +func mutation(flags *flagpole) { + if flags.Options.DashboardPort != 0 { + flags.EnableComponents = append(flags.EnableComponents, "dashboard") + flags.Param = append(flags.Param, + ".dashboard.port="+format.String(flags.Options.DashboardPort), + ".dashboard.image="+strconv.Quote(flags.Options.DashboardImage), + ) + + flags.Options.DashboardPort = 0 + } + + if flags.Options.PrometheusPort != 0 { + flags.EnableComponents = append(flags.EnableComponents, "prometheus") + flags.Param = append(flags.Param, + ".prometheus.port="+format.String(flags.Options.PrometheusPort), + ".prometheus.image="+strconv.Quote(flags.Options.PrometheusImage), + ) + + flags.Options.PrometheusPort = 0 + } +} + func runE(ctx context.Context, flags *flagpole) error { + mutation(flags) + name := config.ClusterName(flags.Name) workdir := path.Join(config.ClustersDir, flags.Name) @@ -228,6 +265,33 @@ func runE(ctx context.Context, flags *flagpole) error { cleanUp() return err } + + baseComponents := []string{"kwok-controller"} + if !flags.Options.DisableKubeControllerManager { + baseComponents = append(baseComponents, "kube-controller-manager") + } + if !flags.Options.DisableKubeScheduler { + baseComponents = append(baseComponents, "kube-scheduler") + } + + baseComponents = append(baseComponents, flags.EnableComponents...) + // Enable components + for _, componentName := range baseComponents { + componentPrefix := fmt.Sprintf(".%s.", componentName) + params := slices.FilterAndMap(flags.Param, func(s string) (string, bool) { + if !strings.HasPrefix(s, componentPrefix) { + return "", false + } + return s[len(componentPrefix)-1:], true + }) + err := rt.SetComponents(ctx, componentName, params...) + if err != nil { + logger.Error("Failed to set components", err, "component", componentName) + cleanUp() + return err + } + } + err = rt.Save(ctx) if err != nil { logger.Error("Failed to save config", err) diff --git a/pkg/kwokctl/components/dashboard.go b/pkg/kwokctl/components/dashboard.go deleted file mode 100644 index cf99bb0bda..0000000000 --- a/pkg/kwokctl/components/dashboard.go +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package components - -import ( - "sigs.k8s.io/kwok/pkg/apis/internalversion" - "sigs.k8s.io/kwok/pkg/consts" - "sigs.k8s.io/kwok/pkg/utils/format" - "sigs.k8s.io/kwok/pkg/utils/version" -) - -// BuildDashboardComponentConfig is the configuration for building the dashboard component. -type BuildDashboardComponentConfig struct { - Runtime string - Binary string - Image string - Version version.Version - Workdir string - BindAddress string - Port uint32 - - Banner string - - CaCertPath string - AdminCertPath string - AdminKeyPath string - KubeconfigPath string -} - -// BuildDashboardComponent builds the dashboard component. -func BuildDashboardComponent(conf BuildDashboardComponentConfig) (component internalversion.Component, err error) { - dashboardArgs := []string{ - "--insecure-bind-address=" + conf.BindAddress, - "--bind-address=127.0.0.1", - "--port=0", - "--enable-insecure-login", - "--enable-skip-login", - "--disable-settings-authorizer", - "--metrics-provider=none", - } - if conf.Banner != "" { - dashboardArgs = append(dashboardArgs, "--system-banner="+conf.Banner) - } - - user := "" - var volumes []internalversion.Volume - var ports []internalversion.Port - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { - dashboardArgs = append(dashboardArgs, - "--kubeconfig=/root/.kube/config", - "--insecure-port=8000", - ) - volumes = append(volumes, - internalversion.Volume{ - HostPath: conf.KubeconfigPath, - MountPath: "/root/.kube/config", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: conf.CaCertPath, - MountPath: "/etc/kubernetes/pki/ca.crt", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: conf.AdminCertPath, - MountPath: "/etc/kubernetes/pki/admin.crt", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: conf.AdminKeyPath, - MountPath: "/etc/kubernetes/pki/admin.key", - ReadOnly: true, - }, - ) - ports = append(ports, - internalversion.Port{ - Name: "http", - Port: 8000, - HostPort: conf.Port, - Protocol: internalversion.ProtocolTCP, - }, - ) - } else { - dashboardArgs = append(dashboardArgs, - "--kubeconfig="+conf.KubeconfigPath, - "--insecure-port="+format.String(conf.Port), - ) - } - - component = internalversion.Component{ - Name: consts.ComponentDashboard, - Image: conf.Image, - Links: []string{ - consts.ComponentKubeApiserver, - }, - WorkDir: conf.Workdir, - Ports: ports, - Volumes: volumes, - Args: dashboardArgs, - User: user, - } - return component, nil -} diff --git a/pkg/kwokctl/components/etcd.go b/pkg/kwokctl/components/etcd.go index ea33a1716d..510c9a1c8c 100644 --- a/pkg/kwokctl/components/etcd.go +++ b/pkg/kwokctl/components/etcd.go @@ -71,7 +71,7 @@ func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversio var metric *internalversion.ComponentMetric - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { // TODO: use a volume for the data path // volumes = append(volumes, // internalversion.Volume{ diff --git a/pkg/kwokctl/components/jaeger.go b/pkg/kwokctl/components/jaeger.go index b49de9d178..3360849298 100644 --- a/pkg/kwokctl/components/jaeger.go +++ b/pkg/kwokctl/components/jaeger.go @@ -49,7 +49,7 @@ func BuildJaegerComponent(conf BuildJaegerComponentConfig) (component internalve volumes = append(volumes, conf.ExtraVolumes...) var ports []internalversion.Port - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { ports = []internalversion.Port{ { HostPort: conf.Port, diff --git a/pkg/kwokctl/components/kube_apiserver.go b/pkg/kwokctl/components/kube_apiserver.go index 55e99e47b5..fe5eb639e7 100644 --- a/pkg/kwokctl/components/kube_apiserver.go +++ b/pkg/kwokctl/components/kube_apiserver.go @@ -125,7 +125,7 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon volumes = append(volumes, conf.ExtraVolumes...) var metric *internalversion.ComponentMetric - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { kubeApiserverArgs = append(kubeApiserverArgs, "--etcd-servers=http://"+conf.EtcdAddress+":2379", ) @@ -142,7 +142,7 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon ) } - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { ports = []internalversion.Port{ { HostPort: conf.Port, @@ -205,7 +205,7 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon } } } else { - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { ports = []internalversion.Port{ { HostPort: conf.Port, @@ -236,7 +236,7 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon } if conf.AuditPolicyPath != "" { - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { volumes = append(volumes, internalversion.Volume{ HostPath: conf.AuditPolicyPath, @@ -262,7 +262,7 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon } if conf.TracingConfigPath != "" { - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { volumes = append(volumes, internalversion.Volume{ HostPath: conf.TracingConfigPath, diff --git a/pkg/kwokctl/components/kube_controller_manager.go b/pkg/kwokctl/components/kube_controller_manager.go index bd30327c51..dabee0a6c6 100644 --- a/pkg/kwokctl/components/kube_controller_manager.go +++ b/pkg/kwokctl/components/kube_controller_manager.go @@ -81,7 +81,7 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen var ports []internalversion.Port var metric *internalversion.ComponentMetric - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { volumes = append(volumes, internalversion.Volume{ HostPath: conf.KubeconfigPath, @@ -120,7 +120,7 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen ) } - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { kubeControllerManagerArgs = append(kubeControllerManagerArgs, "--bind-address="+conf.BindAddress, "--secure-port=10257", @@ -162,7 +162,7 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen // "--port=0", // ) } else { - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { kubeControllerManagerArgs = append(kubeControllerManagerArgs, "--address="+conf.BindAddress, "--port=10252", @@ -199,7 +199,7 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen } if conf.KubeAuthorization { - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { kubeControllerManagerArgs = append(kubeControllerManagerArgs, "--root-ca-file=/etc/kubernetes/pki/ca.crt", "--service-account-private-key-file=/etc/kubernetes/pki/admin.key", diff --git a/pkg/kwokctl/components/kube_scheduler.go b/pkg/kwokctl/components/kube_scheduler.go index d5dca96116..0abb1d96ed 100644 --- a/pkg/kwokctl/components/kube_scheduler.go +++ b/pkg/kwokctl/components/kube_scheduler.go @@ -65,7 +65,7 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon var ports []internalversion.Port var metric *internalversion.ComponentMetric - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { volumes = append(volumes, internalversion.Volume{ HostPath: conf.KubeconfigPath, @@ -124,7 +124,7 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon ) } - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { kubeSchedulerArgs = append(kubeSchedulerArgs, "--bind-address="+conf.BindAddress, "--secure-port=10259", @@ -165,7 +165,7 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon // "--port=0", // ) } else { - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { + if GetRuntimeMode(conf.Runtime) != RuntimeModeRaw { kubeSchedulerArgs = append(kubeSchedulerArgs, "--address="+conf.BindAddress, "--port=10251", diff --git a/pkg/kwokctl/components/kwok_controller.go b/pkg/kwokctl/components/kwok_controller.go deleted file mode 100644 index 8c888aa02b..0000000000 --- a/pkg/kwokctl/components/kwok_controller.go +++ /dev/null @@ -1,193 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package components - -import ( - "strings" - - "sigs.k8s.io/kwok/pkg/apis/internalversion" - "sigs.k8s.io/kwok/pkg/consts" - "sigs.k8s.io/kwok/pkg/log" - "sigs.k8s.io/kwok/pkg/utils/format" - "sigs.k8s.io/kwok/pkg/utils/net" - "sigs.k8s.io/kwok/pkg/utils/version" -) - -// BuildKwokControllerComponentConfig is the configuration for building a kwok controller component. -type BuildKwokControllerComponentConfig struct { - Runtime string - ProjectName string - Binary string - Image string - Version version.Version - Workdir string - BindAddress string - Port uint32 - ConfigPath string - KubeconfigPath string - CaCertPath string - AdminCertPath string - AdminKeyPath string - NodeIP string - NodeName string - ManageNodesWithAnnotationSelector string - Verbosity log.Level - NodeLeaseDurationSeconds uint - EnableCRDs []string - ExtraArgs []internalversion.ExtraArgs - ExtraVolumes []internalversion.Volume - ExtraEnvs []internalversion.Env -} - -// BuildKwokControllerComponent builds a kwok controller component. -func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (component internalversion.Component) { - kwokControllerArgs := []string{} - if conf.ManageNodesWithAnnotationSelector == "" { - kwokControllerArgs = append(kwokControllerArgs, - "--manage-all-nodes=true", - ) - } else { - kwokControllerArgs = append(kwokControllerArgs, - "--manage-all-nodes=false", - "--manage-nodes-with-annotation-selector="+conf.ManageNodesWithAnnotationSelector, - ) - } - - kwokControllerArgs = append(kwokControllerArgs, extraArgsToStrings(conf.ExtraArgs)...) - - var volumes []internalversion.Volume - volumes = append(volumes, conf.ExtraVolumes...) - var ports []internalversion.Port - - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { - volumes = append(volumes, - internalversion.Volume{ - HostPath: conf.KubeconfigPath, - MountPath: "/root/.kube/config", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: conf.CaCertPath, - MountPath: "/etc/kubernetes/pki/ca.crt", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: conf.AdminCertPath, - MountPath: "/etc/kubernetes/pki/admin.crt", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: conf.AdminKeyPath, - MountPath: "/etc/kubernetes/pki/admin.key", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: conf.ConfigPath, - MountPath: "/root/.kwok/kwok.yaml", - ReadOnly: true, - }, - ) - - if conf.Port != 0 { - ports = append(ports, - internalversion.Port{ - HostPort: conf.Port, - Port: 10247, - }, - ) - } - kwokControllerArgs = append(kwokControllerArgs, - "--kubeconfig=/root/.kube/config", - "--config=/root/.kwok/kwok.yaml", - "--tls-cert-file=/etc/kubernetes/pki/admin.crt", - "--tls-private-key-file=/etc/kubernetes/pki/admin.key", - "--node-ip="+conf.NodeIP, - "--node-name="+conf.NodeName, - "--node-port=10247", - "--server-address="+conf.BindAddress+":10247", - "--node-lease-duration-seconds="+format.String(conf.NodeLeaseDurationSeconds), - ) - } else { - kwokControllerArgs = append(kwokControllerArgs, - "--kubeconfig="+conf.KubeconfigPath, - "--config="+conf.ConfigPath, - "--tls-cert-file="+conf.AdminCertPath, - "--tls-private-key-file="+conf.AdminKeyPath, - "--node-ip="+conf.NodeIP, - "--node-name="+conf.NodeName, - "--node-port="+format.String(conf.Port), - "--server-address="+conf.BindAddress+":"+format.String(conf.Port), - "--node-lease-duration-seconds="+format.String(conf.NodeLeaseDurationSeconds), - ) - } - - var metricsHost string - switch getRuntimeKind(conf.Runtime) { - case runtimeKindRaw: - metricsHost = net.LocalAddress + ":" + format.String(conf.Port) - case runtimeKindContainer: - metricsHost = conf.ProjectName + "-" + consts.ComponentKwokController + ":10247" - case runtimeKindCluster: - metricsHost = net.LocalAddress + ":10247" - } - - var metric *internalversion.ComponentMetric - var metricsDiscoveries []internalversion.ComponentMetric - - if metricsHost != "" { - metric = &internalversion.ComponentMetric{ - Scheme: "http", - Host: metricsHost, - Path: "/metrics", - } - metricsDiscoveries = append(metricsDiscoveries, internalversion.ComponentMetric{ - Name: "kwok-service-discovery", - Scheme: "http", - Host: metricsHost, - Path: "/discovery/prometheus", - }) - } - - if conf.Verbosity != log.LevelInfo { - kwokControllerArgs = append(kwokControllerArgs, "--v="+format.String(conf.Verbosity)) - } - - if len(conf.EnableCRDs) != 0 { - kwokControllerArgs = append(kwokControllerArgs, "--enable-crds="+strings.Join(conf.EnableCRDs, ",")) - } - - envs := []internalversion.Env{} - envs = append(envs, conf.ExtraEnvs...) - - return internalversion.Component{ - Name: consts.ComponentKwokController, - Version: conf.Version.String(), - Links: []string{ - consts.ComponentKubeApiserver, - }, - Ports: ports, - Command: []string{"kwok"}, - Volumes: volumes, - Args: kwokControllerArgs, - Binary: conf.Binary, - Image: conf.Image, - Metric: metric, - MetricsDiscoveries: metricsDiscoveries, - WorkDir: conf.Workdir, - Envs: envs, - } -} diff --git a/pkg/kwokctl/components/prometheus.go b/pkg/kwokctl/components/prometheus.go deleted file mode 100644 index 90b8c7cfa9..0000000000 --- a/pkg/kwokctl/components/prometheus.go +++ /dev/null @@ -1,152 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package components - -import ( - "bytes" - "fmt" - "text/template" - - "github.com/Masterminds/sprig/v3" - - "sigs.k8s.io/kwok/pkg/apis/internalversion" - "sigs.k8s.io/kwok/pkg/consts" - "sigs.k8s.io/kwok/pkg/log" - "sigs.k8s.io/kwok/pkg/utils/format" - "sigs.k8s.io/kwok/pkg/utils/net" - "sigs.k8s.io/kwok/pkg/utils/version" - - _ "embed" -) - -// BuildPrometheusComponentConfig is the configuration for building a prometheus component. -type BuildPrometheusComponentConfig struct { - Runtime string - Binary string - Image string - Version version.Version - Workdir string - BindAddress string - Port uint32 - ConfigPath string - AdminCertPath string - AdminKeyPath string - Verbosity log.Level - ExtraArgs []internalversion.ExtraArgs - ExtraVolumes []internalversion.Volume - ExtraEnvs []internalversion.Env -} - -// BuildPrometheusComponent builds a prometheus component. -func BuildPrometheusComponent(conf BuildPrometheusComponentConfig) (component internalversion.Component, err error) { - prometheusArgs := []string{} - prometheusArgs = append(prometheusArgs, extraArgsToStrings(conf.ExtraArgs)...) - - var volumes []internalversion.Volume - volumes = append(volumes, conf.ExtraVolumes...) - var ports []internalversion.Port - var metric *internalversion.ComponentMetric - - if getRuntimeKind(conf.Runtime) != runtimeKindRaw { - volumes = append(volumes, - internalversion.Volume{ - HostPath: conf.ConfigPath, - MountPath: "/etc/prometheus/prometheus.yaml", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: conf.AdminCertPath, - MountPath: "/etc/kubernetes/pki/admin.crt", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: conf.AdminKeyPath, - MountPath: "/etc/kubernetes/pki/admin.key", - ReadOnly: true, - }, - ) - ports = []internalversion.Port{ - { - HostPort: conf.Port, - Port: 9090, - }, - } - prometheusArgs = append(prometheusArgs, - "--config.file=/etc/prometheus/prometheus.yaml", - "--web.listen-address="+conf.BindAddress+":9090", - ) - } else { - prometheusArgs = append(prometheusArgs, - "--config.file="+conf.ConfigPath, - "--web.listen-address="+conf.BindAddress+":"+format.String(conf.Port), - ) - } - - metric = &internalversion.ComponentMetric{ - Scheme: "http", - Host: net.LocalAddress + ":" + format.String(conf.Port), - Path: "/metrics", - } - - if conf.Verbosity != log.LevelInfo { - prometheusArgs = append(prometheusArgs, "--log.level="+log.ToLogSeverityLevel(conf.Verbosity)) - } - - envs := []internalversion.Env{} - envs = append(envs, conf.ExtraEnvs...) - - return internalversion.Component{ - Name: consts.ComponentPrometheus, - Version: conf.Version.String(), - Links: []string{ - consts.ComponentEtcd, - consts.ComponentKubeApiserver, - consts.ComponentKubeControllerManager, - consts.ComponentKubeScheduler, - consts.ComponentKwokController, - }, - Command: []string{consts.ComponentPrometheus}, - Ports: ports, - Volumes: volumes, - Args: prometheusArgs, - Binary: conf.Binary, - Image: conf.Image, - WorkDir: conf.Workdir, - Metric: metric, - Envs: envs, - }, nil -} - -//go:embed prometheus.yaml.tpl -var prometheusYamlTpl string - -var prometheusYamlTemplate = template.Must(template.New("prometheus_config").Funcs(sprig.TxtFuncMap()).Parse(prometheusYamlTpl)) - -// BuildPrometheus builds the prometheus yaml content. -func BuildPrometheus(conf BuildPrometheusConfig) (string, error) { - buf := bytes.NewBuffer(nil) - err := prometheusYamlTemplate.Execute(buf, conf) - if err != nil { - return "", fmt.Errorf("build prometheus error: %w", err) - } - return buf.String(), nil -} - -// BuildPrometheusConfig is the configuration for building the prometheus config -type BuildPrometheusConfig struct { - Components []internalversion.Component -} diff --git a/pkg/kwokctl/components/prometheus.yaml.tpl b/pkg/kwokctl/components/prometheus.yaml.tpl deleted file mode 100644 index 64efe82161..0000000000 --- a/pkg/kwokctl/components/prometheus.yaml.tpl +++ /dev/null @@ -1,47 +0,0 @@ -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - follow_redirects: true - enable_http2: true - scheme: http - timeout: 10s - api_version: v2 - static_configs: - - targets: [] -scrape_configs: -{{ range .Components }} -{{ range .MetricsDiscoveries }} -- job_name: {{ or .Name | quote }} - http_sd_configs: - - url: {{ .Scheme }}://{{ .Host }}{{ .Path }} -{{ end }} -{{ if .Metric }} -- job_name: {{ or .Metric.Name .Name | quote }} -{{ if eq .Metric.Scheme "https" }} - scheme: https - honor_timestamps: true - metrics_path: {{ .Metric.Path }} - follow_redirects: true - enable_http2: true - tls_config: - cert_file: {{ .Metric.CertPath | quote }} - key_file: {{ .Metric.KeyPath | quote }} - insecure_skip_verify: {{ .Metric.InsecureSkipVerify }} - static_configs: - - targets: - - {{ .Metric.Host }} -{{ else }} - scheme: http - honor_timestamps: true - metrics_path: {{ .Metric.Path }} - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - {{ .Metric.Host }} -{{ end }} -{{ end }} -{{ end }} diff --git a/pkg/kwokctl/components/utils.go b/pkg/kwokctl/components/utils.go index a0b08d6b9c..07cbe401cc 100644 --- a/pkg/kwokctl/components/utils.go +++ b/pkg/kwokctl/components/utils.go @@ -74,22 +74,49 @@ func extraArgsToStrings(args []internalversion.ExtraArgs) []string { } const ( - runtimeKindRaw = "raw" - runtimeKindContainer = "container" - runtimeKindCluster = "cluster" + RuntimeModeRaw = "raw" + RuntimeModeContainer = "container" + RuntimeModeCluster = "cluster" ) var ( runtimeTypeMap = map[string]string{ - consts.RuntimeTypeKind: runtimeKindCluster, - consts.RuntimeTypeKindPodman: runtimeKindCluster, - consts.RuntimeTypeDocker: runtimeKindContainer, - consts.RuntimeTypeNerdctl: runtimeKindContainer, - consts.RuntimeTypePodman: runtimeKindContainer, - consts.RuntimeTypeBinary: runtimeKindRaw, + consts.RuntimeTypeKind: RuntimeModeCluster, + consts.RuntimeTypeKindPodman: RuntimeModeCluster, + consts.RuntimeTypeDocker: RuntimeModeContainer, + consts.RuntimeTypeNerdctl: RuntimeModeContainer, + consts.RuntimeTypePodman: RuntimeModeContainer, + consts.RuntimeTypeBinary: RuntimeModeRaw, } ) -func getRuntimeKind(runtime string) string { +// GetRuntimeMode returns the mode of runtime. +func GetRuntimeMode(runtime string) string { return runtimeTypeMap[runtime] } + +// PatchComponent patches a component. +func PatchComponent(c internalversion.Component, patch internalversion.ComponentPatches) internalversion.Component { + for _, a := range patch.ExtraArgs { + c.Args = append(c.Args, fmt.Sprintf("--%s=%s", a.Key, a.Value)) + } + + for _, v := range patch.ExtraVolumes { + c.Volumes = append(c.Volumes, internalversion.Volume{ + Name: v.Name, + HostPath: v.HostPath, + PathType: v.PathType, + MountPath: v.MountPath, + ReadOnly: v.ReadOnly, + }) + } + + for _, e := range patch.ExtraEnvs { + c.Envs = append(c.Envs, internalversion.Env{ + Name: e.Name, + Value: e.Value, + }) + } + + return c +} diff --git a/pkg/kwokctl/dryrun/dryrun.go b/pkg/kwokctl/dryrun/dryrun.go index 92721285f3..44401b8c3e 100644 --- a/pkg/kwokctl/dryrun/dryrun.go +++ b/pkg/kwokctl/dryrun/dryrun.go @@ -71,7 +71,7 @@ func (d *dryRunWriter) Close() error { _, _ = fmt.Fprintf(d.w, "echo %s >%s\n", line, d.name) return nil } - _, _ = fmt.Fprintf(d.w, "cat <%s\n%s\nEOF\n", buf, d.name) + _, _ = fmt.Fprintf(d.w, "cat <%s\n%s\nEOF\n", d.name, buf) return nil } diff --git a/pkg/kwokctl/runtime/binary/cluster.go b/pkg/kwokctl/runtime/binary/cluster.go index 736b65f851..5b34a01bf1 100644 --- a/pkg/kwokctl/runtime/binary/cluster.go +++ b/pkg/kwokctl/runtime/binary/cluster.go @@ -66,60 +66,11 @@ func (c *Cluster) Available(ctx context.Context) error { func (c *Cluster) download(ctx context.Context, env *env) error { conf := &env.kwokctlConfig.Options - kubeApiserverPath := c.GetBinPath(consts.ComponentKubeApiserver + conf.BinSuffix) - err := c.DownloadWithCache(ctx, conf.CacheDir, conf.KubeApiserverBinary, kubeApiserverPath, 0750, conf.QuietPull) - if err != nil { - return err - } - - if !conf.DisableKubeControllerManager { - kubeControllerManagerPath := c.GetBinPath(consts.ComponentKubeControllerManager + conf.BinSuffix) - err = c.DownloadWithCache(ctx, conf.CacheDir, conf.KubeControllerManagerBinary, kubeControllerManagerPath, 0750, conf.QuietPull) - if err != nil { - return err - } - } - - if !conf.DisableKubeScheduler { - kubeSchedulerPath := c.GetBinPath(consts.ComponentKubeScheduler + conf.BinSuffix) - err = c.DownloadWithCache(ctx, conf.CacheDir, conf.KubeSchedulerBinary, kubeSchedulerPath, 0750, conf.QuietPull) - if err != nil { - return err - } - } - - // TODO: Add dashboard binary - // if conf.DashboardPort != 0 { - // kubeDashboardPath := c.GetBinPath(consts.ComponentDashboard + conf.BinSuffix) - // err = c.DownloadWithCache(ctx, conf.CacheDir, conf.DashboardBinary, kubeDashboardPath, 0750, conf.QuietPull) - // if err != nil { - // return err - // } - // } - - kwokControllerPath := c.GetBinPath(consts.ComponentKwokController + conf.BinSuffix) - err = c.DownloadWithCache(ctx, conf.CacheDir, conf.KwokControllerBinary, kwokControllerPath, 0750, conf.QuietPull) - if err != nil { - return err - } - - etcdPath := c.GetBinPath(consts.ComponentEtcd + conf.BinSuffix) - err = c.DownloadWithCache(ctx, conf.CacheDir, conf.EtcdBinary, etcdPath, 0750, conf.QuietPull) - if err != nil { - return err - } - - if conf.PrometheusPort != 0 { - prometheusPath := c.GetBinPath(consts.ComponentPrometheus + conf.BinSuffix) - err = c.DownloadWithCache(ctx, conf.CacheDir, conf.PrometheusBinary, prometheusPath, 0750, conf.QuietPull) - if err != nil { - return err + for _, component := range env.kwokctlConfig.Components { + if component.Binary == "" { + continue } - } - - if conf.JaegerPort != 0 { - jaegerPath := c.GetBinPath("jaeger-all-in-one" + conf.BinSuffix) - err = c.DownloadWithCache(ctx, conf.CacheDir, conf.JaegerBinary, jaegerPath, 0750, conf.QuietPull) + err := c.DownloadWithCache(ctx, conf.CacheDir, component.Binary, c.GetBinPath(component.Name+conf.BinSuffix), 0750, conf.QuietPull) if err != nil { return err } @@ -294,11 +245,6 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - err = c.download(ctx, env) - if err != nil { - return err - } - err = c.setup(ctx, env) if err != nil { return err @@ -325,32 +271,17 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - err = c.addKubeControllerManager(ctx, env) - if err != nil { - return err - } - - err = c.addKubeScheduler(ctx, env) - if err != nil { - return err - } - - err = c.addKwokController(ctx, env) - if err != nil { - return err - } - - err = c.addPrometheus(ctx, env) + err = c.addJaeger(ctx, env) if err != nil { return err } - err = c.addJaeger(ctx, env) + err = c.setupFiles(ctx, env) if err != nil { return err } - err = c.setupPrometheusConfig(ctx, env) + err = c.download(ctx, env) if err != nil { return err } @@ -369,6 +300,11 @@ func (c *Cluster) addEtcd(ctx context.Context, env *env) (err error) { // Configure the etcd etcdPath := c.GetBinPath(consts.ComponentEtcd + conf.BinSuffix) + err = c.DownloadWithCache(ctx, conf.CacheDir, conf.EtcdBinary, etcdPath, 0750, conf.QuietPull) + if err != nil { + return err + } + etcdVersion, err := c.ParseVersionFromBinary(ctx, etcdPath) if err != nil { return err @@ -403,6 +339,11 @@ func (c *Cluster) addKubeApiserver(ctx context.Context, env *env) (err error) { // Configure the kube-apiserver kubeApiserverPath := c.GetBinPath(consts.ComponentKubeApiserver + conf.BinSuffix) + err = c.DownloadWithCache(ctx, conf.CacheDir, conf.KubeApiserverBinary, kubeApiserverPath, 0750, conf.QuietPull) + if err != nil { + return err + } + kubeApiserverVersion, err := c.ParseVersionFromBinary(ctx, kubeApiserverPath) if err != nil { return err @@ -467,222 +408,17 @@ func (c *Cluster) addKubeApiserver(ctx context.Context, env *env) (err error) { return nil } -func (c *Cluster) addKubeControllerManager(ctx context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - // Configure the kube-controller-manager - if !conf.DisableKubeControllerManager { - kubeControllerManagerPath := c.GetBinPath(consts.ComponentKubeControllerManager + conf.BinSuffix) - - err = c.setupPorts(ctx, - env.usedPorts, - &conf.KubeControllerManagerPort, - ) - if err != nil { - return err - } - - kubeControllerManagerVersion, err := c.ParseVersionFromBinary(ctx, kubeControllerManagerPath) - if err != nil { - return err - } - - kubeControllerManagerPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeControllerManager) - kubeControllerManagerComponent, err := components.BuildKubeControllerManagerComponent(components.BuildKubeControllerManagerComponentConfig{ - Runtime: conf.Runtime, - ProjectName: c.Name(), - Workdir: env.workdir, - Binary: kubeControllerManagerPath, - Version: kubeControllerManagerVersion, - BindAddress: conf.BindAddress, - Port: conf.KubeControllerManagerPort, - SecurePort: conf.SecurePort, - CaCertPath: env.caCertPath, - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - KubeAuthorization: conf.KubeAuthorization, - KubeconfigPath: env.kubeconfigPath, - KubeFeatureGates: conf.KubeFeatureGates, - NodeMonitorPeriodMilliseconds: conf.KubeControllerManagerNodeMonitorPeriodMilliseconds, - NodeMonitorGracePeriodMilliseconds: conf.KubeControllerManagerNodeMonitorGracePeriodMilliseconds, - Verbosity: env.verbosity, - DisableQPSLimits: conf.DisableQPSLimits, - ExtraArgs: kubeControllerManagerPatches.ExtraArgs, - ExtraVolumes: kubeControllerManagerPatches.ExtraVolumes, - ExtraEnvs: kubeControllerManagerPatches.ExtraEnvs, - }) - if err != nil { - return err - } - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kubeControllerManagerComponent) - } - return nil -} - -func (c *Cluster) addKubeScheduler(ctx context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - // Configure the kube-scheduler - if !conf.DisableKubeScheduler { - kubeSchedulerPath := c.GetBinPath(consts.ComponentKubeScheduler + conf.BinSuffix) - - schedulerConfigPath := "" - if conf.KubeSchedulerConfig != "" { - schedulerConfigPath = c.GetWorkdirPath(runtime.SchedulerConfigName) - err = c.CopySchedulerConfig(conf.KubeSchedulerConfig, schedulerConfigPath, env.kubeconfigPath) - if err != nil { - return err - } - } - - err = c.setupPorts(ctx, - env.usedPorts, - &conf.KubeSchedulerPort, - ) - if err != nil { - return err - } - - kubeSchedulerVersion, err := c.ParseVersionFromBinary(ctx, kubeSchedulerPath) - if err != nil { - return err - } - - kubeSchedulerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeScheduler) - kubeSchedulerComponent, err := components.BuildKubeSchedulerComponent(components.BuildKubeSchedulerComponentConfig{ - Runtime: conf.Runtime, - ProjectName: c.Name(), - Workdir: env.workdir, - Binary: kubeSchedulerPath, - Version: kubeSchedulerVersion, - BindAddress: conf.BindAddress, - Port: conf.KubeSchedulerPort, - SecurePort: conf.SecurePort, - CaCertPath: env.caCertPath, - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - ConfigPath: schedulerConfigPath, - KubeconfigPath: env.kubeconfigPath, - KubeFeatureGates: conf.KubeFeatureGates, - Verbosity: env.verbosity, - DisableQPSLimits: conf.DisableQPSLimits, - ExtraArgs: kubeSchedulerComponentPatches.ExtraArgs, - ExtraVolumes: kubeSchedulerComponentPatches.ExtraVolumes, - ExtraEnvs: kubeSchedulerComponentPatches.ExtraEnvs, - }) - if err != nil { - return err - } - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kubeSchedulerComponent) - } - return nil -} - -func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - // Configure the kwok-controller - kwokControllerPath := c.GetBinPath(consts.ComponentKwokController + conf.BinSuffix) - - kwokControllerVersion, err := c.ParseVersionFromBinary(ctx, kwokControllerPath) - if err != nil { - return err - } - - kwokControllerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKwokController) - - kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ - Runtime: conf.Runtime, - ProjectName: c.Name(), - Workdir: env.workdir, - Binary: kwokControllerPath, - Version: kwokControllerVersion, - BindAddress: conf.BindAddress, - Port: conf.KwokControllerPort, - ConfigPath: env.kwokConfigPath, - KubeconfigPath: env.kubeconfigPath, - CaCertPath: env.caCertPath, - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - NodeName: "localhost", - Verbosity: env.verbosity, - NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds, - EnableCRDs: conf.EnableCRDs, - ExtraArgs: kwokControllerComponentPatches.ExtraArgs, - ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs, - }) - if err != nil { - return err - } - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kwokControllerComponent) - return nil -} - -func (c *Cluster) setupPrometheusConfig(_ context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - // Configure the prometheus - if conf.PrometheusPort != 0 { - prometheusData, err := components.BuildPrometheus(components.BuildPrometheusConfig{ - Components: env.kwokctlConfig.Components, - }) - if err != nil { - return fmt.Errorf("failed to generate prometheus yaml: %w", err) - } - prometheusConfigPath := c.GetWorkdirPath(runtime.Prometheus) - err = c.WriteFile(prometheusConfigPath, []byte(prometheusData)) - if err != nil { - return fmt.Errorf("failed to write prometheus yaml: %w", err) - } - } - return nil -} - -func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { +func (c *Cluster) addJaeger(ctx context.Context, env *env) error { conf := &env.kwokctlConfig.Options - // Configure the prometheus - if conf.PrometheusPort != 0 { - prometheusPath := c.GetBinPath(consts.ComponentPrometheus + conf.BinSuffix) - - if err != nil { - return fmt.Errorf("failed to generate prometheus yaml: %w", err) - } - prometheusConfigPath := c.GetWorkdirPath(runtime.Prometheus) - - prometheusVersion, err := c.ParseVersionFromBinary(ctx, prometheusPath) - if err != nil { - return err - } + // Configure the jaeger + if conf.JaegerPort != 0 { + jaegerPath := c.GetBinPath(consts.ComponentJaeger + conf.BinSuffix) - prometheusComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentPrometheus) - prometheusComponent, err := components.BuildPrometheusComponent(components.BuildPrometheusComponentConfig{ - Runtime: conf.Runtime, - Workdir: env.workdir, - Binary: prometheusPath, - Version: prometheusVersion, - BindAddress: conf.BindAddress, - Port: conf.PrometheusPort, - ConfigPath: prometheusConfigPath, - Verbosity: env.verbosity, - ExtraArgs: prometheusComponentPatches.ExtraArgs, - ExtraVolumes: prometheusComponentPatches.ExtraVolumes, - ExtraEnvs: prometheusComponentPatches.ExtraEnvs, - }) + err := c.DownloadWithCache(ctx, conf.CacheDir, conf.JaegerBinary, jaegerPath, 0750, conf.QuietPull) if err != nil { return err } - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, prometheusComponent) - } - return nil -} - -func (c *Cluster) addJaeger(ctx context.Context, env *env) error { - conf := &env.kwokctlConfig.Options - - // Configure the jaeger - if conf.JaegerPort != 0 { - jaegerPath := c.GetBinPath("jaeger-all-in-one" + conf.BinSuffix) jaegerVersion, err := c.ParseVersionFromBinary(ctx, jaegerPath) if err != nil { @@ -753,10 +489,26 @@ func (c *Cluster) Uninstall(ctx context.Context) error { } func (c *Cluster) isRunning(ctx context.Context, component internalversion.Component) bool { - return c.ForkExecIsRunning(ctx, component.WorkDir, component.Binary) + config, err := c.Config(ctx) + if err != nil { + return false + } + conf := config.Options + binaryPath := c.GetBinPath(component.Name + conf.BinSuffix) + workdir := component.WorkDir + if workdir == "" { + workdir = c.Workdir() + } + return c.ForkExecIsRunning(ctx, workdir, binaryPath) } func (c *Cluster) startComponent(ctx context.Context, component internalversion.Component) error { + config, err := c.Config(ctx) + if err != nil { + return err + } + conf := config.Options + logger := log.FromContext(ctx) logger = logger.With("component", component.Name) if c.isRunning(ctx, component) { @@ -785,9 +537,14 @@ func (c *Cluster) startComponent(ctx context.Context, component internalversion. } ctx = exec.WithUser(ctx, &uid, &gid) } - logger.Debug("Starting component") - return c.ForkExec(ctx, component.WorkDir, component.Binary, component.Args...) + binaryPath := c.GetBinPath(component.Name + conf.BinSuffix) + workdir := component.WorkDir + if workdir == "" { + workdir = c.Workdir() + } + + return c.ForkExec(ctx, workdir, binaryPath, component.Args...) } func (c *Cluster) startComponents(ctx context.Context) error { @@ -801,6 +558,12 @@ func (c *Cluster) startComponents(ctx context.Context) error { } func (c *Cluster) stopComponent(ctx context.Context, component internalversion.Component) error { + config, err := c.Config(ctx) + if err != nil { + return err + } + conf := config.Options + logger := log.FromContext(ctx) logger = logger.With("component", component.Name) if !c.isRunning(ctx, component) { @@ -808,7 +571,12 @@ func (c *Cluster) stopComponent(ctx context.Context, component internalversion.C return nil } logger.Debug("Stopping component") - return c.ForkExecKill(ctx, component.WorkDir, component.Binary) + binaryPath := c.GetBinPath(component.Name + conf.BinSuffix) + workdir := component.WorkDir + if workdir == "" { + workdir = c.Workdir() + } + return c.ForkExecKill(ctx, workdir, binaryPath) } func (c *Cluster) stopComponents(ctx context.Context) error { diff --git a/pkg/kwokctl/runtime/binary/component.go b/pkg/kwokctl/runtime/binary/component.go new file mode 100644 index 0000000000..abfb99dcc5 --- /dev/null +++ b/pkg/kwokctl/runtime/binary/component.go @@ -0,0 +1,127 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package binary + +import ( + "context" + "encoding/json" + "fmt" + + "sigs.k8s.io/kwok/kustomize/kwokctl/component" + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/config" + "sigs.k8s.io/kwok/pkg/consts" + "sigs.k8s.io/kwok/pkg/kwokctl/components" + "sigs.k8s.io/kwok/pkg/kwokctl/runtime" + "sigs.k8s.io/kwok/pkg/kwokctl/scale" + "sigs.k8s.io/kwok/pkg/utils/gotpl" + "sigs.k8s.io/kwok/pkg/utils/path" + "sigs.k8s.io/kwok/pkg/utils/slices" +) + +// SetComponents returns the components of cluster +func (c *Cluster) SetComponents(ctx context.Context, name string, args ...string) error { + conf, err := c.Config(ctx) + if err != nil { + return err + } + _, ok := slices.Find(conf.Components, func(component internalversion.Component) bool { + return component.Name == name + }) + if ok { + return fmt.Errorf("component %s is already exists", name) + } + + kcp := config.FilterWithTypeFromContext[*internalversion.KwokctlComponent](ctx) + renderer := gotpl.NewRenderer(gotpl.FuncMap{ + "ClusterName": c.Name, + "Workdir": c.Workdir, + "Runtime": func() string { + return c.Runtime(ctx) + }, + "Mode": func() string { + return c.Mode(ctx) + }, + "Address": func() string { + return c.ComponentAddress(ctx, name) + }, + "FilesDir": func() string { + return path.Join(c.Workdir(), "files", name) + }, + "PkiDir": func() string { + return path.Join(c.Workdir(), runtime.PkiName) + }, + "Kubeconfig": func() string { + return c.GetWorkdirPath(runtime.InHostKubeconfigName) + }, + "Config": func() *internalversion.KwokctlConfiguration { + return conf + }, + }) + + krc, ok := slices.Find(kcp, func(krc *internalversion.KwokctlComponent) bool { + return krc.Name == name + }) + if !ok { + var componentData string + switch name { + default: + return fmt.Errorf("component %s is not exists", name) + case consts.ComponentDashboard: + componentData = component.DefaultDashboard + case consts.ComponentPrometheus: + componentData = component.DefaultPrometheus + case consts.ComponentKwokController: + componentData = component.DefaultKwokController + case consts.ComponentKubeControllerManager: + componentData = component.DefaultKubeControllerManager + case consts.ComponentKubeScheduler: + componentData = component.DefaultKubeScheduler + } + krc, err = config.UnmarshalWithType[*internalversion.KwokctlComponent](componentData) + if err != nil { + return err + } + } + + param, err := scale.NewParameters(ctx, krc.Parameters, args) + if err != nil { + return err + } + + componentData, err := renderer.ToJSON(krc.Template, param) + if err != nil { + return err + } + var component internalversion.Component + err = json.Unmarshal(componentData, &component) + if err != nil { + return err + } + component.Name = name + + for _, patch := range conf.ComponentsPatches { + if patch.Name != name { + continue + } + component = components.PatchComponent(component, patch) + } + + conf.Components = append(conf.Components, component) + + return c.SetConfig(ctx, conf) +} diff --git a/pkg/kwokctl/runtime/binary/file.go b/pkg/kwokctl/runtime/binary/file.go new file mode 100644 index 0000000000..36026cace9 --- /dev/null +++ b/pkg/kwokctl/runtime/binary/file.go @@ -0,0 +1,64 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package binary + +import ( + "context" + + "sigs.k8s.io/kwok/pkg/utils/gotpl" + "sigs.k8s.io/kwok/pkg/utils/path" +) + +func (c *Cluster) setupFiles(ctx context.Context, env *env) (err error) { + r := gotpl.NewRenderer(nil) + pathDir := c.GetWorkdirPath("files") + for _, component := range env.kwokctlConfig.Components { + for _, file := range component.Files { + var data []byte + if file.Data != "" { + data = []byte(file.Data) + } else if file.Template != "" { + d, err := r.ToText(file.Template, env.kwokctlConfig) + if err != nil { + return err + } + data = d + } + + filePath := path.Join(pathDir, component.Name, file.Path) + + err = c.MkdirAll(path.Dir(filePath)) + if err != nil { + return err + } + wc, err := c.OpenFile(filePath) + if err != nil { + return err + } + _, err = wc.Write(data) + if err != nil { + return err + } + + err = wc.Close() + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/kwokctl/runtime/cluster.go b/pkg/kwokctl/runtime/cluster.go index 00b71ed9d8..c8f8bf6eb1 100644 --- a/pkg/kwokctl/runtime/cluster.go +++ b/pkg/kwokctl/runtime/cluster.go @@ -55,7 +55,6 @@ var ( EtcdDataDirName = "etcd" PkiName = "pki" ManifestsName = "manifests" - Prometheus = "prometheus.yaml" KindName = "kind.yaml" AuditPolicyName = "audit.yaml" AuditLogName = "audit.log" diff --git a/pkg/kwokctl/runtime/component.go b/pkg/kwokctl/runtime/component.go new file mode 100644 index 0000000000..8d150a964c --- /dev/null +++ b/pkg/kwokctl/runtime/component.go @@ -0,0 +1,31 @@ +package runtime + +import ( + "context" + + "sigs.k8s.io/kwok/pkg/kwokctl/components" + "sigs.k8s.io/kwok/pkg/utils/net" +) + +func (c *Cluster) Runtime(ctx context.Context) string { + config, err := c.Config(ctx) + if err != nil { + return "" + } + conf := &config.Options + + return conf.Runtime +} + +func (c *Cluster) Mode(ctx context.Context) string { + return components.GetRuntimeMode(c.Runtime(ctx)) +} + +func (c *Cluster) ComponentAddress(ctx context.Context, name string) string { + switch c.Mode(ctx) { + case components.RuntimeModeContainer: + return c.Name() + "-" + name + default: + return net.LocalAddress + } +} diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index b46c2c3e46..3502a23bb0 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -40,6 +40,7 @@ import ( "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/path" "sigs.k8s.io/kwok/pkg/utils/sets" + "sigs.k8s.io/kwok/pkg/utils/slices" "sigs.k8s.io/kwok/pkg/utils/wait" "sigs.k8s.io/kwok/pkg/utils/yaml" ) @@ -134,26 +135,17 @@ func (c *Cluster) Available(ctx context.Context) error { func (c *Cluster) pullAllImages(ctx context.Context, env *env) error { conf := &env.kwokctlConfig.Options - images := []string{ - conf.EtcdImage, - conf.KubeApiserverImage, - conf.KwokControllerImage, - } - if !conf.DisableKubeControllerManager { - images = append(images, conf.KubeControllerManagerImage) - } - if !conf.DisableKubeScheduler { - images = append(images, conf.KubeSchedulerImage) - } - if conf.DashboardPort != 0 { - images = append(images, conf.DashboardImage) - } - if conf.PrometheusPort != 0 { - images = append(images, conf.PrometheusImage) - } - if conf.JaegerPort != 0 { - images = append(images, conf.JaegerImage) - } + + images := slices.FilterAndMap(env.kwokctlConfig.Components, func(component internalversion.Component) (string, bool) { + if component.Image == "" { + return "", false + } + return component.Image, true + }) + + images = slices.Unique(images) + images = slices.Sort(images) + err := c.PullImages(ctx, c.runtime, images, conf.QuietPull) if err != nil { return err @@ -349,37 +341,12 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - err = c.addKubeControllerManager(ctx, env) - if err != nil { - return err - } - - err = c.addKubeScheduler(ctx, env) - if err != nil { - return err - } - - err = c.addKwokController(ctx, env) - if err != nil { - return err - } - - err = c.addPrometheus(ctx, env) - if err != nil { - return err - } - err = c.addJaeger(ctx, env) if err != nil { return err } - err = c.addDashboard(ctx, env) - if err != nil { - return err - } - - err = c.setupPrometheusConfig(ctx, env) + err = c.setupFiles(ctx, env) if err != nil { return err } @@ -497,241 +464,6 @@ func (c *Cluster) addKubeApiserver(ctx context.Context, env *env) (err error) { return nil } -func (c *Cluster) addKubeControllerManager(ctx context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - // Configure the kube-controller-manager - if !conf.DisableKubeControllerManager { - kubeControllerManagerVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.KubeControllerManagerImage, consts.ComponentKubeControllerManager) - if err != nil { - return err - } - - kubeControllerManagerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeControllerManager) - kubeControllerManagerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kubeControllerManagerComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for kube controller manager component: %w", err) - } - kubeControllerManagerComponent, err := components.BuildKubeControllerManagerComponent(components.BuildKubeControllerManagerComponentConfig{ - Runtime: conf.Runtime, - ProjectName: c.Name(), - Workdir: env.workdir, - Image: conf.KubeControllerManagerImage, - Version: kubeControllerManagerVersion, - BindAddress: net.PublicAddress, - Port: conf.KubeControllerManagerPort, - SecurePort: conf.SecurePort, - CaCertPath: env.caCertPath, - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - KubeAuthorization: conf.KubeAuthorization, - KubeconfigPath: env.inClusterOnHostKubeconfigPath, - KubeFeatureGates: conf.KubeFeatureGates, - Verbosity: env.verbosity, - DisableQPSLimits: conf.DisableQPSLimits, - NodeMonitorPeriodMilliseconds: conf.KubeControllerManagerNodeMonitorPeriodMilliseconds, - NodeMonitorGracePeriodMilliseconds: conf.KubeControllerManagerNodeMonitorGracePeriodMilliseconds, - ExtraArgs: kubeControllerManagerComponentPatches.ExtraArgs, - ExtraVolumes: kubeControllerManagerComponentPatches.ExtraVolumes, - ExtraEnvs: kubeControllerManagerComponentPatches.ExtraEnvs, - }) - if err != nil { - return err - } - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kubeControllerManagerComponent) - } - return nil -} - -func (c *Cluster) addKubeScheduler(ctx context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - // Configure the kube-scheduler - if !conf.DisableKubeScheduler { - schedulerConfigPath := "" - if conf.KubeSchedulerConfig != "" { - schedulerConfigPath = c.GetWorkdirPath(runtime.SchedulerConfigName) - err = c.CopySchedulerConfig(conf.KubeSchedulerConfig, schedulerConfigPath, env.inClusterKubeconfig) - if err != nil { - return err - } - } - - kubeSchedulerVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.KubeSchedulerImage, consts.ComponentKubeScheduler) - if err != nil { - return err - } - - kubeSchedulerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeScheduler) - kubeSchedulerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kubeSchedulerComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for kube scheduler component: %w", err) - } - kubeSchedulerComponent, err := components.BuildKubeSchedulerComponent(components.BuildKubeSchedulerComponentConfig{ - Runtime: conf.Runtime, - ProjectName: c.Name(), - Workdir: env.workdir, - Image: conf.KubeSchedulerImage, - Version: kubeSchedulerVersion, - BindAddress: net.PublicAddress, - Port: conf.KubeSchedulerPort, - SecurePort: conf.SecurePort, - CaCertPath: env.caCertPath, - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - ConfigPath: schedulerConfigPath, - KubeconfigPath: env.inClusterOnHostKubeconfigPath, - KubeFeatureGates: conf.KubeFeatureGates, - Verbosity: env.verbosity, - DisableQPSLimits: conf.DisableQPSLimits, - ExtraArgs: kubeSchedulerComponentPatches.ExtraArgs, - ExtraVolumes: kubeSchedulerComponentPatches.ExtraVolumes, - ExtraEnvs: kubeSchedulerComponentPatches.ExtraEnvs, - }) - if err != nil { - return err - } - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kubeSchedulerComponent) - } - return nil -} - -func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - // Configure the kwok-controller - kwokControllerVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.KwokControllerImage, "kwok") - if err != nil { - return err - } - - kwokControllerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKwokController) - kwokControllerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kwokControllerComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for kwok controller component: %w", err) - } - - logVolumes := runtime.GetLogVolumes(ctx) - kwokControllerExtraVolumes := kwokControllerComponentPatches.ExtraVolumes - kwokControllerExtraVolumes = append(kwokControllerExtraVolumes, logVolumes...) - - kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ - Runtime: conf.Runtime, - ProjectName: c.Name(), - Workdir: env.workdir, - Image: conf.KwokControllerImage, - Version: kwokControllerVersion, - BindAddress: net.PublicAddress, - Port: conf.KwokControllerPort, - ConfigPath: env.kwokConfigPath, - KubeconfigPath: env.inClusterOnHostKubeconfigPath, - CaCertPath: env.caCertPath, - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - NodeName: c.Name() + "-kwok-controller", - Verbosity: env.verbosity, - NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds, - EnableCRDs: conf.EnableCRDs, - ExtraArgs: kwokControllerComponentPatches.ExtraArgs, - ExtraVolumes: kwokControllerExtraVolumes, - ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs, - }) - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kwokControllerComponent) - return nil -} - -func (c *Cluster) setupPrometheusConfig(_ context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - // Configure the prometheus - if conf.PrometheusPort != 0 { - prometheusData, err := components.BuildPrometheus(components.BuildPrometheusConfig{ - Components: env.kwokctlConfig.Components, - }) - if err != nil { - return fmt.Errorf("failed to generate prometheus yaml: %w", err) - } - prometheusConfigPath := c.GetWorkdirPath(runtime.Prometheus) - - // We don't need to check the permissions of the prometheus config file, - // because it's working in a non-root container. - err = c.WriteFileWithMode(prometheusConfigPath, []byte(prometheusData), 0644) - if err != nil { - return fmt.Errorf("failed to write prometheus yaml: %w", err) - } - } - - return nil -} - -func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - // Configure the prometheus - if conf.PrometheusPort != 0 { - prometheusConfigPath := c.GetWorkdirPath(runtime.Prometheus) - prometheusVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.PrometheusImage, "") - if err != nil { - return err - } - - prometheusComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentPrometheus) - prometheusComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(prometheusComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for prometheus component: %w", err) - } - prometheusComponent, err := components.BuildPrometheusComponent(components.BuildPrometheusComponentConfig{ - Runtime: conf.Runtime, - Workdir: env.workdir, - Image: conf.PrometheusImage, - Version: prometheusVersion, - BindAddress: net.PublicAddress, - Port: conf.PrometheusPort, - ConfigPath: prometheusConfigPath, - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - Verbosity: env.verbosity, - ExtraArgs: prometheusComponentPatches.ExtraArgs, - ExtraVolumes: prometheusComponentPatches.ExtraVolumes, - ExtraEnvs: prometheusComponentPatches.ExtraEnvs, - }) - if err != nil { - return err - } - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, prometheusComponent) - } - return nil -} - -func (c *Cluster) addDashboard(_ context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - if conf.DashboardPort != 0 { - dashboardComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentDashboard) - dashboardComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(dashboardComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for dashboard component: %w", err) - } - dashboardComponent, err := components.BuildDashboardComponent(components.BuildDashboardComponentConfig{ - Runtime: conf.Runtime, - Workdir: env.workdir, - Image: conf.DashboardImage, - BindAddress: net.PublicAddress, - KubeconfigPath: env.inClusterOnHostKubeconfigPath, - CaCertPath: env.caCertPath, - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - Port: conf.DashboardPort, - Banner: fmt.Sprintf("Welcome to %s", c.Name()), - }) - if err != nil { - return err - } - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, dashboardComponent) - } - return nil -} - func (c *Cluster) addJaeger(ctx context.Context, env *env) error { conf := &env.kwokctlConfig.Options diff --git a/pkg/kwokctl/runtime/compose/component.go b/pkg/kwokctl/runtime/compose/component.go new file mode 100644 index 0000000000..bb920bdf14 --- /dev/null +++ b/pkg/kwokctl/runtime/compose/component.go @@ -0,0 +1,146 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package compose + +import ( + "context" + "encoding/json" + "fmt" + + "sigs.k8s.io/kwok/kustomize/kwokctl/component" + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/config" + "sigs.k8s.io/kwok/pkg/consts" + "sigs.k8s.io/kwok/pkg/kwokctl/components" + "sigs.k8s.io/kwok/pkg/kwokctl/runtime" + "sigs.k8s.io/kwok/pkg/kwokctl/scale" + "sigs.k8s.io/kwok/pkg/utils/gotpl" + "sigs.k8s.io/kwok/pkg/utils/slices" +) + +// SetComponents returns the components of cluster +func (c *Cluster) SetComponents(ctx context.Context, name string, args ...string) error { + conf, err := c.Config(ctx) + if err != nil { + return err + } + _, ok := slices.Find(conf.Components, func(component internalversion.Component) bool { + return component.Name == name + }) + if ok { + return fmt.Errorf("component %s is already exists", name) + } + + kcp := config.FilterWithTypeFromContext[*internalversion.KwokctlComponent](ctx) + renderer := gotpl.NewRenderer(gotpl.FuncMap{ + "ClusterName": c.Name, + "Workdir": c.Workdir, + "Runtime": func() string { + return c.Runtime(ctx) + }, + "Mode": func() string { + return c.Mode(ctx) + }, + "Address": func() string { + return c.ComponentAddress(ctx, name) + }, + "FilesDir": func() string { + return "" + }, + "PkiDir": func() string { + return "/etc/kubernetes/pki" + }, + "Kubeconfig": func() string { + return "/root/.kube/config" + }, + "Config": func() *internalversion.KwokctlConfiguration { + return conf + }, + }) + + krc, ok := slices.Find(kcp, func(krc *internalversion.KwokctlComponent) bool { + return krc.Name == name + }) + if !ok { + var componentData string + switch name { + default: + return fmt.Errorf("component %s is not exists", name) + case consts.ComponentDashboard: + componentData = component.DefaultDashboard + case consts.ComponentPrometheus: + componentData = component.DefaultPrometheus + case consts.ComponentKwokController: + componentData = component.DefaultKwokController + case consts.ComponentKubeControllerManager: + componentData = component.DefaultKubeControllerManager + case consts.ComponentKubeScheduler: + componentData = component.DefaultKubeScheduler + } + krc, err = config.UnmarshalWithType[*internalversion.KwokctlComponent](componentData) + if err != nil { + return err + } + } + + param, err := scale.NewParameters(ctx, krc.Parameters, args) + if err != nil { + return err + } + + componentData, err := renderer.ToJSON(krc.Template, param) + if err != nil { + return err + } + var component internalversion.Component + err = json.Unmarshal(componentData, &component) + if err != nil { + return err + } + component.Name = name + + volumes := []internalversion.Volume{ + { + HostPath: c.GetWorkdirPath(runtime.InClusterKubeconfigName), + MountPath: "/root/.kube/config", + }, + { + HostPath: c.GetWorkdirPath(runtime.PkiName), + MountPath: "/etc/kubernetes/pki/", + }, + } + + if name == consts.ComponentKwokController { + volumes = append(volumes, internalversion.Volume{ + HostPath: c.GetWorkdirPath(runtime.ConfigName), + MountPath: "/root/.kwok/kwok.yaml", + }) + } + + component.Volumes = append(component.Volumes, volumes...) + + for _, patch := range conf.ComponentsPatches { + if patch.Name != name { + continue + } + component = components.PatchComponent(component, patch) + } + + conf.Components = append(conf.Components, component) + + return c.SetConfig(ctx, conf) +} diff --git a/pkg/kwokctl/runtime/compose/file.go b/pkg/kwokctl/runtime/compose/file.go new file mode 100644 index 0000000000..28c183e734 --- /dev/null +++ b/pkg/kwokctl/runtime/compose/file.go @@ -0,0 +1,74 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package compose + +import ( + "context" + + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/gotpl" + "sigs.k8s.io/kwok/pkg/utils/path" +) + +func (c *Cluster) setupFiles(ctx context.Context, env *env) (err error) { + r := gotpl.NewRenderer(nil) + pathDir := c.GetWorkdirPath("files") + for index, component := range env.kwokctlConfig.Components { + for i, file := range component.Files { + var data []byte + if file.Data != "" { + data = []byte(file.Data) + } else if file.Template != "" { + d, err := r.ToText(file.Template, env.kwokctlConfig) + if err != nil { + return err + } + data = d + } + + filePath := path.Join(pathDir, component.Name, file.Path) + + err = c.MkdirAll(path.Dir(filePath)) + if err != nil { + return err + } + wc, err := c.OpenFile(filePath) + if err != nil { + return err + } + _, err = wc.Write(data) + if err != nil { + return err + } + + err = wc.Close() + if err != nil { + return err + } + + v := internalversion.Volume{ + Name: "file-" + format.String(i), + MountPath: file.Path, + HostPath: filePath, + } + + env.kwokctlConfig.Components[index].Volumes = append(env.kwokctlConfig.Components[index].Volumes, v) + } + } + return nil +} diff --git a/pkg/kwokctl/runtime/config.go b/pkg/kwokctl/runtime/config.go index 3c3b72d0d9..812593e934 100644 --- a/pkg/kwokctl/runtime/config.go +++ b/pkg/kwokctl/runtime/config.go @@ -65,6 +65,9 @@ type Runtime interface { // GetComponent return the component if it exists GetComponent(ctx context.Context, name string) (internalversion.Component, error) + // SetComponents returns the components of cluster + SetComponents(ctx context.Context, name string, args ...string) error + // Ready check the cluster is ready Ready(ctx context.Context) (bool, error) diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 814a3ed950..f1235ba644 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -105,6 +105,13 @@ func (c *Cluster) setup(ctx context.Context, env *env) error { if err != nil { return fmt.Errorf("failed to create pki dir: %w", err) } + + manifestsPath := c.GetWorkdirPath(runtime.ManifestsName) + err = c.MkdirAll(manifestsPath) + if err != nil { + return err + } + return nil } @@ -196,11 +203,6 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - err = c.addKind(ctx, env) - if err != nil { - return err - } - err = c.addEtcd(ctx, env) if err != nil { return err @@ -211,37 +213,22 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - err = c.addKubeControllerManager(ctx, env) - if err != nil { - return err - } - - err = c.addKubeScheduler(ctx, env) - if err != nil { - return err - } - - err = c.addKwokController(ctx, env) - if err != nil { - return err - } - - err = c.addDashboard(ctx, env) + err = c.addJaeger(ctx, env) if err != nil { return err } - err = c.addPrometheus(ctx, env) + err = c.setupFiles(ctx, env) if err != nil { return err } - err = c.addJaeger(ctx, env) + err = c.setupManifests(ctx, env) if err != nil { return err } - err = c.setupPrometheusConfig(ctx, env) + err = c.addKind(ctx, env) if err != nil { return err } @@ -277,12 +264,6 @@ func (c *Cluster) addKind(ctx context.Context, env *env) (err error) { return err } - manifestsPath := c.GetWorkdirPath(runtime.ManifestsName) - err = c.MkdirAll(manifestsPath) - if err != nil { - return err - } - if conf.KubeAuditPolicy != "" { err = c.MkdirAll(c.GetWorkdirPath("logs")) if err != nil { @@ -325,18 +306,6 @@ func (c *Cluster) addKind(ctx context.Context, env *env) (err error) { } } - var prometheusPatches internalversion.ComponentPatches - if conf.PrometheusPort != 0 { - prometheusPatches = runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentPrometheus) - prometheusConfigPath := c.GetWorkdirPath(runtime.Prometheus) - - prometheusPatches.ExtraVolumes = append(prometheusPatches.ExtraVolumes, internalversion.Volume{ - Name: "prometheus-config", - HostPath: prometheusConfigPath, - MountPath: env.prometheusConfigPath, - }) - } - kubeVersion, err := version.ParseVersion(conf.KubeVersion) if err != nil { return err @@ -356,13 +325,17 @@ func (c *Cluster) addKind(ctx context.Context, env *env) (err error) { len(kubeControllerManagerComponentPatches.ExtraEnvs) > 0 { logger.Warn("extraEnvs config in etcd, kube-apiserver, kube-scheduler or kube-controller-manager is not supported in kind") } + + ports := []internalversion.Port{} + for _, component := range env.kwokctlConfig.Components { + ports = append(ports, component.Ports...) + } + kindYaml, err := BuildKind(BuildKindConfig{ BindAddress: conf.BindAddress, KubeApiserverPort: conf.KubeApiserverPort, EtcdPort: conf.EtcdPort, JaegerPort: conf.JaegerPort, - DashboardPort: conf.DashboardPort, - PrometheusPort: conf.PrometheusPort, KwokControllerPort: conf.KwokControllerPort, FeatureGates: featureGates, RuntimeConfig: runtimeConfig, @@ -381,9 +354,10 @@ func (c *Cluster) addKind(ctx context.Context, env *env) (err error) { ControllerManagerExtraArgs: kubeControllerManagerComponentPatches.ExtraArgs, ControllerManagerExtraVolumes: kubeControllerManagerComponentPatches.ExtraVolumes, KwokControllerExtraVolumes: kwokControllerExtraVolumes, - PrometheusExtraVolumes: prometheusPatches.ExtraVolumes, DisableQPSLimits: conf.DisableQPSLimits, KubeVersion: kubeVersion, + Ports: ports, + Components: env.kwokctlConfig.Components, }) if err != nil { return err @@ -426,232 +400,6 @@ func (c *Cluster) addKubeApiserver(_ context.Context, env *env) (err error) { return nil } -func (c *Cluster) addKubeControllerManager(_ context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - if !conf.DisableKubeControllerManager { - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, internalversion.Component{ - Name: consts.ComponentKubeControllerManager, - Metric: &internalversion.ComponentMetric{ - Scheme: "https", - Host: "127.0.0.1:10257", - Path: "/metrics", - CertPath: "/etc/kubernetes/pki/admin.crt", - KeyPath: "/etc/kubernetes/pki/admin.key", - InsecureSkipVerify: true, - }, - }) - } - return nil -} - -func (c *Cluster) addKubeScheduler(_ context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - if !conf.DisableKubeScheduler { - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, internalversion.Component{ - Name: consts.ComponentKubeScheduler, - Metric: &internalversion.ComponentMetric{ - Scheme: "https", - Host: "127.0.0.1:10259", - Path: "/metrics", - CertPath: "/etc/kubernetes/pki/admin.crt", - KeyPath: "/etc/kubernetes/pki/admin.key", - InsecureSkipVerify: true, - }, - }) - } - return nil -} - -func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - // Configure the kwok-controller - kwokControllerVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.KwokControllerImage, "kwok") - if err != nil { - return err - } - - kwokControllerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKwokController) - kwokControllerComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(kwokControllerComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for kwok controller component: %w", err) - } - - logVolumes := runtime.GetLogVolumes(ctx) - logVolumes = slices.Map(logVolumes, func(v internalversion.Volume) internalversion.Volume { - v.HostPath = path.Join("/var/components/controller", v.HostPath) - return v - }) - - kwokControllerExtraVolumes := kwokControllerComponentPatches.ExtraVolumes - kwokControllerExtraVolumes = append(kwokControllerExtraVolumes, logVolumes...) - - kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ - Runtime: conf.Runtime, - ProjectName: c.Name(), - Workdir: env.workdir, - Image: conf.KwokControllerImage, - Version: kwokControllerVersion, - BindAddress: net.PublicAddress, - Port: conf.KwokControllerPort, - ConfigPath: env.kwokConfigPath, - KubeconfigPath: env.inClusterOnHostKubeconfigPath, - CaCertPath: env.caCertPath, - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - NodeIP: "$(POD_IP)", - NodeName: "kwok-controller.kube-system.svc", - ManageNodesWithAnnotationSelector: "kwok.x-k8s.io/node=fake", - Verbosity: env.verbosity, - NodeLeaseDurationSeconds: 40, - EnableCRDs: conf.EnableCRDs, - ExtraArgs: kwokControllerComponentPatches.ExtraArgs, - ExtraVolumes: kwokControllerExtraVolumes, - ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs, - }) - - pod := components.ConvertToPod(kwokControllerComponent) - pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{ - Name: "POD_IP", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIP", - }, - }, - }) - kwokControllerPod, err := yaml.Marshal(pod) - if err != nil { - return fmt.Errorf("failed to marshal kwok controller pod: %w", err) - } - err = c.WriteFile(path.Join(c.GetWorkdirPath(runtime.ManifestsName), consts.ComponentKwokController+".yaml"), kwokControllerPod) - if err != nil { - return fmt.Errorf("failed to write: %w", err) - } - - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, kwokControllerComponent) - return nil -} - -func (c *Cluster) addDashboard(ctx context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - if conf.DashboardPort != 0 { - dashboardVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.DashboardImage, "") - if err != nil { - return err - } - - dashboardComponent, err := components.BuildDashboardComponent(components.BuildDashboardComponentConfig{ - Runtime: conf.Runtime, - Workdir: env.workdir, - Image: conf.DashboardImage, - Version: dashboardVersion, - BindAddress: net.PublicAddress, - KubeconfigPath: env.inClusterOnHostKubeconfigPath, - CaCertPath: env.caCertPath, - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - Port: 8000, - Banner: fmt.Sprintf("Welcome to %s", c.Name()), - }) - if err != nil { - return fmt.Errorf("failed to build dashboard component: %w", err) - } - - dashboardPod, err := yaml.Marshal(components.ConvertToPod(dashboardComponent)) - if err != nil { - return fmt.Errorf("failed to marshal dashboard pod: %w", err) - } - err = c.WriteFile(path.Join(c.GetWorkdirPath(runtime.ManifestsName), consts.ComponentDashboard+".yaml"), dashboardPod) - if err != nil { - return fmt.Errorf("failed to write: %w", err) - } - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, dashboardComponent) - } - return nil -} - -func (c *Cluster) setupPrometheusConfig(_ context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - // Configure the prometheus - if conf.PrometheusPort != 0 { - prometheusData, err := components.BuildPrometheus(components.BuildPrometheusConfig{ - Components: env.kwokctlConfig.Components, - }) - if err != nil { - return fmt.Errorf("failed to generate prometheus yaml: %w", err) - } - prometheusConfigPath := c.GetWorkdirPath(runtime.Prometheus) - err = c.WriteFile(prometheusConfigPath, []byte(prometheusData)) - if err != nil { - return fmt.Errorf("failed to write prometheus yaml: %w", err) - } - } - return nil -} - -func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { - conf := &env.kwokctlConfig.Options - - if conf.PrometheusPort != 0 { - prometheusVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.PrometheusImage, "") - if err != nil { - return err - } - - prometheusComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentPrometheus) - prometheusComponentPatches.ExtraVolumes, err = runtime.ExpandVolumesHostPaths(prometheusComponentPatches.ExtraVolumes) - if err != nil { - return fmt.Errorf("failed to expand host volumes for prometheus component: %w", err) - } - - prometheusComponentPatches.ExtraVolumes = append(prometheusComponentPatches.ExtraVolumes, - internalversion.Volume{ - HostPath: "/etc/kubernetes/pki/apiserver-etcd-client.crt", - MountPath: "/etc/kubernetes/pki/apiserver-etcd-client.crt", - ReadOnly: true, - }, - internalversion.Volume{ - HostPath: "/etc/kubernetes/pki/apiserver-etcd-client.key", - MountPath: "/etc/kubernetes/pki/apiserver-etcd-client.key", - ReadOnly: true, - }, - ) - - prometheusComponent, err := components.BuildPrometheusComponent(components.BuildPrometheusComponentConfig{ - Runtime: conf.Runtime, - Workdir: env.workdir, - Image: conf.PrometheusImage, - Version: prometheusVersion, - BindAddress: net.PublicAddress, - Port: 9090, - ConfigPath: "/var/components/prometheus/etc/prometheus/prometheus.yaml", - AdminCertPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - Verbosity: env.verbosity, - ExtraArgs: prometheusComponentPatches.ExtraArgs, - ExtraVolumes: prometheusComponentPatches.ExtraVolumes, - ExtraEnvs: prometheusComponentPatches.ExtraEnvs, - }) - if err != nil { - return err - } - - prometheusPod, err := yaml.Marshal(components.ConvertToPod(prometheusComponent)) - if err != nil { - return fmt.Errorf("failed to marshal prometheus pod: %w", err) - } - err = c.WriteFile(path.Join(c.GetWorkdirPath(runtime.ManifestsName), consts.ComponentPrometheus+".yaml"), prometheusPod) - if err != nil { - return fmt.Errorf("failed to write: %w", err) - } - - env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, prometheusComponent) - } - return nil -} - func (c *Cluster) addJaeger(ctx context.Context, env *env) error { conf := &env.kwokctlConfig.Options @@ -818,17 +566,20 @@ func (c *Cluster) listAllImages(ctx context.Context) ([]string, error) { if err != nil { return nil, err } - conf := &config.Options - images := []string{conf.KwokControllerImage} - if conf.DashboardPort != 0 { - images = append(images, conf.DashboardImage) - } - if conf.PrometheusPort != 0 { - images = append(images, conf.PrometheusImage) - } - if conf.JaegerPort != 0 { - images = append(images, conf.JaegerImage) - } + + images := slices.FilterAndMap(config.Components, func(component internalversion.Component) (string, bool) { + if component.Image == "" || + component.Name == consts.ComponentEtcd || + component.Name == consts.ComponentKubeApiserver || + component.Name == consts.ComponentKubeControllerManager || + component.Name == consts.ComponentKubeScheduler { + return "", false + } + return component.Image, true + }) + + images = slices.Unique(images) + images = slices.Sort(images) return images, nil } diff --git a/pkg/kwokctl/runtime/kind/component.go b/pkg/kwokctl/runtime/kind/component.go new file mode 100644 index 0000000000..5be9991f5b --- /dev/null +++ b/pkg/kwokctl/runtime/kind/component.go @@ -0,0 +1,125 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kind + +import ( + "context" + "encoding/json" + "fmt" + + "sigs.k8s.io/kwok/kustomize/kwokctl/component" + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/config" + "sigs.k8s.io/kwok/pkg/consts" + "sigs.k8s.io/kwok/pkg/kwokctl/components" + "sigs.k8s.io/kwok/pkg/kwokctl/scale" + "sigs.k8s.io/kwok/pkg/utils/gotpl" + "sigs.k8s.io/kwok/pkg/utils/slices" +) + +// SetComponents returns the components of cluster +func (c *Cluster) SetComponents(ctx context.Context, name string, args ...string) error { + conf, err := c.Config(ctx) + if err != nil { + return err + } + _, ok := slices.Find(conf.Components, func(component internalversion.Component) bool { + return component.Name == name + }) + if ok { + return fmt.Errorf("component %s is already exists", name) + } + + kcp := config.FilterWithTypeFromContext[*internalversion.KwokctlComponent](ctx) + renderer := gotpl.NewRenderer(gotpl.FuncMap{ + "ClusterName": c.Name, + "Workdir": c.Workdir, + "Runtime": func() string { + return c.Runtime(ctx) + }, + "Mode": func() string { + return c.Mode(ctx) + }, + "Address": func() string { + return c.ComponentAddress(ctx, name) + }, + "FilesDir": func() string { + return "" + }, + "PkiDir": func() string { + return "/etc/kubernetes/pki" + }, + "Kubeconfig": func() string { + return "/root/.kube/config" + }, + "Config": func() *internalversion.KwokctlConfiguration { + return conf + }, + }) + + krc, ok := slices.Find(kcp, func(krc *internalversion.KwokctlComponent) bool { + return krc.Name == name + }) + if !ok { + var componentData string + switch name { + default: + return fmt.Errorf("component %s is not exists", name) + case consts.ComponentDashboard: + componentData = component.DefaultDashboard + case consts.ComponentPrometheus: + componentData = component.DefaultPrometheus + case consts.ComponentKwokController: + componentData = component.DefaultKwokController + case consts.ComponentKubeControllerManager: + componentData = component.DefaultKubeControllerManager + case consts.ComponentKubeScheduler: + componentData = component.DefaultKubeScheduler + } + krc, err = config.UnmarshalWithType[*internalversion.KwokctlComponent](componentData) + if err != nil { + return err + } + } + + param, err := scale.NewParameters(ctx, krc.Parameters, args) + if err != nil { + return err + } + + componentData, err := renderer.ToJSON(krc.Template, param) + if err != nil { + return err + } + var component internalversion.Component + err = json.Unmarshal(componentData, &component) + if err != nil { + return err + } + component.Name = name + + for _, patch := range conf.ComponentsPatches { + if patch.Name != name { + continue + } + component = components.PatchComponent(component, patch) + } + + conf.Components = append(conf.Components, component) + + return c.SetConfig(ctx, conf) +} diff --git a/pkg/kwokctl/runtime/kind/file.go b/pkg/kwokctl/runtime/kind/file.go new file mode 100644 index 0000000000..0f3481323d --- /dev/null +++ b/pkg/kwokctl/runtime/kind/file.go @@ -0,0 +1,148 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kind + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" + "sigs.k8s.io/kwok/pkg/kwokctl/components" + "sigs.k8s.io/kwok/pkg/kwokctl/runtime" + "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/gotpl" + "sigs.k8s.io/kwok/pkg/utils/path" + "sigs.k8s.io/kwok/pkg/utils/slices" + "sigs.k8s.io/kwok/pkg/utils/yaml" +) + +func (c *Cluster) setupFiles(ctx context.Context, env *env) (err error) { + r := gotpl.NewRenderer(nil) + pathDir := c.GetWorkdirPath("files") + for index, component := range env.kwokctlConfig.Components { + for i, file := range component.Files { + var data []byte + if file.Data != "" { + data = []byte(file.Data) + } else if file.Template != "" { + d, err := r.ToText(file.Template, env.kwokctlConfig) + if err != nil { + return err + } + data = d + } + + filePath := path.Join(pathDir, component.Name, file.Path) + + err = c.MkdirAll(path.Dir(filePath)) + if err != nil { + return err + } + wc, err := c.OpenFile(filePath) + if err != nil { + return err + } + _, err = wc.Write(data) + if err != nil { + return err + } + + err = wc.Close() + if err != nil { + return err + } + + v := internalversion.Volume{ + Name: "file-" + format.String(i), + MountPath: file.Path, + HostPath: filePath, + } + + env.kwokctlConfig.Components[index].Volumes = append(env.kwokctlConfig.Components[index].Volumes, v) + } + } + + return nil +} + +func (c *Cluster) setupManifests(ctx context.Context, env *env) (err error) { + manifestsDir := c.GetWorkdirPath(runtime.ManifestsName) + for _, component := range env.kwokctlConfig.Components { + if component.Name == consts.ComponentEtcd || + component.Name == consts.ComponentKubeApiserver || + component.Name == consts.ComponentKubeControllerManager || + component.Name == consts.ComponentKubeScheduler { + continue + } + + component := component + component.Volumes = slices.Map(component.Volumes, func(v internalversion.Volume) internalversion.Volume { + v.HostPath = path.Join("/var/components", component.Name, v.MountPath) + return v + }) + + component.Volumes = append(component.Volumes, + internalversion.Volume{ + HostPath: "/etc/kubernetes/pki", + MountPath: "/etc/kubernetes/pki", + }, + internalversion.Volume{ + HostPath: "/etc/kubernetes/admin.conf", + MountPath: "/root/.kube/config", + }, + ) + + if component.Name == consts.ComponentKwokController { + component.Volumes = append(component.Volumes, + internalversion.Volume{ + HostPath: "/etc/kwok/kwok.yaml", + MountPath: "/root/.kwok/kwok.yaml", + }, + ) + } + + pod := components.ConvertToPod(component) + if component.Name == consts.ComponentKwokController { + pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{ + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }) + } + + componentPod, err := yaml.Marshal(components.ConvertToPod(component)) + if err != nil { + return fmt.Errorf("failed to marshal pod: %w", err) + } + + if err != nil { + return err + } + podFile := component.Name + ".yaml" + err = c.WriteFile(path.Join(manifestsDir, podFile), componentPod) + if err != nil { + return fmt.Errorf("failed to write %s: %w", podFile, err) + } + } + return nil +} diff --git a/pkg/kwokctl/runtime/kind/kind.go b/pkg/kwokctl/runtime/kind/kind.go index edb083f2f9..fcc1d2f02e 100644 --- a/pkg/kwokctl/runtime/kind/kind.go +++ b/pkg/kwokctl/runtime/kind/kind.go @@ -250,8 +250,6 @@ func expandHostVolumePaths(conf BuildKindConfig) (BuildKindConfig, error) { type BuildKindConfig struct { KubeApiserverPort uint32 EtcdPort uint32 - DashboardPort uint32 - PrometheusPort uint32 JaegerPort uint32 KwokControllerPort uint32 @@ -276,9 +274,12 @@ type BuildKindConfig struct { ControllerManagerExtraVolumes []internalversion.Volume Verbosity log.Level KwokControllerExtraVolumes []internalversion.Volume - PrometheusExtraVolumes []internalversion.Volume BindAddress string DisableQPSLimits bool KubeVersion version.Version + + Ports []internalversion.Port + + Components []internalversion.Component } diff --git a/pkg/kwokctl/runtime/kind/kind.yaml.tpl b/pkg/kwokctl/runtime/kind/kind.yaml.tpl index eff962f40d..a5e9204fac 100644 --- a/pkg/kwokctl/runtime/kind/kind.yaml.tpl +++ b/pkg/kwokctl/runtime/kind/kind.yaml.tpl @@ -8,18 +8,8 @@ networking: nodes: - role: control-plane - {{ if or .DashboardPort .PrometheusPort .KwokControllerPort .EtcdPort .JaegerPort}} + {{ if or .KwokControllerPort .EtcdPort .JaegerPort .Ports }} extraPortMappings: - {{ if .DashboardPort }} - - containerPort: 8000 - hostPort: {{ .DashboardPort }} - protocol: TCP - {{ end }} - {{ if .PrometheusPort }} - - containerPort: 9090 - hostPort: {{ .PrometheusPort }} - protocol: TCP - {{ end }} {{ if .JaegerPort }} - containerPort: 16686 hostPort: {{ .JaegerPort }} @@ -35,6 +25,11 @@ nodes: hostPort: {{ .EtcdPort }} protocol: TCP {{ end }} + {{ range .Ports }} + - containerPort: {{ .Port }} + hostPort: {{ .HostPort }} + protocol: {{ .Protocol }} + {{ end }} {{ end }} kubeadmConfigPatches: @@ -142,40 +137,12 @@ nodes: - hostPath: {{ .Workdir }}/pki containerPath: /etc/kubernetes/pki - {{ range .EtcdExtraVolumes }} - - hostPath: {{ .HostPath }} - containerPath: /var/components/etcd{{ .MountPath }} - readOnly: {{ .ReadOnly }} - {{ end }} - - {{ range .ApiserverExtraVolumes }} - - hostPath: {{ .HostPath }} - containerPath: /var/components/apiserver{{ .MountPath }} - readOnly: {{ .ReadOnly }} - {{ end }} - - {{ range .ControllerManagerExtraVolumes }} - - hostPath: {{ .HostPath }} - containerPath: /var/components/controller-manager{{ .MountPath }} - readOnly: {{ .ReadOnly }} - {{ end }} - - {{ range .SchedulerExtraVolumes }} - - hostPath: {{ .HostPath }} - containerPath: /var/components/scheduler{{ .MountPath }} - readOnly: {{ .ReadOnly }} - {{ end }} - - {{ range .KwokControllerExtraVolumes }} + {{ range $components := .Components }} + {{ range $components.Volumes }} - hostPath: {{ .HostPath }} - containerPath: /var/components/controller{{ .MountPath }} + containerPath: /var/components/{{ $components.Name }}{{ .MountPath }} readOnly: {{ .ReadOnly }} {{ end }} - - {{ range .PrometheusExtraVolumes }} - - hostPath: {{ .HostPath }} - containerPath: /var/components/prometheus{{ .MountPath }} - readOnly: {{ .ReadOnly }} {{ end }} {{ if .FeatureGates }} diff --git a/pkg/utils/gotpl/funcs.go b/pkg/utils/gotpl/funcs.go index 73303be445..c6b191db46 100644 --- a/pkg/utils/gotpl/funcs.go +++ b/pkg/utils/gotpl/funcs.go @@ -17,6 +17,8 @@ limitations under the License. package gotpl import ( + "runtime" + "github.com/Masterminds/sprig/v3" ) @@ -24,3 +26,16 @@ var ( // genericFuncs is generic template functions. genericFuncs = sprig.TxtFuncMap() ) + +func init() { + genericFuncs["goos"] = goos + genericFuncs["goarch"] = goarch +} + +func goos() string { + return runtime.GOOS +} + +func goarch() string { + return runtime.GOARCH +} diff --git a/pkg/utils/gotpl/renderer.go b/pkg/utils/gotpl/renderer.go index 88ef1f9f38..808d65e626 100644 --- a/pkg/utils/gotpl/renderer.go +++ b/pkg/utils/gotpl/renderer.go @@ -100,7 +100,7 @@ func (r *renderer) ToText(text string, original interface{}) ([]byte, error) { err := r.render(buf, text, original) if err != nil { - return nil, fmt.Errorf("%w: %s", err, buf.String()) + return nil, fmt.Errorf("%w: %s", err, strings.TrimRight(buf.String(), "\x00")) } return slices.Clone(buf.Bytes()), nil } @@ -112,12 +112,12 @@ func (r *renderer) ToJSON(text string, original interface{}) ([]byte, error) { err := r.render(buf, text, original) if err != nil { - return nil, fmt.Errorf("%w: %s", err, buf.String()) + return nil, fmt.Errorf("%w: %s", err, strings.TrimRight(buf.String(), "\x00")) } out, err := yaml.YAMLToJSON(buf.Bytes()) if err != nil { - return nil, fmt.Errorf("%w: %s", err, buf.String()) + return nil, fmt.Errorf("%w: %s", err, strings.TrimRight(buf.String(), "\x00")) } return out, nil } diff --git a/pkg/utils/slices/slices.go b/pkg/utils/slices/slices.go index d69f36502a..6edcb914a2 100644 --- a/pkg/utils/slices/slices.go +++ b/pkg/utils/slices/slices.go @@ -16,6 +16,11 @@ limitations under the License. package slices +import ( + "cmp" + "sort" +) + // Map returns a new slice containing the results of applying the given function func Map[S ~[]T, T any, O any](s S, f func(T) O) []O { out := make([]O, len(s)) @@ -119,3 +124,13 @@ func Reverse[S ~[]T, T any](s S) []T { } return out } + +// Sort returns a new slice containing the elements of the slice in sorted order. +func Sort[S ~[]T, T cmp.Ordered](s S) []T { + out := make([]T, len(s)) + copy(out, s) + sort.Slice(out, func(i, j int) bool { + return cmp.Less(out[i], out[j]) + }) + return out +} diff --git a/site/content/en/docs/generated/apis.md b/site/content/en/docs/generated/apis.md index e8d30a071b..285e00b04a 100644 --- a/site/content/en/docs/generated/apis.md +++ b/site/content/en/docs/generated/apis.md @@ -25,6 +25,9 @@ Resource Types: KwokConfiguration
  • +KwokctlComponent +
    • +
  • KwokctlConfiguration
  • @@ -94,6 +97,79 @@ KwokConfigurationOptions +

    +KwokctlComponent + # +

    +

    +

    KwokctlComponent holds information about the kwokctl component.

    +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    FieldDescription
    +apiVersion +string + + +config.kwok.x-k8s.io/v1alpha1 + +
    +kind +string +KwokctlComponent
    +metadata + + +Kubernetes meta/v1.ObjectMeta + + + +

    Standard list metadata. +More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata

    +Refer to the Kubernetes API documentation for the fields of the +metadata field. +
    +parameters + +encoding/json.RawMessage + + +

    Parameters is the parameters for the kwokctl component configuration.

    +
    +template + +string + + +

    Template is the template for the kwokctl component configuration.

    +

    KwokctlConfiguration # @@ -1518,6 +1594,20 @@ string +files + + +[]File + + + + +(Optional) +

    Files is a list of files that can be mounted by containers belonging to the component.

    + + + + metric @@ -1544,6 +1634,18 @@ ComponentMetric +address + +string + + + +(Optional) +

    Address is the address of the component.

    + + + + version string @@ -1814,6 +1916,71 @@ string +

    +File + # +

    +

    +Appears on: +Component +

    +

    +

    File represents a file that is accessible to the containers running in a component.

    +

    + + + + + + + + + + + + + + + + + + + + + + + + + +
    FieldDescription
    +path + +string + + +

    Path is the path of the file.

    +
    +data + +string + + +

    Data is the content of the file.

    +
    +template + +string + + +

    Template is the template of the file.

    +
    +mode + +string + + +

    Mode is the mode of the file.

    +

    HostPathType (string alias) diff --git a/site/content/en/docs/generated/kwokctl_create_cluster.md b/site/content/en/docs/generated/kwokctl_create_cluster.md index 4c3dada423..c7b396f82f 100644 --- a/site/content/en/docs/generated/kwokctl_create_cluster.md +++ b/site/content/en/docs/generated/kwokctl_create_cluster.md @@ -10,13 +10,10 @@ kwokctl create cluster [flags] ``` --controller-port uint32 Port of kwok-controller given to the host - --dashboard-image string Image of dashboard, only for docker/podman/nerdctl/kind/kind-podman runtime - '${KWOK_DASHBOARD_IMAGE_PREFIX}/dashboard:${KWOK_DASHBOARD_VERSION}' - (default "docker.io/kubernetesui/dashboard:v2.7.0") - --dashboard-port uint32 Port of dashboard given to the host --disable-kube-controller-manager Disable the kube-controller-manager --disable-kube-scheduler Disable the kube-scheduler --disable-qps-limits Disable QPS limits for components + --enable-components stringArray Enable components for the cluster --enable-crds strings List of CRDs to enable --etcd-binary string Binary of etcd, only for binary runtime (default "https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9-linux-amd64.tar.gz#etcd") --etcd-image string Image of etcd, only for docker/podman/nerdctl runtime @@ -64,11 +61,7 @@ kwokctl create cluster [flags] --kwok-controller-image string Image of kwok-controller, only for docker/podman/nerdctl/kind/kind-podman runtime '${KWOK_IMAGE_PREFIX}/kwok:${KWOK_VERSION}' (default "registry.k8s.io/kwok/kwok:v0.4.0") - --prometheus-binary string Binary of Prometheus, only for binary runtime (default "https://github.com/prometheus/prometheus/releases/download/v2.44.0/prometheus-2.44.0.linux-amd64.tar.gz#prometheus") - --prometheus-image string Image of Prometheus, only for docker/podman/nerdctl/kind/kind-podman runtime - '${KWOK_PROMETHEUS_IMAGE_PREFIX}/prometheus:${KWOK_PROMETHEUS_VERSION}' - (default "docker.io/prom/prometheus:v2.44.0") - --prometheus-port uint32 Port to expose Prometheus metrics + --param stringArray Set parameters for the cluster, format: --param=.key1=value1 --param=.key2=value2 --quiet-pull Pull without printing progress information --runtime string Runtime of the cluster (binary or docker or kind or kind-podman or nerdctl or podman) --secure-port The apiserver port on which to serve HTTPS with authentication and authorization, is not available before Kubernetes 1.13.0 (default true) diff --git a/test/kwokctl/testdata/binary/create_cluster.txt b/test/kwokctl/testdata/binary/create_cluster.txt index 5fdd726c3b..fdb78e77ce 100644 --- a/test/kwokctl/testdata/binary/create_cluster.txt +++ b/test/kwokctl/testdata/binary/create_cluster.txt @@ -2,14 +2,16 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pids mkdir -p /workdir/clusters//logs -# Download https://dl.k8s.io/release/v1.28.0/bin///kube-apiserver to /workdir/clusters//bin/kube-apiserver -# Download https://dl.k8s.io/release/v1.28.0/bin///kube-controller-manager to /workdir/clusters//bin/kube-controller-manager -# Download https://dl.k8s.io/release/v1.28.0/bin///kube-scheduler to /workdir/clusters//bin/kube-scheduler -# Download /bin///kwok to /workdir/clusters//bin/kwok-controller -# Download https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9--. and extract etcd to /workdir/clusters//bin/etcd mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd +# Download https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9--. and extract etcd to /workdir/clusters//bin/etcd +# Download https://dl.k8s.io/release/v1.28.0/bin///kube-apiserver to /workdir/clusters//bin/kube-apiserver +# Download /bin///kwok to /workdir/clusters//bin/kwok-controller +# Download https://dl.k8s.io/release/v1.28.0/bin///kube-controller-manager to /workdir/clusters//bin/kube-controller-manager +# Download https://dl.k8s.io/release/v1.28.0/bin///kube-scheduler to /workdir/clusters//bin/kube-scheduler +# Download /workdir/clusters//bin/etcd to /workdir/clusters//bin/etcd +# Download /workdir/clusters//bin/kube-apiserver to /workdir/clusters//bin/kube-apiserver cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: @@ -37,9 +39,9 @@ cd /workdir/clusters/ && etcd --name=node0 --auto-compac echo $! >/workdir/clusters//pids/etcd.pid cd /workdir/clusters/ && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local >/workdir/clusters//logs/kube-apiserver.log 2>&1 & echo $! >/workdir/clusters//pids/kube-apiserver.pid -cd /workdir/clusters/ && kube-controller-manager --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & +cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=~/.kwok/kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-name=127.0.0.1 --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 >/workdir/clusters//logs/kwok-controller.log 2>&1 & +echo $! >/workdir/clusters//pids/kwok-controller.pid +cd /workdir/clusters/ && kube-controller-manager --node-monitor-period=600000ms --node-monitor-grace-period=3600000ms --kubeconfig=/workdir/clusters//kubeconfig.yaml --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & echo $! >/workdir/clusters//pids/kube-controller-manager.pid -cd /workdir/clusters/ && kube-scheduler --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & +cd /workdir/clusters/ && kube-scheduler --kubeconfig=/workdir/clusters//kubeconfig.yaml --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics >/workdir/clusters//logs/kube-scheduler.log 2>&1 & echo $! >/workdir/clusters//pids/kube-scheduler.pid -cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=1200 >/workdir/clusters//logs/kwok-controller.log 2>&1 & -echo $! >/workdir/clusters//pids/kwok-controller.pid diff --git a/test/kwokctl/testdata/binary/create_cluster_with_extra.txt b/test/kwokctl/testdata/binary/create_cluster_with_extra.txt index 8962bdcad5..dc7dd2a752 100644 --- a/test/kwokctl/testdata/binary/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/binary/create_cluster_with_extra.txt @@ -2,16 +2,13 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pids mkdir -p /workdir/clusters//logs -# Download https://dl.k8s.io/release/v1.28.0/bin///kube-apiserver to /workdir/clusters//bin/kube-apiserver -# Download https://dl.k8s.io/release/v1.28.0/bin///kube-controller-manager to /workdir/clusters//bin/kube-controller-manager -# Download https://dl.k8s.io/release/v1.28.0/bin///kube-scheduler to /workdir/clusters//bin/kube-scheduler -# Download /bin///kwok to /workdir/clusters//bin/kwok-controller -# Download https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9--. and extract etcd to /workdir/clusters//bin/etcd -# Download https://github.com/prometheus/prometheus/releases/download/v2.44.0/prometheus-2.44.0.-. and extract prometheus to /workdir/clusters//bin/prometheus mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd -cat </workdir/clusters//prometheus.yaml +# Download https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9--. and extract etcd to /workdir/clusters//bin/etcd +# Download https://dl.k8s.io/release/v1.28.0/bin///kube-apiserver to /workdir/clusters//bin/kube-apiserver +mkdir -p /workdir/clusters//files/prometheus/etc/prometheus +cat </workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml global: scrape_interval: 15s scrape_timeout: 10s @@ -26,7 +23,7 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "etcd" +- job_name: "kwok-controller" scheme: http honor_timestamps: true metrics_path: /metrics @@ -34,8 +31,8 @@ scrape_configs: enable_http2: true static_configs: - targets: - - 127.0.0.1:32765 -- job_name: "kube-apiserver" + - 127.0.0.1:10247 +- job_name: "kube-controller-manager" scheme: https honor_timestamps: true metrics_path: /metrics @@ -47,8 +44,8 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:32764 -- job_name: "kube-controller-manager" + - 127.0.0.1:10257 +- job_name: "kube-scheduler" scheme: https honor_timestamps: true metrics_path: /metrics @@ -60,24 +57,17 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:32762 -- job_name: "kube-scheduler" - scheme: https + - 127.0.0.1:10259 +- job_name: "prometheus" + scheme: http honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true - tls_config: - cert_file: "/workdir/clusters//pki/admin.crt" - key_file: "/workdir/clusters//pki/admin.key" - insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:32761 -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://127.0.0.1:32763/discovery/prometheus -- job_name: "kwok-controller" + - 127.0.0.1:9090 +- job_name: "etcd" scheme: http honor_timestamps: true metrics_path: /metrics @@ -85,17 +75,27 @@ scrape_configs: enable_http2: true static_configs: - targets: - - 127.0.0.1:32763 -- job_name: "prometheus" - scheme: http + - 127.0.0.1:32765 +- job_name: "kube-apiserver" + scheme: https honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true + tls_config: + cert_file: "/workdir/clusters//pki/admin.crt" + key_file: "/workdir/clusters//pki/admin.key" + insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:9090 + - 127.0.0.1:32764 EOF +# Download /bin///kwok to /workdir/clusters//bin/kwok-controller +# Download https://dl.k8s.io/release/v1.28.0/bin///kube-controller-manager to /workdir/clusters//bin/kube-controller-manager +# Download https://dl.k8s.io/release/v1.28.0/bin///kube-scheduler to /workdir/clusters//bin/kube-scheduler +# Download https://github.com/prometheus/prometheus/releases/download/v2.44.0//prometheus-2.44.0.-. and extract prometheus to /workdir/clusters//bin/prometheus +# Download /workdir/clusters//bin/etcd to /workdir/clusters//bin/etcd +# Download /workdir/clusters//bin/kube-apiserver to /workdir/clusters//bin/kube-apiserver cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: @@ -123,11 +123,11 @@ cd /workdir/clusters/ && TEST_KEY=TEST_VALUE etcd --name echo $! >/workdir/clusters//pids/etcd.pid cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-apiserver --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local >/workdir/clusters//logs/kube-apiserver.log 2>&1 & echo $! >/workdir/clusters//pids/kube-apiserver.pid -cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-controller-manager --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & +cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=~/.kwok/kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-name=127.0.0.1 --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 >/workdir/clusters//logs/kwok-controller.log 2>&1 & +echo $! >/workdir/clusters//pids/kwok-controller.pid +cd /workdir/clusters/ && kube-controller-manager --node-monitor-period=600000ms --node-monitor-grace-period=3600000ms --kubeconfig=/workdir/clusters//kubeconfig.yaml --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & echo $! >/workdir/clusters//pids/kube-controller-manager.pid -cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-scheduler --v=5 --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & +cd /workdir/clusters/ && kube-scheduler --kubeconfig=/workdir/clusters//kubeconfig.yaml --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics >/workdir/clusters//logs/kube-scheduler.log 2>&1 & echo $! >/workdir/clusters//pids/kube-scheduler.pid -cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kwok-controller --manage-all-nodes=true --v=-4 --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=1200 >/workdir/clusters//logs/kwok-controller.log 2>&1 & -echo $! >/workdir/clusters//pids/kwok-controller.pid -cd /workdir/clusters/ && TEST_KEY=TEST_VALUE prometheus --log.level=debug --config.file=/workdir/clusters//prometheus.yaml --web.listen-address=0.0.0.0:9090 >/workdir/clusters//logs/prometheus.log 2>&1 & +cd /workdir/clusters/ && prometheus --config.file=/workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 >/workdir/clusters//logs/prometheus.log 2>&1 & echo $! >/workdir/clusters//pids/prometheus.pid diff --git a/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt index 20a5ffeeec..ac9ed3963c 100644 --- a/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt @@ -2,30 +2,22 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pids mkdir -p /workdir/clusters//logs -# Download https://dl.k8s.io/release/v1.28.0/bin///kube-apiserver to /workdir/clusters//bin/kube-apiserver -# Download https://dl.k8s.io/release/v1.28.0/bin///kube-controller-manager to /workdir/clusters//bin/kube-controller-manager -# Download https://dl.k8s.io/release/v1.28.0/bin///kube-scheduler to /workdir/clusters//bin/kube-scheduler -# Download /bin///kwok to /workdir/clusters//bin/kwok-controller -# Download https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9--. and extract etcd to /workdir/clusters//bin/etcd -# Download https://github.com/prometheus/prometheus/releases/download/v2.44.0/prometheus-2.44.0.-. and extract prometheus to /workdir/clusters//bin/prometheus -# Download https://github.com/jaegertracing/jaeger/releases/download/v1.45.0/jaeger-1.45.0--. and extract jaeger-all-in-one to /workdir/clusters//bin/jaeger-all-in-one mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki touch /workdir/clusters//logs/audit.log cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml mkdir -p /workdir/clusters//etcd +# Download https://github.com/etcd-io/etcd/releases/download/v3.5.9/etcd-v3.5.9--. and extract etcd to /workdir/clusters//bin/etcd +# Download https://dl.k8s.io/release/v1.28.0/bin///kube-apiserver to /workdir/clusters//bin/kube-apiserver cat </workdir/clusters//apiserver-tracing-config.yaml apiVersion: apiserver.config.k8s.io/v1alpha1 kind: TracingConfiguration endpoint: 127.0.0.1:32762 samplingRatePerMillion: 1000000 EOF -cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml -cat <>/workdir/clusters//scheduler.yaml -clientConnection: - kubeconfig: "/workdir/clusters//kubeconfig.yaml" -EOF -cat </workdir/clusters//prometheus.yaml +# Download https://github.com/jaegertracing/jaeger/releases/download/v1.45.0/jaeger-1.45.0--. and extract jaeger-all-in-one to /workdir/clusters//bin/jaeger +mkdir -p /workdir/clusters//files/prometheus/etc/prometheus +cat </workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml global: scrape_interval: 15s scrape_timeout: 10s @@ -40,7 +32,7 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "etcd" +- job_name: "kwok-controller" scheme: http honor_timestamps: true metrics_path: /metrics @@ -48,8 +40,8 @@ scrape_configs: enable_http2: true static_configs: - targets: - - 127.0.0.1:32765 -- job_name: "kube-apiserver" + - 127.0.0.1:10247 +- job_name: "kube-controller-manager" scheme: https honor_timestamps: true metrics_path: /metrics @@ -61,8 +53,8 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:32764 -- job_name: "kube-controller-manager" + - 127.0.0.1:10257 +- job_name: "kube-scheduler" scheme: https honor_timestamps: true metrics_path: /metrics @@ -74,24 +66,17 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:32761 -- job_name: "kube-scheduler" - scheme: https + - 127.0.0.1:10259 +- job_name: "prometheus" + scheme: http honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true - tls_config: - cert_file: "/workdir/clusters//pki/admin.crt" - key_file: "/workdir/clusters//pki/admin.key" - insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:32760 -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://127.0.0.1:32763/discovery/prometheus -- job_name: "kwok-controller" + - 127.0.0.1:9090 +- job_name: "etcd" scheme: http honor_timestamps: true metrics_path: /metrics @@ -99,17 +84,28 @@ scrape_configs: enable_http2: true static_configs: - targets: - - 127.0.0.1:32763 -- job_name: "prometheus" - scheme: http + - 127.0.0.1:32765 +- job_name: "kube-apiserver" + scheme: https honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true + tls_config: + cert_file: "/workdir/clusters//pki/admin.crt" + key_file: "/workdir/clusters//pki/admin.key" + insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:9090 + - 127.0.0.1:32764 EOF +# Download /bin///kwok to /workdir/clusters//bin/kwok-controller +# Download https://dl.k8s.io/release/v1.28.0/bin///kube-controller-manager to /workdir/clusters//bin/kube-controller-manager +# Download https://dl.k8s.io/release/v1.28.0/bin///kube-scheduler to /workdir/clusters//bin/kube-scheduler +# Download https://github.com/prometheus/prometheus/releases/download/v2.44.0//prometheus-2.44.0.-. and extract prometheus to /workdir/clusters//bin/prometheus +# Download /workdir/clusters//bin/etcd to /workdir/clusters//bin/etcd +# Download /workdir/clusters//bin/kube-apiserver to /workdir/clusters//bin/kube-apiserver +# Download /workdir/clusters//bin/jaeger to /workdir/clusters//bin/jaeger cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: @@ -135,15 +131,17 @@ EOF # Add context kwok- to ~/.kube/config cd /workdir/clusters/ && etcd --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/workdir/clusters//etcd --initial-advertise-peer-urls=http://0.0.0.0:32766 --listen-peer-urls=http://0.0.0.0:32766 --advertise-client-urls=http://0.0.0.0:32765 --listen-client-urls=http://0.0.0.0:32765 --initial-cluster=node0=http://0.0.0.0:32766 --log-level=debug >/workdir/clusters//logs/etcd.log 2>&1 & echo $! >/workdir/clusters//pids/etcd.pid -cd /workdir/clusters/ && jaeger-all-in-one --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --collector.otlp.grpc.host-port=127.0.0.1:32762 --log-level=debug >/workdir/clusters//logs/jaeger-all-in-one.log 2>&1 & -echo $! >/workdir/clusters//pids/jaeger-all-in-one.pid +cd /workdir/clusters/ && jaeger --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --collector.otlp.grpc.host-port=127.0.0.1:32762 --log-level=debug >/workdir/clusters//logs/jaeger.log 2>&1 & +echo $! >/workdir/clusters//pids/jaeger.pid cd /workdir/clusters/ && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --audit-policy-file=/workdir/clusters//audit.yaml --audit-log-path=/workdir/clusters//logs/audit.log --tracing-config-file=/workdir/clusters//apiserver-tracing-config.yaml --v=4 >/workdir/clusters//logs/kube-apiserver.log 2>&1 & echo $! >/workdir/clusters//pids/kube-apiserver.pid -cd /workdir/clusters/ && kube-controller-manager --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & +cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=~/.kwok/kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-name=127.0.0.1 --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 >/workdir/clusters//logs/kwok-controller.log 2>&1 & +echo $! >/workdir/clusters//pids/kwok-controller.pid +cd /workdir/clusters/ && kube-controller-manager --node-monitor-period=600000ms --node-monitor-grace-period=3600000ms --kubeconfig=/workdir/clusters//kubeconfig.yaml --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & echo $! >/workdir/clusters//pids/kube-controller-manager.pid -cd /workdir/clusters/ && kube-scheduler --config=/workdir/clusters//scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32760 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & +cd /workdir/clusters/ && kube-scheduler --kubeconfig=/workdir/clusters//kubeconfig.yaml --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics >/workdir/clusters//logs/kube-scheduler.log 2>&1 & echo $! >/workdir/clusters//pids/kube-scheduler.pid -cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=1200 --v=DEBUG >/workdir/clusters//logs/kwok-controller.log 2>&1 & -echo $! >/workdir/clusters//pids/kwok-controller.pid -cd /workdir/clusters/ && prometheus --config.file=/workdir/clusters//prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug >/workdir/clusters//logs/prometheus.log 2>&1 & +cd /workdir/clusters/ && dashboard --insecure-bind-address=0.0.0.0 --insecure-port=8000 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=/workdir/clusters//kubeconfig.yaml >/workdir/clusters//logs/dashboard.log 2>&1 & +echo $! >/workdir/clusters//pids/dashboard.pid +cd /workdir/clusters/ && prometheus --config.file=/workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 >/workdir/clusters//logs/prometheus.log 2>&1 & echo $! >/workdir/clusters//pids/prometheus.pid diff --git a/test/kwokctl/testdata/docker/create_cluster.txt b/test/kwokctl/testdata/docker/create_cluster.txt index 08b98cb9c3..99e21ca053 100644 --- a/test/kwokctl/testdata/docker/create_cluster.txt +++ b/test/kwokctl/testdata/docker/create_cluster.txt @@ -3,9 +3,9 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd +docker pull localhost/kwok:test docker pull registry.k8s.io/etcd:3.5.9-0 docker pull registry.k8s.io/kube-apiserver:v1.28.0 -docker pull localhost/kwok:test docker pull registry.k8s.io/kube-controller-manager:v1.28.0 docker pull registry.k8s.io/kube-scheduler:v1.28.0 cat </workdir/clusters//kubeconfig.yaml @@ -54,12 +54,12 @@ EOF docker network create kwok- --label=com.docker.compose.project=kwok- docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local -docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --kube-api-qps=5000 --kube-api-burst=10000 -docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +docker create --name=kwok--kwok-controller --pull=never --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=600000ms --node-monitor-grace-period=3600000ms --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics # Add context kwok- to ~/.kube/config docker start kwok--etcd docker start kwok--kube-apiserver +docker start kwok--kwok-controller docker start kwok--kube-controller-manager docker start kwok--kube-scheduler -docker start kwok--kwok-controller diff --git a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt index 8353e51fc3..3a9244bb13 100644 --- a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt @@ -3,7 +3,8 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd -cat </workdir/clusters//prometheus.yaml +mkdir -p /workdir/clusters//files/prometheus/etc/prometheus +cat </workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml global: scrape_interval: 15s scrape_timeout: 10s @@ -18,7 +19,7 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "etcd" +- job_name: "kwok-controller" scheme: http honor_timestamps: true metrics_path: /metrics @@ -26,8 +27,8 @@ scrape_configs: enable_http2: true static_configs: - targets: - - kwok--etcd:2379 -- job_name: "kube-apiserver" + - kwok--kwok-controller:10247 +- job_name: "kube-controller-manager" scheme: https honor_timestamps: true metrics_path: /metrics @@ -39,8 +40,8 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - kwok--kube-apiserver:6443 -- job_name: "kube-controller-manager" + - kwok--kube-controller-manager:10257 +- job_name: "kube-scheduler" scheme: https honor_timestamps: true metrics_path: /metrics @@ -52,24 +53,17 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - kwok--kube-controller-manager:10257 -- job_name: "kube-scheduler" - scheme: https + - kwok--kube-scheduler:10259 +- job_name: "prometheus" + scheme: http honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true static_configs: - targets: - - kwok--kube-scheduler:10259 -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://kwok--kwok-controller:10247/discovery/prometheus -- job_name: "kwok-controller" + - 127.0.0.1:9090 +- job_name: "etcd" scheme: http honor_timestamps: true metrics_path: /metrics @@ -77,24 +71,27 @@ scrape_configs: enable_http2: true static_configs: - targets: - - kwok--kwok-controller:10247 -- job_name: "prometheus" - scheme: http + - kwok--etcd:2379 +- job_name: "kube-apiserver" + scheme: https honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:9090 + - kwok--kube-apiserver:6443 EOF -chmod 0644 /workdir/clusters//prometheus.yaml +docker pull docker.io/prom/prometheus:v2.44.0 +docker pull localhost/kwok:test docker pull registry.k8s.io/etcd:3.5.9-0 docker pull registry.k8s.io/kube-apiserver:v1.28.0 -docker pull localhost/kwok:test docker pull registry.k8s.io/kube-controller-manager:v1.28.0 docker pull registry.k8s.io/kube-scheduler:v1.28.0 -docker pull docker.io/prom/prometheus:v2.44.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: @@ -141,14 +138,14 @@ EOF docker network create kwok- --label=com.docker.compose.project=kwok- docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --log-level=debug --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local -docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller-manager:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 -docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/scheduler:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --v=5 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 -docker create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --link=kwok--etcd --link=kwok--kube-apiserver --link=kwok--kube-controller-manager --link=kwok--kube-scheduler --link=kwok--kwok-controller --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/extras/prometheus:/extras/tmp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.44.0 --log.level=debug --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 +docker create --name=kwok--kwok-controller --pull=never --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=600000ms --node-monitor-grace-period=3600000ms --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +docker create --name=kwok--prometheus --pull=never --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 # Add context kwok- to ~/.kube/config docker start kwok--etcd docker start kwok--kube-apiserver +docker start kwok--kwok-controller docker start kwok--kube-controller-manager docker start kwok--kube-scheduler -docker start kwok--kwok-controller docker start kwok--prometheus diff --git a/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt index 226e5e738e..06dba3e3d3 100644 --- a/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt @@ -12,12 +12,8 @@ kind: TracingConfiguration endpoint: kwok--jaeger:4317 samplingRatePerMillion: 1000000 EOF -cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml -cat <>/workdir/clusters//scheduler.yaml -clientConnection: - kubeconfig: "~/.kube/config" -EOF -cat </workdir/clusters//prometheus.yaml +mkdir -p /workdir/clusters//files/prometheus/etc/prometheus +cat </workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml global: scrape_interval: 15s scrape_timeout: 10s @@ -32,7 +28,7 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "etcd" +- job_name: "kwok-controller" scheme: http honor_timestamps: true metrics_path: /metrics @@ -40,8 +36,8 @@ scrape_configs: enable_http2: true static_configs: - targets: - - kwok--etcd:2379 -- job_name: "kube-apiserver" + - kwok--kwok-controller:10247 +- job_name: "kube-controller-manager" scheme: https honor_timestamps: true metrics_path: /metrics @@ -53,8 +49,8 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - kwok--kube-apiserver:6443 -- job_name: "kube-controller-manager" + - kwok--kube-controller-manager:10257 +- job_name: "kube-scheduler" scheme: https honor_timestamps: true metrics_path: /metrics @@ -66,24 +62,17 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - kwok--kube-controller-manager:10257 -- job_name: "kube-scheduler" - scheme: https + - kwok--kube-scheduler:10259 +- job_name: "prometheus" + scheme: http honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true static_configs: - targets: - - kwok--kube-scheduler:10259 -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://kwok--kwok-controller:10247/discovery/prometheus -- job_name: "kwok-controller" + - 127.0.0.1:9090 +- job_name: "etcd" scheme: http honor_timestamps: true metrics_path: /metrics @@ -91,26 +80,29 @@ scrape_configs: enable_http2: true static_configs: - targets: - - kwok--kwok-controller:10247 -- job_name: "prometheus" - scheme: http + - kwok--etcd:2379 +- job_name: "kube-apiserver" + scheme: https honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:9090 + - kwok--kube-apiserver:6443 EOF -chmod 0644 /workdir/clusters//prometheus.yaml +docker pull docker.io/jaegertracing/all-in-one:1.45.0 +docker pull docker.io/kubernetesui/dashboard:v2.7.0 +docker pull docker.io/prom/prometheus:v2.44.0 +docker pull localhost/kwok:test docker pull registry.k8s.io/etcd:3.5.9-0 docker pull registry.k8s.io/kube-apiserver:v1.28.0 -docker pull localhost/kwok:test docker pull registry.k8s.io/kube-controller-manager:v1.28.0 docker pull registry.k8s.io/kube-scheduler:v1.28.0 -docker pull docker.io/kubernetesui/dashboard:v2.7.0 -docker pull docker.io/prom/prometheus:v2.44.0 -docker pull docker.io/jaegertracing/all-in-one:1.45.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: @@ -158,17 +150,17 @@ docker network create kwok- --label=com.docker.compose.project=kwo docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug docker create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.45.0 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --log-level=debug docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --link=kwok--jaeger --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 -docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 -docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.28.0 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 -docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 --v=DEBUG -docker create --name=kwok--dashboard --pull=never --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config --insecure-port=8000 -docker create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --link=kwok--etcd --link=kwok--kube-apiserver --link=kwok--kube-controller-manager --link=kwok--kube-scheduler --link=kwok--kwok-controller --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug +docker create --name=kwok--kwok-controller --pull=never --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=600000ms --node-monitor-grace-period=3600000ms --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +docker create --name=kwok--dashboard --pull=never --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --insecure-port=8000 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config +docker create --name=kwok--prometheus --pull=never --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 # Add context kwok- to ~/.kube/config docker start kwok--etcd docker start kwok--jaeger docker start kwok--kube-apiserver +docker start kwok--kwok-controller docker start kwok--kube-controller-manager docker start kwok--kube-scheduler -docker start kwok--kwok-controller docker start kwok--dashboard docker start kwok--prometheus diff --git a/test/kwokctl/testdata/kind-podman/create_cluster.txt b/test/kwokctl/testdata/kind-podman/create_cluster.txt index b72bb359b5..719e7080e0 100644 --- a/test/kwokctl/testdata/kind-podman/create_cluster.txt +++ b/test/kwokctl/testdata/kind-podman/create_cluster.txt @@ -3,43 +3,7 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//pki/etcd -mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//manifests -cat </workdir/clusters//kind.yaml -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -networking: -nodes: -- role: control-plane - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - "max-requests-inflight": "0" - "max-mutating-requests-inflight": "0" - "enable-priority-and-fairness": "false" - - | - kind: ClusterConfiguration - controllerManager: - extraArgs: - "kube-api-qps": "5000" - "kube-api-burst": "10000" - - | - kind: ClusterConfiguration - scheduler: - extraArgs: - "kube-api-qps": "5000" - "kube-api-burst": "10000" - # mount the local file on the control plane - extraMounts: - - hostPath: /workdir/clusters/ - containerPath: /etc/kwok/ - - hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/manifests - - hostPath: /workdir/clusters//pki - containerPath: /etc/kubernetes/pki -EOF cat </workdir/clusters//manifests/kwok-controller.yaml apiVersion: v1 kind: Pod @@ -60,34 +24,18 @@ spec: - --node-name=kwok-controller.kube-system.svc - --node-port=10247 - --server-address=0.0.0.0:10247 - - --node-lease-duration-seconds=40 - command: - - kwok - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP + - --node-lease-duration-seconds=1200 image: localhost/kwok:test imagePullPolicy: Never name: kwok-controller resources: {} volumeMounts: - - mountPath: ~/.kube/config + - mountPath: /etc/kubernetes/pki name: volume-0 - readOnly: true - - mountPath: /etc/kubernetes/pki/ca.crt + - mountPath: ~/.kube/config name: volume-1 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-2 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-3 - readOnly: true - mountPath: ~/.kwok/kwok.yaml - name: volume-4 - readOnly: true + name: volume-2 hostNetwork: true restartPolicy: Always securityContext: @@ -95,22 +43,52 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /etc/kubernetes/admin.conf + path: /etc/kubernetes/pki name: volume-0 - hostPath: - path: /etc/kubernetes/pki/ca.crt + path: /etc/kubernetes/admin.conf name: volume-1 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-2 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-3 - hostPath: path: /etc/kwok/kwok.yaml - name: volume-4 + name: volume-2 status: {} EOF +mkdir -p /workdir/clusters//pki +cat </workdir/clusters//kind.yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: +nodes: +- role: control-plane + kubeadmConfigPatches: + - | + kind: ClusterConfiguration + apiServer: + extraArgs: + "max-requests-inflight": "0" + "max-mutating-requests-inflight": "0" + "enable-priority-and-fairness": "false" + - | + kind: ClusterConfiguration + controllerManager: + extraArgs: + "kube-api-qps": "5000" + "kube-api-burst": "10000" + - | + kind: ClusterConfiguration + scheduler: + extraArgs: + "kube-api-qps": "5000" + "kube-api-burst": "10000" + # mount the local file on the control plane + extraMounts: + - hostPath: /workdir/clusters/ + containerPath: /etc/kwok/ + - hostPath: /workdir/clusters//manifests + containerPath: /etc/kubernetes/manifests + - hostPath: /workdir/clusters//pki + containerPath: /etc/kubernetes/pki +EOF podman pull docker.io/kindest/node:v1.28.0 podman pull localhost/kwok:test # Save cluster config to /workdir/clusters//kwok.yaml diff --git a/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt b/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt index 66d4a40393..1bae1fdfd6 100644 --- a/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt @@ -3,100 +3,93 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//pki/etcd -mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//manifests -cat </workdir/clusters//kind.yaml -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -networking: -nodes: -- role: control-plane - extraPortMappings: - - containerPort: 9090 - hostPort: 9090 - protocol: TCP - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - etcd: - local: - extraArgs: - "log-level": "debug" - extraVolumes: - - name: tmp-etcd - hostPath: /var/components/etcd/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - "v": "5" - "max-requests-inflight": "0" - "max-mutating-requests-inflight": "0" - "enable-priority-and-fairness": "false" - extraVolumes: - - name: tmp-apiserver - hostPath: /var/components/apiserver/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - - | - kind: ClusterConfiguration - controllerManager: - extraArgs: - "v": "5" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - extraVolumes: - - name: tmp-controller-manager - hostPath: /var/components/controller-manager/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - - | - kind: ClusterConfiguration - scheduler: - extraArgs: - "v": "5" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - extraVolumes: - - name: tmp-scheduler - hostPath: /var/components/scheduler/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - # mount the local file on the control plane - extraMounts: - - hostPath: /workdir/clusters/ - containerPath: /etc/kwok/ - - hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/manifests - - hostPath: /workdir/clusters//pki - containerPath: /etc/kubernetes/pki - - hostPath: /extras/etcd - containerPath: /var/components/etcd/extras/tmp - readOnly: false - - hostPath: /extras/apiserver - containerPath: /var/components/apiserver/extras/tmp - readOnly: false - - hostPath: /extras/controller-manager - containerPath: /var/components/controller-manager/extras/tmp - readOnly: false - - hostPath: /extras/scheduler - containerPath: /var/components/scheduler/extras/tmp - readOnly: false - - hostPath: /extras/controller - containerPath: /var/components/controller/extras/tmp - readOnly: false - - hostPath: ./extras/prometheus - containerPath: /var/components/prometheus/extras/tmp - readOnly: false - - hostPath: /workdir/clusters//prometheus.yaml - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml - readOnly: false +mkdir -p /workdir/clusters//files/prometheus/etc/prometheus +cat </workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:10247 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10259 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +- job_name: "etcd" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:6443 EOF cat </workdir/clusters//manifests/kwok-controller.yaml apiVersion: v1 @@ -110,7 +103,6 @@ spec: - args: - --manage-all-nodes=false - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake - - --v=-4 - --kubeconfig=~/.kube/config - --config=~/.kwok/kwok.yaml - --tls-cert-file=/etc/kubernetes/pki/admin.crt @@ -119,38 +111,18 @@ spec: - --node-name=kwok-controller.kube-system.svc - --node-port=10247 - --server-address=0.0.0.0:10247 - - --node-lease-duration-seconds=40 - command: - - kwok - env: - - name: TEST_KEY - value: TEST_VALUE - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP + - --node-lease-duration-seconds=1200 image: localhost/kwok:test imagePullPolicy: Never name: kwok-controller resources: {} volumeMounts: - - mountPath: /extras/tmp - name: tmp-controller + - mountPath: /etc/kubernetes/pki + name: volume-0 - mountPath: ~/.kube/config name: volume-1 - readOnly: true - - mountPath: /etc/kubernetes/pki/ca.crt - name: volume-2 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-3 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-4 - readOnly: true - mountPath: ~/.kwok/kwok.yaml - name: volume-5 - readOnly: true + name: volume-2 hostNetwork: true restartPolicy: Always securityContext: @@ -158,24 +130,14 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /extras/controller - type: DirectoryOrCreate - name: tmp-controller + path: /etc/kubernetes/pki + name: volume-0 - hostPath: path: /etc/kubernetes/admin.conf name: volume-1 - - hostPath: - path: /etc/kubernetes/pki/ca.crt - name: volume-2 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-3 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-4 - hostPath: path: /etc/kwok/kwok.yaml - name: volume-5 + name: volume-2 status: {} EOF cat </workdir/clusters//manifests/prometheus.yaml @@ -188,39 +150,24 @@ metadata: spec: containers: - args: - - --log.level=debug - --config.file=/etc/prometheus/prometheus.yaml - --web.listen-address=0.0.0.0:9090 - command: - - prometheus - env: - - name: TEST_KEY - value: TEST_VALUE image: docker.io/prom/prometheus:v2.44.0 imagePullPolicy: Never name: prometheus ports: - containerPort: 9090 hostPort: 9090 + name: http + protocol: TCP resources: {} volumeMounts: - - mountPath: /extras/tmp - name: tmp-prometheus - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt + - mountPath: /etc/prometheus/prometheus.yaml + name: file-0 + - mountPath: /etc/kubernetes/pki name: volume-1 - readOnly: true - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key + - mountPath: ~/.kube/config name: volume-2 - readOnly: true - - mountPath: /etc/prometheus/prometheus.yaml - name: volume-3 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-4 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-5 - readOnly: true hostNetwork: true restartPolicy: Always securityContext: @@ -228,128 +175,105 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /extras/prometheus - type: DirectoryOrCreate - name: tmp-prometheus + path: /var/components/prometheus/etc/prometheus/prometheus.yaml + name: file-0 - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.crt + path: /etc/kubernetes/pki name: volume-1 - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.key + path: /etc/kubernetes/admin.conf name: volume-2 - - hostPath: - path: /var/components/prometheus/etc/prometheus/prometheus.yaml - name: volume-3 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-4 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-5 status: {} EOF -cat </workdir/clusters//prometheus.yaml -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - follow_redirects: true - enable_http2: true - scheme: http - timeout: 10s - api_version: v2 - static_configs: - - targets: [] -scrape_configs: -- job_name: "etcd" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" - key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:2379 -- job_name: "kube-apiserver" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:6443 -- job_name: "kube-controller-manager" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:10257 -- job_name: "kube-scheduler" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:10259 -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://127.0.0.1:10247/discovery/prometheus -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - 127.0.0.1:10247 -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - 127.0.0.1:9090 +mkdir -p /workdir/clusters//pki +cat </workdir/clusters//kind.yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 9090 + hostPort: 9090 + protocol: TCP + kubeadmConfigPatches: + - | + kind: ClusterConfiguration + etcd: + local: + extraArgs: + "log-level": "debug" + extraVolumes: + - name: tmp-etcd + hostPath: /var/components/etcd/extras/tmp + mountPath: /extras/tmp + readOnly: false + pathType: DirectoryOrCreate + - | + kind: ClusterConfiguration + apiServer: + extraArgs: + "v": "5" + "max-requests-inflight": "0" + "max-mutating-requests-inflight": "0" + "enable-priority-and-fairness": "false" + extraVolumes: + - name: tmp-apiserver + hostPath: /var/components/apiserver/extras/tmp + mountPath: /extras/tmp + readOnly: false + pathType: DirectoryOrCreate + - | + kind: ClusterConfiguration + controllerManager: + extraArgs: + "v": "5" + "kube-api-qps": "5000" + "kube-api-burst": "10000" + extraVolumes: + - name: tmp-controller-manager + hostPath: /var/components/controller-manager/extras/tmp + mountPath: /extras/tmp + readOnly: false + pathType: DirectoryOrCreate + - | + kind: ClusterConfiguration + scheduler: + extraArgs: + "v": "5" + "kube-api-qps": "5000" + "kube-api-burst": "10000" + extraVolumes: + - name: tmp-scheduler + hostPath: /var/components/scheduler/extras/tmp + mountPath: /extras/tmp + readOnly: false + pathType: DirectoryOrCreate + # mount the local file on the control plane + extraMounts: + - hostPath: /workdir/clusters/ + containerPath: /etc/kwok/ + - hostPath: /workdir/clusters//manifests + containerPath: /etc/kubernetes/manifests + - hostPath: /workdir/clusters//pki + containerPath: /etc/kubernetes/pki + - hostPath: /workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml + containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml + readOnly: false EOF podman pull docker.io/kindest/node:v1.28.0 -podman pull localhost/kwok:test podman pull docker.io/prom/prometheus:v2.44.0 +podman pull localhost/kwok:test # Save cluster config to /workdir/clusters//kwok.yaml KIND_EXPERIMENTAL_PROVIDER=podman kind create cluster --config /workdir/clusters//kind.yaml --name kwok- --image docker.io/kindest/node:v1.28.0 --wait 29m -mkdir -p /workdir/cache/image-archive/localhost/kwok -podman save localhost/kwok:test -o /workdir/cache/image-archive/localhost/kwok/test.tar -KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/localhost/kwok/test.tar --name kwok- -rm /workdir/cache/image-archive/localhost/kwok/test.tar mkdir -p /workdir/cache/image-archive/docker.io/prom/prometheus podman save docker.io/prom/prometheus:v2.44.0 -o /workdir/cache/image-archive/docker.io/prom/prometheus/v2.44.0.tar KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/prom/prometheus/v2.44.0.tar --name kwok- rm /workdir/cache/image-archive/docker.io/prom/prometheus/v2.44.0.tar +mkdir -p /workdir/cache/image-archive/localhost/kwok +podman save localhost/kwok:test -o /workdir/cache/image-archive/localhost/kwok/test.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/localhost/kwok/test.tar --name kwok- +rm /workdir/cache/image-archive/localhost/kwok/test.tar kubectl config view --minify=true --raw=true cat </workdir/clusters//kubeconfig.yaml EOF diff --git a/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt index 1fb66c56d8..e8c919c17b 100644 --- a/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt @@ -3,116 +3,120 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//pki/etcd -mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//manifests -mkdir -p /workdir/clusters//logs -touch /workdir/clusters//logs/audit.log -cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml -cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml -cat <>/workdir/clusters//scheduler.yaml -clientConnection: - kubeconfig: "/etc/kubernetes/scheduler.conf" -EOF -cat </workdir/clusters//apiserver-tracing-config.yaml -apiVersion: apiserver.config.k8s.io/v1alpha1 -kind: TracingConfiguration -endpoint: 0.0.0.0:4317 -samplingRatePerMillion: 1000000 +cat </workdir/clusters//manifests/jaeger.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: jaeger + namespace: kube-system +spec: + containers: + - args: + - --collector.otlp.enabled=true + - --query.http-server.host-port=0.0.0.0:16686 + - --log-level=debug + image: docker.io/jaegertracing/all-in-one:1.45.0 + imagePullPolicy: Never + name: jaeger + ports: + - containerPort: 16686 + hostPort: 16686 + resources: {} + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 +status: {} EOF -cat </workdir/clusters//kind.yaml -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -networking: -nodes: -- role: control-plane - extraPortMappings: - - containerPort: 8000 - hostPort: 8000 - protocol: TCP - - containerPort: 9090 - hostPort: 9090 - protocol: TCP - - containerPort: 16686 - hostPort: 16686 - protocol: TCP - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - etcd: - local: - extraArgs: - "log-level": "debug" - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - "audit-policy-file": "/etc/kubernetes/audit/audit.yaml" - "audit-log-path": "/var/log/kubernetes/audit.log" - "tracing-config-file": "/etc/kubernetes/apiserver-tracing-config.yaml" - "v": "4" - "max-requests-inflight": "0" - "max-mutating-requests-inflight": "0" - "enable-priority-and-fairness": "false" - extraVolumes: - - name: audit-policy-file - hostPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml - mountPath: /etc/kubernetes/audit/audit.yaml - readOnly: true - pathType: File - - name: audit-log-path - hostPath: /var/components/apiserver/var/log/kubernetes/audit.log - mountPath: /var/log/kubernetes/audit.log - readOnly: false - pathType: File - - name: apiserver-tracing-config - hostPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml - mountPath: /etc/kubernetes/apiserver-tracing-config.yaml - readOnly: true - pathType: File - - | - kind: ClusterConfiguration - controllerManager: - extraArgs: - "v": "4" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - - | - kind: ClusterConfiguration - scheduler: - extraArgs: - "config": "/etc/kubernetes/scheduler/scheduler.yaml" - "v": "4" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - extraVolumes: - - name: config - hostPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml - mountPath: /etc/kubernetes/scheduler/scheduler.yaml - readOnly: true - pathType: File - # mount the local file on the control plane - extraMounts: - - hostPath: /workdir/clusters/ - containerPath: /etc/kwok/ - - hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/manifests - - hostPath: /workdir/clusters//pki - containerPath: /etc/kubernetes/pki - - hostPath: /workdir/clusters//audit.yaml - containerPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml - readOnly: true - - hostPath: /workdir/clusters//logs/audit.log - containerPath: /var/components/apiserver/var/log/kubernetes/audit.log - readOnly: false - - hostPath: /workdir/clusters//apiserver-tracing-config.yaml - containerPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml - readOnly: true - - hostPath: /workdir/clusters//scheduler.yaml - containerPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml - readOnly: true - - hostPath: /workdir/clusters//prometheus.yaml - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml - readOnly: false +mkdir -p /workdir/clusters//files/prometheus/etc/prometheus +cat </workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:10247 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10259 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +- job_name: "etcd" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:6443 EOF cat </workdir/clusters//manifests/kwok-controller.yaml apiVersion: v1 @@ -134,35 +138,18 @@ spec: - --node-name=kwok-controller.kube-system.svc - --node-port=10247 - --server-address=0.0.0.0:10247 - - --node-lease-duration-seconds=40 - - --v=DEBUG - command: - - kwok - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP + - --node-lease-duration-seconds=1200 image: localhost/kwok:test imagePullPolicy: Never name: kwok-controller resources: {} volumeMounts: - - mountPath: ~/.kube/config + - mountPath: /etc/kubernetes/pki name: volume-0 - readOnly: true - - mountPath: /etc/kubernetes/pki/ca.crt + - mountPath: ~/.kube/config name: volume-1 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-2 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-3 - readOnly: true - mountPath: ~/.kwok/kwok.yaml - name: volume-4 - readOnly: true + name: volume-2 hostNetwork: true restartPolicy: Always securityContext: @@ -170,20 +157,14 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /etc/kubernetes/admin.conf + path: /etc/kubernetes/pki name: volume-0 - hostPath: - path: /etc/kubernetes/pki/ca.crt + path: /etc/kubernetes/admin.conf name: volume-1 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-2 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-3 - hostPath: path: /etc/kwok/kwok.yaml - name: volume-4 + name: volume-2 status: {} EOF cat </workdir/clusters//manifests/dashboard.yaml @@ -197,6 +178,7 @@ spec: containers: - args: - --insecure-bind-address=0.0.0.0 + - --insecure-port=8000 - --bind-address=127.0.0.1 - --port=0 - --enable-insecure-login @@ -205,7 +187,6 @@ spec: - --metrics-provider=none - --system-banner=Welcome to kwok- - --kubeconfig=~/.kube/config - - --insecure-port=8000 image: docker.io/kubernetesui/dashboard:v2.7.0 imagePullPolicy: Never name: dashboard @@ -216,18 +197,10 @@ spec: protocol: TCP resources: {} volumeMounts: - - mountPath: ~/.kube/config + - mountPath: /etc/kubernetes/pki name: volume-0 - readOnly: true - - mountPath: /etc/kubernetes/pki/ca.crt + - mountPath: ~/.kube/config name: volume-1 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-2 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-3 - readOnly: true hostNetwork: true restartPolicy: Always securityContext: @@ -235,17 +208,11 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /etc/kubernetes/admin.conf + path: /etc/kubernetes/pki name: volume-0 - hostPath: - path: /etc/kubernetes/pki/ca.crt + path: /etc/kubernetes/admin.conf name: volume-1 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-2 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-3 status: {} EOF cat </workdir/clusters//manifests/prometheus.yaml @@ -260,32 +227,22 @@ spec: - args: - --config.file=/etc/prometheus/prometheus.yaml - --web.listen-address=0.0.0.0:9090 - - --log.level=debug - command: - - prometheus image: docker.io/prom/prometheus:v2.44.0 imagePullPolicy: Never name: prometheus ports: - containerPort: 9090 hostPort: 9090 + name: http + protocol: TCP resources: {} volumeMounts: - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt - name: volume-0 - readOnly: true - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key - name: volume-1 - readOnly: true - mountPath: /etc/prometheus/prometheus.yaml + name: file-0 + - mountPath: /etc/kubernetes/pki + name: volume-1 + - mountPath: ~/.kube/config name: volume-2 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-3 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-4 - readOnly: true hostNetwork: true restartPolicy: Always securityContext: @@ -293,20 +250,14 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.crt - name: volume-0 + path: /var/components/prometheus/etc/prometheus/prometheus.yaml + name: file-0 - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.key + path: /etc/kubernetes/pki name: volume-1 - hostPath: - path: /var/components/prometheus/etc/prometheus/prometheus.yaml + path: /etc/kubernetes/admin.conf name: volume-2 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-3 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-4 status: {} EOF cat </workdir/clusters//manifests/jaeger.yaml @@ -329,113 +280,137 @@ spec: - containerPort: 16686 hostPort: 16686 resources: {} + volumeMounts: + - mountPath: /etc/kubernetes/pki + name: volume-0 + - mountPath: ~/.kube/config + name: volume-1 hostNetwork: true restartPolicy: Always securityContext: runAsGroup: 0 runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/pki + name: volume-0 + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-1 status: {} EOF -cat </workdir/clusters//prometheus.yaml -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - follow_redirects: true - enable_http2: true - scheme: http - timeout: 10s - api_version: v2 - static_configs: - - targets: [] -scrape_configs: -- job_name: "etcd" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" - key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:2379 -- job_name: "kube-apiserver" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:6443 -- job_name: "kube-controller-manager" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:10257 -- job_name: "kube-scheduler" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:10259 -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://127.0.0.1:10247/discovery/prometheus -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - 127.0.0.1:10247 -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - 127.0.0.1:9090 +mkdir -p /workdir/clusters//pki +mkdir -p /workdir/clusters//logs +touch /workdir/clusters//logs/audit.log +cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml +cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml +cat <>/workdir/clusters//scheduler.yaml +clientConnection: + kubeconfig: "/etc/kubernetes/scheduler.conf" +EOF +cat </workdir/clusters//apiserver-tracing-config.yaml +apiVersion: apiserver.config.k8s.io/v1alpha1 +kind: TracingConfiguration +endpoint: 0.0.0.0:4317 +samplingRatePerMillion: 1000000 +EOF +cat </workdir/clusters//kind.yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 16686 + hostPort: 16686 + protocol: TCP + - containerPort: 8000 + hostPort: 8000 + protocol: TCP + - containerPort: 9090 + hostPort: 9090 + protocol: TCP + - containerPort: 16686 + hostPort: 16686 + protocol: + kubeadmConfigPatches: + - | + kind: ClusterConfiguration + etcd: + local: + extraArgs: + "log-level": "debug" + - | + kind: ClusterConfiguration + apiServer: + extraArgs: + "audit-policy-file": "/etc/kubernetes/audit/audit.yaml" + "audit-log-path": "/var/log/kubernetes/audit.log" + "tracing-config-file": "/etc/kubernetes/apiserver-tracing-config.yaml" + "v": "4" + "max-requests-inflight": "0" + "max-mutating-requests-inflight": "0" + "enable-priority-and-fairness": "false" + extraVolumes: + - name: audit-policy-file + hostPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml + mountPath: /etc/kubernetes/audit/audit.yaml + readOnly: true + pathType: File + - name: audit-log-path + hostPath: /var/components/apiserver/var/log/kubernetes/audit.log + mountPath: /var/log/kubernetes/audit.log + readOnly: false + pathType: File + - name: apiserver-tracing-config + hostPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml + mountPath: /etc/kubernetes/apiserver-tracing-config.yaml + readOnly: true + pathType: File + - | + kind: ClusterConfiguration + controllerManager: + extraArgs: + "v": "4" + "kube-api-qps": "5000" + "kube-api-burst": "10000" + - | + kind: ClusterConfiguration + scheduler: + extraArgs: + "config": "/etc/kubernetes/scheduler/scheduler.yaml" + "v": "4" + "kube-api-qps": "5000" + "kube-api-burst": "10000" + extraVolumes: + - name: config + hostPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml + mountPath: /etc/kubernetes/scheduler/scheduler.yaml + readOnly: true + pathType: File + # mount the local file on the control plane + extraMounts: + - hostPath: /workdir/clusters/ + containerPath: /etc/kwok/ + - hostPath: /workdir/clusters//manifests + containerPath: /etc/kubernetes/manifests + - hostPath: /workdir/clusters//pki + containerPath: /etc/kubernetes/pki + - hostPath: /workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml + containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml + readOnly: false EOF podman pull docker.io/kindest/node:v1.28.0 -podman pull localhost/kwok:test +podman pull docker.io/jaegertracing/all-in-one:1.45.0 podman pull docker.io/kubernetesui/dashboard:v2.7.0 podman pull docker.io/prom/prometheus:v2.44.0 -podman pull docker.io/jaegertracing/all-in-one:1.45.0 +podman pull localhost/kwok:test # Save cluster config to /workdir/clusters//kwok.yaml KIND_EXPERIMENTAL_PROVIDER=podman kind create cluster --config /workdir/clusters//kind.yaml --name kwok- --image docker.io/kindest/node:v1.28.0 --wait 29m -mkdir -p /workdir/cache/image-archive/localhost/kwok -podman save localhost/kwok:test -o /workdir/cache/image-archive/localhost/kwok/test.tar -KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/localhost/kwok/test.tar --name kwok- -rm /workdir/cache/image-archive/localhost/kwok/test.tar +mkdir -p /workdir/cache/image-archive/docker.io/jaegertracing/all-in-one +podman save docker.io/jaegertracing/all-in-one:1.45.0 -o /workdir/cache/image-archive/docker.io/jaegertracing/all-in-one/1.45.0.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/jaegertracing/all-in-one/1.45.0.tar --name kwok- +rm /workdir/cache/image-archive/docker.io/jaegertracing/all-in-one/1.45.0.tar mkdir -p /workdir/cache/image-archive/docker.io/kubernetesui/dashboard podman save docker.io/kubernetesui/dashboard:v2.7.0 -o /workdir/cache/image-archive/docker.io/kubernetesui/dashboard/v2.7.0.tar KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/kubernetesui/dashboard/v2.7.0.tar --name kwok- @@ -444,10 +419,10 @@ mkdir -p /workdir/cache/image-archive/docker.io/prom/prometheus podman save docker.io/prom/prometheus:v2.44.0 -o /workdir/cache/image-archive/docker.io/prom/prometheus/v2.44.0.tar KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/prom/prometheus/v2.44.0.tar --name kwok- rm /workdir/cache/image-archive/docker.io/prom/prometheus/v2.44.0.tar -mkdir -p /workdir/cache/image-archive/docker.io/jaegertracing/all-in-one -podman save docker.io/jaegertracing/all-in-one:1.45.0 -o /workdir/cache/image-archive/docker.io/jaegertracing/all-in-one/1.45.0.tar -KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/jaegertracing/all-in-one/1.45.0.tar --name kwok- -rm /workdir/cache/image-archive/docker.io/jaegertracing/all-in-one/1.45.0.tar +mkdir -p /workdir/cache/image-archive/localhost/kwok +podman save localhost/kwok:test -o /workdir/cache/image-archive/localhost/kwok/test.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/localhost/kwok/test.tar --name kwok- +rm /workdir/cache/image-archive/localhost/kwok/test.tar kubectl config view --minify=true --raw=true cat </workdir/clusters//kubeconfig.yaml EOF diff --git a/test/kwokctl/testdata/kind/create_cluster.txt b/test/kwokctl/testdata/kind/create_cluster.txt index 98a7459054..457d96e312 100644 --- a/test/kwokctl/testdata/kind/create_cluster.txt +++ b/test/kwokctl/testdata/kind/create_cluster.txt @@ -3,43 +3,7 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//pki/etcd -mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//manifests -cat </workdir/clusters//kind.yaml -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -networking: -nodes: -- role: control-plane - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - "max-requests-inflight": "0" - "max-mutating-requests-inflight": "0" - "enable-priority-and-fairness": "false" - - | - kind: ClusterConfiguration - controllerManager: - extraArgs: - "kube-api-qps": "5000" - "kube-api-burst": "10000" - - | - kind: ClusterConfiguration - scheduler: - extraArgs: - "kube-api-qps": "5000" - "kube-api-burst": "10000" - # mount the local file on the control plane - extraMounts: - - hostPath: /workdir/clusters/ - containerPath: /etc/kwok/ - - hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/manifests - - hostPath: /workdir/clusters//pki - containerPath: /etc/kubernetes/pki -EOF cat </workdir/clusters//manifests/kwok-controller.yaml apiVersion: v1 kind: Pod @@ -60,34 +24,18 @@ spec: - --node-name=kwok-controller.kube-system.svc - --node-port=10247 - --server-address=0.0.0.0:10247 - - --node-lease-duration-seconds=40 - command: - - kwok - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP + - --node-lease-duration-seconds=1200 image: localhost/kwok:test imagePullPolicy: Never name: kwok-controller resources: {} volumeMounts: - - mountPath: ~/.kube/config + - mountPath: /etc/kubernetes/pki name: volume-0 - readOnly: true - - mountPath: /etc/kubernetes/pki/ca.crt + - mountPath: ~/.kube/config name: volume-1 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-2 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-3 - readOnly: true - mountPath: ~/.kwok/kwok.yaml - name: volume-4 - readOnly: true + name: volume-2 hostNetwork: true restartPolicy: Always securityContext: @@ -95,22 +43,52 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /etc/kubernetes/admin.conf + path: /etc/kubernetes/pki name: volume-0 - hostPath: - path: /etc/kubernetes/pki/ca.crt + path: /etc/kubernetes/admin.conf name: volume-1 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-2 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-3 - hostPath: path: /etc/kwok/kwok.yaml - name: volume-4 + name: volume-2 status: {} EOF +mkdir -p /workdir/clusters//pki +cat </workdir/clusters//kind.yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: +nodes: +- role: control-plane + kubeadmConfigPatches: + - | + kind: ClusterConfiguration + apiServer: + extraArgs: + "max-requests-inflight": "0" + "max-mutating-requests-inflight": "0" + "enable-priority-and-fairness": "false" + - | + kind: ClusterConfiguration + controllerManager: + extraArgs: + "kube-api-qps": "5000" + "kube-api-burst": "10000" + - | + kind: ClusterConfiguration + scheduler: + extraArgs: + "kube-api-qps": "5000" + "kube-api-burst": "10000" + # mount the local file on the control plane + extraMounts: + - hostPath: /workdir/clusters/ + containerPath: /etc/kwok/ + - hostPath: /workdir/clusters//manifests + containerPath: /etc/kubernetes/manifests + - hostPath: /workdir/clusters//pki + containerPath: /etc/kubernetes/pki +EOF docker pull docker.io/kindest/node:v1.28.0 docker pull localhost/kwok:test # Save cluster config to /workdir/clusters//kwok.yaml diff --git a/test/kwokctl/testdata/kind/create_cluster_with_extra.txt b/test/kwokctl/testdata/kind/create_cluster_with_extra.txt index 4a6745dfd5..4f45bfe0f5 100644 --- a/test/kwokctl/testdata/kind/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/kind/create_cluster_with_extra.txt @@ -3,100 +3,93 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//pki/etcd -mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//manifests -cat </workdir/clusters//kind.yaml -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -networking: -nodes: -- role: control-plane - extraPortMappings: - - containerPort: 9090 - hostPort: 9090 - protocol: TCP - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - etcd: - local: - extraArgs: - "log-level": "debug" - extraVolumes: - - name: tmp-etcd - hostPath: /var/components/etcd/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - "v": "5" - "max-requests-inflight": "0" - "max-mutating-requests-inflight": "0" - "enable-priority-and-fairness": "false" - extraVolumes: - - name: tmp-apiserver - hostPath: /var/components/apiserver/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - - | - kind: ClusterConfiguration - controllerManager: - extraArgs: - "v": "5" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - extraVolumes: - - name: tmp-controller-manager - hostPath: /var/components/controller-manager/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - - | - kind: ClusterConfiguration - scheduler: - extraArgs: - "v": "5" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - extraVolumes: - - name: tmp-scheduler - hostPath: /var/components/scheduler/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - # mount the local file on the control plane - extraMounts: - - hostPath: /workdir/clusters/ - containerPath: /etc/kwok/ - - hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/manifests - - hostPath: /workdir/clusters//pki - containerPath: /etc/kubernetes/pki - - hostPath: /extras/etcd - containerPath: /var/components/etcd/extras/tmp - readOnly: false - - hostPath: /extras/apiserver - containerPath: /var/components/apiserver/extras/tmp - readOnly: false - - hostPath: /extras/controller-manager - containerPath: /var/components/controller-manager/extras/tmp - readOnly: false - - hostPath: /extras/scheduler - containerPath: /var/components/scheduler/extras/tmp - readOnly: false - - hostPath: /extras/controller - containerPath: /var/components/controller/extras/tmp - readOnly: false - - hostPath: ./extras/prometheus - containerPath: /var/components/prometheus/extras/tmp - readOnly: false - - hostPath: /workdir/clusters//prometheus.yaml - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml - readOnly: false +mkdir -p /workdir/clusters//files/prometheus/etc/prometheus +cat </workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:10247 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10259 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +- job_name: "etcd" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:6443 EOF cat </workdir/clusters//manifests/kwok-controller.yaml apiVersion: v1 @@ -110,7 +103,6 @@ spec: - args: - --manage-all-nodes=false - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake - - --v=-4 - --kubeconfig=~/.kube/config - --config=~/.kwok/kwok.yaml - --tls-cert-file=/etc/kubernetes/pki/admin.crt @@ -119,38 +111,18 @@ spec: - --node-name=kwok-controller.kube-system.svc - --node-port=10247 - --server-address=0.0.0.0:10247 - - --node-lease-duration-seconds=40 - command: - - kwok - env: - - name: TEST_KEY - value: TEST_VALUE - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP + - --node-lease-duration-seconds=1200 image: localhost/kwok:test imagePullPolicy: Never name: kwok-controller resources: {} volumeMounts: - - mountPath: /extras/tmp - name: tmp-controller + - mountPath: /etc/kubernetes/pki + name: volume-0 - mountPath: ~/.kube/config name: volume-1 - readOnly: true - - mountPath: /etc/kubernetes/pki/ca.crt - name: volume-2 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-3 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-4 - readOnly: true - mountPath: ~/.kwok/kwok.yaml - name: volume-5 - readOnly: true + name: volume-2 hostNetwork: true restartPolicy: Always securityContext: @@ -158,24 +130,14 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /extras/controller - type: DirectoryOrCreate - name: tmp-controller + path: /etc/kubernetes/pki + name: volume-0 - hostPath: path: /etc/kubernetes/admin.conf name: volume-1 - - hostPath: - path: /etc/kubernetes/pki/ca.crt - name: volume-2 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-3 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-4 - hostPath: path: /etc/kwok/kwok.yaml - name: volume-5 + name: volume-2 status: {} EOF cat </workdir/clusters//manifests/prometheus.yaml @@ -188,39 +150,24 @@ metadata: spec: containers: - args: - - --log.level=debug - --config.file=/etc/prometheus/prometheus.yaml - --web.listen-address=0.0.0.0:9090 - command: - - prometheus - env: - - name: TEST_KEY - value: TEST_VALUE image: docker.io/prom/prometheus:v2.44.0 imagePullPolicy: Never name: prometheus ports: - containerPort: 9090 hostPort: 9090 + name: http + protocol: TCP resources: {} volumeMounts: - - mountPath: /extras/tmp - name: tmp-prometheus - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt + - mountPath: /etc/prometheus/prometheus.yaml + name: file-0 + - mountPath: /etc/kubernetes/pki name: volume-1 - readOnly: true - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key + - mountPath: ~/.kube/config name: volume-2 - readOnly: true - - mountPath: /etc/prometheus/prometheus.yaml - name: volume-3 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-4 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-5 - readOnly: true hostNetwork: true restartPolicy: Always securityContext: @@ -228,122 +175,99 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /extras/prometheus - type: DirectoryOrCreate - name: tmp-prometheus + path: /var/components/prometheus/etc/prometheus/prometheus.yaml + name: file-0 - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.crt + path: /etc/kubernetes/pki name: volume-1 - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.key + path: /etc/kubernetes/admin.conf name: volume-2 - - hostPath: - path: /var/components/prometheus/etc/prometheus/prometheus.yaml - name: volume-3 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-4 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-5 status: {} EOF -cat </workdir/clusters//prometheus.yaml -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - follow_redirects: true - enable_http2: true - scheme: http - timeout: 10s - api_version: v2 - static_configs: - - targets: [] -scrape_configs: -- job_name: "etcd" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" - key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:2379 -- job_name: "kube-apiserver" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:6443 -- job_name: "kube-controller-manager" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:10257 -- job_name: "kube-scheduler" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:10259 -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://127.0.0.1:10247/discovery/prometheus -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - 127.0.0.1:10247 -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - 127.0.0.1:9090 +mkdir -p /workdir/clusters//pki +cat </workdir/clusters//kind.yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 9090 + hostPort: 9090 + protocol: TCP + kubeadmConfigPatches: + - | + kind: ClusterConfiguration + etcd: + local: + extraArgs: + "log-level": "debug" + extraVolumes: + - name: tmp-etcd + hostPath: /var/components/etcd/extras/tmp + mountPath: /extras/tmp + readOnly: false + pathType: DirectoryOrCreate + - | + kind: ClusterConfiguration + apiServer: + extraArgs: + "v": "5" + "max-requests-inflight": "0" + "max-mutating-requests-inflight": "0" + "enable-priority-and-fairness": "false" + extraVolumes: + - name: tmp-apiserver + hostPath: /var/components/apiserver/extras/tmp + mountPath: /extras/tmp + readOnly: false + pathType: DirectoryOrCreate + - | + kind: ClusterConfiguration + controllerManager: + extraArgs: + "v": "5" + "kube-api-qps": "5000" + "kube-api-burst": "10000" + extraVolumes: + - name: tmp-controller-manager + hostPath: /var/components/controller-manager/extras/tmp + mountPath: /extras/tmp + readOnly: false + pathType: DirectoryOrCreate + - | + kind: ClusterConfiguration + scheduler: + extraArgs: + "v": "5" + "kube-api-qps": "5000" + "kube-api-burst": "10000" + extraVolumes: + - name: tmp-scheduler + hostPath: /var/components/scheduler/extras/tmp + mountPath: /extras/tmp + readOnly: false + pathType: DirectoryOrCreate + # mount the local file on the control plane + extraMounts: + - hostPath: /workdir/clusters/ + containerPath: /etc/kwok/ + - hostPath: /workdir/clusters//manifests + containerPath: /etc/kubernetes/manifests + - hostPath: /workdir/clusters//pki + containerPath: /etc/kubernetes/pki + - hostPath: /workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml + containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml + readOnly: false EOF docker pull docker.io/kindest/node:v1.28.0 -docker pull localhost/kwok:test docker pull docker.io/prom/prometheus:v2.44.0 +docker pull localhost/kwok:test # Save cluster config to /workdir/clusters//kwok.yaml KIND_EXPERIMENTAL_PROVIDER=docker kind create cluster --config /workdir/clusters//kind.yaml --name kwok- --image docker.io/kindest/node:v1.28.0 --wait 29m -KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image localhost/kwok:test --name kwok- KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/prom/prometheus:v2.44.0 --name kwok- +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image localhost/kwok:test --name kwok- kubectl config view --minify=true --raw=true cat </workdir/clusters//kubeconfig.yaml EOF diff --git a/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt index f65bc55228..12b21bb16e 100644 --- a/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt @@ -3,116 +3,120 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//pki/etcd -mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//manifests -mkdir -p /workdir/clusters//logs -touch /workdir/clusters//logs/audit.log -cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml -cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml -cat <>/workdir/clusters//scheduler.yaml -clientConnection: - kubeconfig: "/etc/kubernetes/scheduler.conf" -EOF -cat </workdir/clusters//apiserver-tracing-config.yaml -apiVersion: apiserver.config.k8s.io/v1alpha1 -kind: TracingConfiguration -endpoint: 0.0.0.0:4317 -samplingRatePerMillion: 1000000 +cat </workdir/clusters//manifests/jaeger.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: jaeger + namespace: kube-system +spec: + containers: + - args: + - --collector.otlp.enabled=true + - --query.http-server.host-port=0.0.0.0:16686 + - --log-level=debug + image: docker.io/jaegertracing/all-in-one:1.45.0 + imagePullPolicy: Never + name: jaeger + ports: + - containerPort: 16686 + hostPort: 16686 + resources: {} + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 +status: {} EOF -cat </workdir/clusters//kind.yaml -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -networking: -nodes: -- role: control-plane - extraPortMappings: - - containerPort: 8000 - hostPort: 8000 - protocol: TCP - - containerPort: 9090 - hostPort: 9090 - protocol: TCP - - containerPort: 16686 - hostPort: 16686 - protocol: TCP - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - etcd: - local: - extraArgs: - "log-level": "debug" - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - "audit-policy-file": "/etc/kubernetes/audit/audit.yaml" - "audit-log-path": "/var/log/kubernetes/audit.log" - "tracing-config-file": "/etc/kubernetes/apiserver-tracing-config.yaml" - "v": "4" - "max-requests-inflight": "0" - "max-mutating-requests-inflight": "0" - "enable-priority-and-fairness": "false" - extraVolumes: - - name: audit-policy-file - hostPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml - mountPath: /etc/kubernetes/audit/audit.yaml - readOnly: true - pathType: File - - name: audit-log-path - hostPath: /var/components/apiserver/var/log/kubernetes/audit.log - mountPath: /var/log/kubernetes/audit.log - readOnly: false - pathType: File - - name: apiserver-tracing-config - hostPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml - mountPath: /etc/kubernetes/apiserver-tracing-config.yaml - readOnly: true - pathType: File - - | - kind: ClusterConfiguration - controllerManager: - extraArgs: - "v": "4" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - - | - kind: ClusterConfiguration - scheduler: - extraArgs: - "config": "/etc/kubernetes/scheduler/scheduler.yaml" - "v": "4" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - extraVolumes: - - name: config - hostPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml - mountPath: /etc/kubernetes/scheduler/scheduler.yaml - readOnly: true - pathType: File - # mount the local file on the control plane - extraMounts: - - hostPath: /workdir/clusters/ - containerPath: /etc/kwok/ - - hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/manifests - - hostPath: /workdir/clusters//pki - containerPath: /etc/kubernetes/pki - - hostPath: /workdir/clusters//audit.yaml - containerPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml - readOnly: true - - hostPath: /workdir/clusters//logs/audit.log - containerPath: /var/components/apiserver/var/log/kubernetes/audit.log - readOnly: false - - hostPath: /workdir/clusters//apiserver-tracing-config.yaml - containerPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml - readOnly: true - - hostPath: /workdir/clusters//scheduler.yaml - containerPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml - readOnly: true - - hostPath: /workdir/clusters//prometheus.yaml - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml - readOnly: false +mkdir -p /workdir/clusters//files/prometheus/etc/prometheus +cat </workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:10247 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10259 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +- job_name: "etcd" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:6443 EOF cat </workdir/clusters//manifests/kwok-controller.yaml apiVersion: v1 @@ -134,35 +138,18 @@ spec: - --node-name=kwok-controller.kube-system.svc - --node-port=10247 - --server-address=0.0.0.0:10247 - - --node-lease-duration-seconds=40 - - --v=DEBUG - command: - - kwok - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP + - --node-lease-duration-seconds=1200 image: localhost/kwok:test imagePullPolicy: Never name: kwok-controller resources: {} volumeMounts: - - mountPath: ~/.kube/config + - mountPath: /etc/kubernetes/pki name: volume-0 - readOnly: true - - mountPath: /etc/kubernetes/pki/ca.crt + - mountPath: ~/.kube/config name: volume-1 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-2 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-3 - readOnly: true - mountPath: ~/.kwok/kwok.yaml - name: volume-4 - readOnly: true + name: volume-2 hostNetwork: true restartPolicy: Always securityContext: @@ -170,20 +157,14 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /etc/kubernetes/admin.conf + path: /etc/kubernetes/pki name: volume-0 - hostPath: - path: /etc/kubernetes/pki/ca.crt + path: /etc/kubernetes/admin.conf name: volume-1 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-2 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-3 - hostPath: path: /etc/kwok/kwok.yaml - name: volume-4 + name: volume-2 status: {} EOF cat </workdir/clusters//manifests/dashboard.yaml @@ -197,6 +178,7 @@ spec: containers: - args: - --insecure-bind-address=0.0.0.0 + - --insecure-port=8000 - --bind-address=127.0.0.1 - --port=0 - --enable-insecure-login @@ -205,7 +187,6 @@ spec: - --metrics-provider=none - --system-banner=Welcome to kwok- - --kubeconfig=~/.kube/config - - --insecure-port=8000 image: docker.io/kubernetesui/dashboard:v2.7.0 imagePullPolicy: Never name: dashboard @@ -216,18 +197,10 @@ spec: protocol: TCP resources: {} volumeMounts: - - mountPath: ~/.kube/config + - mountPath: /etc/kubernetes/pki name: volume-0 - readOnly: true - - mountPath: /etc/kubernetes/pki/ca.crt + - mountPath: ~/.kube/config name: volume-1 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-2 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-3 - readOnly: true hostNetwork: true restartPolicy: Always securityContext: @@ -235,17 +208,11 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /etc/kubernetes/admin.conf + path: /etc/kubernetes/pki name: volume-0 - hostPath: - path: /etc/kubernetes/pki/ca.crt + path: /etc/kubernetes/admin.conf name: volume-1 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-2 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-3 status: {} EOF cat </workdir/clusters//manifests/prometheus.yaml @@ -260,32 +227,22 @@ spec: - args: - --config.file=/etc/prometheus/prometheus.yaml - --web.listen-address=0.0.0.0:9090 - - --log.level=debug - command: - - prometheus image: docker.io/prom/prometheus:v2.44.0 imagePullPolicy: Never name: prometheus ports: - containerPort: 9090 hostPort: 9090 + name: http + protocol: TCP resources: {} volumeMounts: - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt - name: volume-0 - readOnly: true - - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key - name: volume-1 - readOnly: true - mountPath: /etc/prometheus/prometheus.yaml + name: file-0 + - mountPath: /etc/kubernetes/pki + name: volume-1 + - mountPath: ~/.kube/config name: volume-2 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.crt - name: volume-3 - readOnly: true - - mountPath: /etc/kubernetes/pki/admin.key - name: volume-4 - readOnly: true hostNetwork: true restartPolicy: Always securityContext: @@ -293,20 +250,14 @@ spec: runAsUser: 0 volumes: - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.crt - name: volume-0 + path: /var/components/prometheus/etc/prometheus/prometheus.yaml + name: file-0 - hostPath: - path: /etc/kubernetes/pki/apiserver-etcd-client.key + path: /etc/kubernetes/pki name: volume-1 - hostPath: - path: /var/components/prometheus/etc/prometheus/prometheus.yaml + path: /etc/kubernetes/admin.conf name: volume-2 - - hostPath: - path: /etc/kubernetes/pki/admin.crt - name: volume-3 - - hostPath: - path: /etc/kubernetes/pki/admin.key - name: volume-4 status: {} EOF cat </workdir/clusters//manifests/jaeger.yaml @@ -329,113 +280,137 @@ spec: - containerPort: 16686 hostPort: 16686 resources: {} + volumeMounts: + - mountPath: /etc/kubernetes/pki + name: volume-0 + - mountPath: ~/.kube/config + name: volume-1 hostNetwork: true restartPolicy: Always securityContext: runAsGroup: 0 runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/pki + name: volume-0 + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-1 status: {} EOF -cat </workdir/clusters//prometheus.yaml -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - follow_redirects: true - enable_http2: true - scheme: http - timeout: 10s - api_version: v2 - static_configs: - - targets: [] -scrape_configs: -- job_name: "etcd" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" - key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:2379 -- job_name: "kube-apiserver" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:6443 -- job_name: "kube-controller-manager" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:10257 -- job_name: "kube-scheduler" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true - static_configs: - - targets: - - 127.0.0.1:10259 -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://127.0.0.1:10247/discovery/prometheus -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - 127.0.0.1:10247 -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - 127.0.0.1:9090 +mkdir -p /workdir/clusters//pki +mkdir -p /workdir/clusters//logs +touch /workdir/clusters//logs/audit.log +cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml +cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml +cat <>/workdir/clusters//scheduler.yaml +clientConnection: + kubeconfig: "/etc/kubernetes/scheduler.conf" +EOF +cat </workdir/clusters//apiserver-tracing-config.yaml +apiVersion: apiserver.config.k8s.io/v1alpha1 +kind: TracingConfiguration +endpoint: 0.0.0.0:4317 +samplingRatePerMillion: 1000000 +EOF +cat </workdir/clusters//kind.yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 16686 + hostPort: 16686 + protocol: TCP + - containerPort: 8000 + hostPort: 8000 + protocol: TCP + - containerPort: 9090 + hostPort: 9090 + protocol: TCP + - containerPort: 16686 + hostPort: 16686 + protocol: + kubeadmConfigPatches: + - | + kind: ClusterConfiguration + etcd: + local: + extraArgs: + "log-level": "debug" + - | + kind: ClusterConfiguration + apiServer: + extraArgs: + "audit-policy-file": "/etc/kubernetes/audit/audit.yaml" + "audit-log-path": "/var/log/kubernetes/audit.log" + "tracing-config-file": "/etc/kubernetes/apiserver-tracing-config.yaml" + "v": "4" + "max-requests-inflight": "0" + "max-mutating-requests-inflight": "0" + "enable-priority-and-fairness": "false" + extraVolumes: + - name: audit-policy-file + hostPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml + mountPath: /etc/kubernetes/audit/audit.yaml + readOnly: true + pathType: File + - name: audit-log-path + hostPath: /var/components/apiserver/var/log/kubernetes/audit.log + mountPath: /var/log/kubernetes/audit.log + readOnly: false + pathType: File + - name: apiserver-tracing-config + hostPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml + mountPath: /etc/kubernetes/apiserver-tracing-config.yaml + readOnly: true + pathType: File + - | + kind: ClusterConfiguration + controllerManager: + extraArgs: + "v": "4" + "kube-api-qps": "5000" + "kube-api-burst": "10000" + - | + kind: ClusterConfiguration + scheduler: + extraArgs: + "config": "/etc/kubernetes/scheduler/scheduler.yaml" + "v": "4" + "kube-api-qps": "5000" + "kube-api-burst": "10000" + extraVolumes: + - name: config + hostPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml + mountPath: /etc/kubernetes/scheduler/scheduler.yaml + readOnly: true + pathType: File + # mount the local file on the control plane + extraMounts: + - hostPath: /workdir/clusters/ + containerPath: /etc/kwok/ + - hostPath: /workdir/clusters//manifests + containerPath: /etc/kubernetes/manifests + - hostPath: /workdir/clusters//pki + containerPath: /etc/kubernetes/pki + - hostPath: /workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml + containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml + readOnly: false EOF docker pull docker.io/kindest/node:v1.28.0 -docker pull localhost/kwok:test +docker pull docker.io/jaegertracing/all-in-one:1.45.0 docker pull docker.io/kubernetesui/dashboard:v2.7.0 docker pull docker.io/prom/prometheus:v2.44.0 -docker pull docker.io/jaegertracing/all-in-one:1.45.0 +docker pull localhost/kwok:test # Save cluster config to /workdir/clusters//kwok.yaml KIND_EXPERIMENTAL_PROVIDER=docker kind create cluster --config /workdir/clusters//kind.yaml --name kwok- --image docker.io/kindest/node:v1.28.0 --wait 29m -KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image localhost/kwok:test --name kwok- +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/jaegertracing/all-in-one:1.45.0 --name kwok- KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/kubernetesui/dashboard:v2.7.0 --name kwok- KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/prom/prometheus:v2.44.0 --name kwok- -KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/jaegertracing/all-in-one:1.45.0 --name kwok- +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image localhost/kwok:test --name kwok- kubectl config view --minify=true --raw=true cat </workdir/clusters//kubeconfig.yaml EOF diff --git a/test/kwokctl/testdata/nerdctl/create_cluster.txt b/test/kwokctl/testdata/nerdctl/create_cluster.txt index cacd5beb41..5845f58433 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster.txt @@ -3,9 +3,9 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd +nerdctl pull localhost/kwok:test nerdctl pull registry.k8s.io/etcd:3.5.9-0 nerdctl pull registry.k8s.io/kube-apiserver:v1.28.0 -nerdctl pull localhost/kwok:test nerdctl pull registry.k8s.io/kube-controller-manager:v1.28.0 nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 cat </workdir/clusters//kubeconfig.yaml @@ -54,12 +54,12 @@ EOF nerdctl network create kwok- --label=com.docker.compose.project=kwok- nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local -nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --kube-api-qps=5000 --kube-api-burst=10000 -nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +nerdctl create --name=kwok--kwok-controller --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=600000ms --node-monitor-grace-period=3600000ms --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics # Add context kwok- to ~/.kube/config nerdctl start kwok--etcd nerdctl start kwok--kube-apiserver +nerdctl start kwok--kwok-controller nerdctl start kwok--kube-controller-manager nerdctl start kwok--kube-scheduler -nerdctl start kwok--kwok-controller diff --git a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt index 53ec45a9e2..ac90f8bcb3 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt @@ -3,7 +3,8 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd -cat </workdir/clusters//prometheus.yaml +mkdir -p /workdir/clusters//files/prometheus/etc/prometheus +cat </workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml global: scrape_interval: 15s scrape_timeout: 10s @@ -18,7 +19,7 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "etcd" +- job_name: "kwok-controller" scheme: http honor_timestamps: true metrics_path: /metrics @@ -26,8 +27,8 @@ scrape_configs: enable_http2: true static_configs: - targets: - - kwok--etcd:2379 -- job_name: "kube-apiserver" + - kwok--kwok-controller:10247 +- job_name: "kube-controller-manager" scheme: https honor_timestamps: true metrics_path: /metrics @@ -39,8 +40,8 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - kwok--kube-apiserver:6443 -- job_name: "kube-controller-manager" + - kwok--kube-controller-manager:10257 +- job_name: "kube-scheduler" scheme: https honor_timestamps: true metrics_path: /metrics @@ -52,24 +53,17 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - kwok--kube-controller-manager:10257 -- job_name: "kube-scheduler" - scheme: https + - kwok--kube-scheduler:10259 +- job_name: "prometheus" + scheme: http honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true static_configs: - targets: - - kwok--kube-scheduler:10259 -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://kwok--kwok-controller:10247/discovery/prometheus -- job_name: "kwok-controller" + - 127.0.0.1:9090 +- job_name: "etcd" scheme: http honor_timestamps: true metrics_path: /metrics @@ -77,24 +71,27 @@ scrape_configs: enable_http2: true static_configs: - targets: - - kwok--kwok-controller:10247 -- job_name: "prometheus" - scheme: http + - kwok--etcd:2379 +- job_name: "kube-apiserver" + scheme: https honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:9090 + - kwok--kube-apiserver:6443 EOF -chmod 0644 /workdir/clusters//prometheus.yaml +nerdctl pull docker.io/prom/prometheus:v2.44.0 +nerdctl pull localhost/kwok:test nerdctl pull registry.k8s.io/etcd:3.5.9-0 nerdctl pull registry.k8s.io/kube-apiserver:v1.28.0 -nerdctl pull localhost/kwok:test nerdctl pull registry.k8s.io/kube-controller-manager:v1.28.0 nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 -nerdctl pull docker.io/prom/prometheus:v2.44.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: @@ -141,14 +138,14 @@ EOF nerdctl network create kwok- --label=com.docker.compose.project=kwok- nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --log-level=debug --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local -nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller-manager:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 -nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/scheduler:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --v=5 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 -nerdctl create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/extras/prometheus:/extras/tmp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.44.0 --log.level=debug --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 +nerdctl create --name=kwok--kwok-controller --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=600000ms --node-monitor-grace-period=3600000ms --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +nerdctl create --name=kwok--prometheus --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 # Add context kwok- to ~/.kube/config nerdctl start kwok--etcd nerdctl start kwok--kube-apiserver +nerdctl start kwok--kwok-controller nerdctl start kwok--kube-controller-manager nerdctl start kwok--kube-scheduler -nerdctl start kwok--kwok-controller nerdctl start kwok--prometheus diff --git a/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt index aca7572508..6ea81bb160 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt @@ -12,12 +12,8 @@ kind: TracingConfiguration endpoint: kwok--jaeger:4317 samplingRatePerMillion: 1000000 EOF -cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml -cat <>/workdir/clusters//scheduler.yaml -clientConnection: - kubeconfig: "~/.kube/config" -EOF -cat </workdir/clusters//prometheus.yaml +mkdir -p /workdir/clusters//files/prometheus/etc/prometheus +cat </workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml global: scrape_interval: 15s scrape_timeout: 10s @@ -32,7 +28,7 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "etcd" +- job_name: "kwok-controller" scheme: http honor_timestamps: true metrics_path: /metrics @@ -40,8 +36,8 @@ scrape_configs: enable_http2: true static_configs: - targets: - - kwok--etcd:2379 -- job_name: "kube-apiserver" + - kwok--kwok-controller:10247 +- job_name: "kube-controller-manager" scheme: https honor_timestamps: true metrics_path: /metrics @@ -53,8 +49,8 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - kwok--kube-apiserver:6443 -- job_name: "kube-controller-manager" + - kwok--kube-controller-manager:10257 +- job_name: "kube-scheduler" scheme: https honor_timestamps: true metrics_path: /metrics @@ -66,24 +62,17 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - kwok--kube-controller-manager:10257 -- job_name: "kube-scheduler" - scheme: https + - kwok--kube-scheduler:10259 +- job_name: "prometheus" + scheme: http honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true static_configs: - targets: - - kwok--kube-scheduler:10259 -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://kwok--kwok-controller:10247/discovery/prometheus -- job_name: "kwok-controller" + - 127.0.0.1:9090 +- job_name: "etcd" scheme: http honor_timestamps: true metrics_path: /metrics @@ -91,26 +80,29 @@ scrape_configs: enable_http2: true static_configs: - targets: - - kwok--kwok-controller:10247 -- job_name: "prometheus" - scheme: http + - kwok--etcd:2379 +- job_name: "kube-apiserver" + scheme: https honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:9090 + - kwok--kube-apiserver:6443 EOF -chmod 0644 /workdir/clusters//prometheus.yaml +nerdctl pull docker.io/jaegertracing/all-in-one:1.45.0 +nerdctl pull docker.io/kubernetesui/dashboard:v2.7.0 +nerdctl pull docker.io/prom/prometheus:v2.44.0 +nerdctl pull localhost/kwok:test nerdctl pull registry.k8s.io/etcd:3.5.9-0 nerdctl pull registry.k8s.io/kube-apiserver:v1.28.0 -nerdctl pull localhost/kwok:test nerdctl pull registry.k8s.io/kube-controller-manager:v1.28.0 nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 -nerdctl pull docker.io/kubernetesui/dashboard:v2.7.0 -nerdctl pull docker.io/prom/prometheus:v2.44.0 -nerdctl pull docker.io/jaegertracing/all-in-one:1.45.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: @@ -158,17 +150,17 @@ nerdctl network create kwok- --label=com.docker.compose.project=kw nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug nerdctl create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.45.0 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --log-level=debug nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 -nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 -nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.28.0 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 -nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 --v=DEBUG -nerdctl create --name=kwok--dashboard --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config --insecure-port=8000 -nerdctl create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug +nerdctl create --name=kwok--kwok-controller --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=600000ms --node-monitor-grace-period=3600000ms --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +nerdctl create --name=kwok--dashboard --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --insecure-port=8000 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config +nerdctl create --name=kwok--prometheus --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 # Add context kwok- to ~/.kube/config nerdctl start kwok--etcd nerdctl start kwok--jaeger nerdctl start kwok--kube-apiserver +nerdctl start kwok--kwok-controller nerdctl start kwok--kube-controller-manager nerdctl start kwok--kube-scheduler -nerdctl start kwok--kwok-controller nerdctl start kwok--dashboard nerdctl start kwok--prometheus diff --git a/test/kwokctl/testdata/podman/create_cluster.txt b/test/kwokctl/testdata/podman/create_cluster.txt index 24705c2931..ec0db64d9d 100644 --- a/test/kwokctl/testdata/podman/create_cluster.txt +++ b/test/kwokctl/testdata/podman/create_cluster.txt @@ -3,9 +3,9 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd +podman pull localhost/kwok:test podman pull registry.k8s.io/etcd:3.5.9-0 podman pull registry.k8s.io/kube-apiserver:v1.28.0 -podman pull localhost/kwok:test podman pull registry.k8s.io/kube-controller-manager:v1.28.0 podman pull registry.k8s.io/kube-scheduler:v1.28.0 cat </workdir/clusters//kubeconfig.yaml @@ -54,12 +54,12 @@ EOF podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local -podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --kube-api-qps=5000 --kube-api-burst=10000 -podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +podman create --name=kwok--kwok-controller --pull=never --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=600000ms --node-monitor-grace-period=3600000ms --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics # Add context kwok- to ~/.kube/config podman start kwok--etcd podman start kwok--kube-apiserver +podman start kwok--kwok-controller podman start kwok--kube-controller-manager podman start kwok--kube-scheduler -podman start kwok--kwok-controller diff --git a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt index dafd31cfc1..a774e546cc 100644 --- a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt @@ -3,7 +3,8 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd -cat </workdir/clusters//prometheus.yaml +mkdir -p /workdir/clusters//files/prometheus/etc/prometheus +cat </workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml global: scrape_interval: 15s scrape_timeout: 10s @@ -18,7 +19,7 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "etcd" +- job_name: "kwok-controller" scheme: http honor_timestamps: true metrics_path: /metrics @@ -26,8 +27,8 @@ scrape_configs: enable_http2: true static_configs: - targets: - - kwok--etcd:2379 -- job_name: "kube-apiserver" + - kwok--kwok-controller:10247 +- job_name: "kube-controller-manager" scheme: https honor_timestamps: true metrics_path: /metrics @@ -39,8 +40,8 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - kwok--kube-apiserver:6443 -- job_name: "kube-controller-manager" + - kwok--kube-controller-manager:10257 +- job_name: "kube-scheduler" scheme: https honor_timestamps: true metrics_path: /metrics @@ -52,24 +53,17 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - kwok--kube-controller-manager:10257 -- job_name: "kube-scheduler" - scheme: https + - kwok--kube-scheduler:10259 +- job_name: "prometheus" + scheme: http honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true static_configs: - targets: - - kwok--kube-scheduler:10259 -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://kwok--kwok-controller:10247/discovery/prometheus -- job_name: "kwok-controller" + - 127.0.0.1:9090 +- job_name: "etcd" scheme: http honor_timestamps: true metrics_path: /metrics @@ -77,24 +71,27 @@ scrape_configs: enable_http2: true static_configs: - targets: - - kwok--kwok-controller:10247 -- job_name: "prometheus" - scheme: http + - kwok--etcd:2379 +- job_name: "kube-apiserver" + scheme: https honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:9090 + - kwok--kube-apiserver:6443 EOF -chmod 0644 /workdir/clusters//prometheus.yaml +podman pull docker.io/prom/prometheus:v2.44.0 +podman pull localhost/kwok:test podman pull registry.k8s.io/etcd:3.5.9-0 podman pull registry.k8s.io/kube-apiserver:v1.28.0 -podman pull localhost/kwok:test podman pull registry.k8s.io/kube-controller-manager:v1.28.0 podman pull registry.k8s.io/kube-scheduler:v1.28.0 -podman pull docker.io/prom/prometheus:v2.44.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: @@ -141,14 +138,14 @@ EOF podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --log-level=debug --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/extras/apiserver:/extras/tmp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --v=5 --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local -podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/controller-manager:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.28.0 --v=5 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 -podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/scheduler:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.28.0 --v=5 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 -podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/extras/controller:/extras/tmp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro --env=TEST_KEY=TEST_VALUE localhost/kwok:test --manage-all-nodes=true --v=-4 --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 -podman create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --requires=kwok--etcd --requires=kwok--kube-apiserver --requires=kwok--kube-controller-manager --requires=kwok--kube-scheduler --requires=kwok--kwok-controller --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/extras/prometheus:/extras/tmp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.44.0 --log.level=debug --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 +podman create --name=kwok--kwok-controller --pull=never --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=600000ms --node-monitor-grace-period=3600000ms --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +podman create --name=kwok--prometheus --pull=never --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 # Add context kwok- to ~/.kube/config podman start kwok--etcd podman start kwok--kube-apiserver +podman start kwok--kwok-controller podman start kwok--kube-controller-manager podman start kwok--kube-scheduler -podman start kwok--kwok-controller podman start kwok--prometheus diff --git a/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt index f130bbdb28..b270c6c357 100644 --- a/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt @@ -12,12 +12,8 @@ kind: TracingConfiguration endpoint: kwok--jaeger:4317 samplingRatePerMillion: 1000000 EOF -cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml -cat <>/workdir/clusters//scheduler.yaml -clientConnection: - kubeconfig: "~/.kube/config" -EOF -cat </workdir/clusters//prometheus.yaml +mkdir -p /workdir/clusters//files/prometheus/etc/prometheus +cat </workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml global: scrape_interval: 15s scrape_timeout: 10s @@ -32,7 +28,7 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "etcd" +- job_name: "kwok-controller" scheme: http honor_timestamps: true metrics_path: /metrics @@ -40,8 +36,8 @@ scrape_configs: enable_http2: true static_configs: - targets: - - kwok--etcd:2379 -- job_name: "kube-apiserver" + - kwok--kwok-controller:10247 +- job_name: "kube-controller-manager" scheme: https honor_timestamps: true metrics_path: /metrics @@ -53,8 +49,8 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - kwok--kube-apiserver:6443 -- job_name: "kube-controller-manager" + - kwok--kube-controller-manager:10257 +- job_name: "kube-scheduler" scheme: https honor_timestamps: true metrics_path: /metrics @@ -66,24 +62,17 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - kwok--kube-controller-manager:10257 -- job_name: "kube-scheduler" - scheme: https + - kwok--kube-scheduler:10259 +- job_name: "prometheus" + scheme: http honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true - tls_config: - cert_file: "/etc/kubernetes/pki/admin.crt" - key_file: "/etc/kubernetes/pki/admin.key" - insecure_skip_verify: true static_configs: - targets: - - kwok--kube-scheduler:10259 -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://kwok--kwok-controller:10247/discovery/prometheus -- job_name: "kwok-controller" + - 127.0.0.1:9090 +- job_name: "etcd" scheme: http honor_timestamps: true metrics_path: /metrics @@ -91,26 +80,29 @@ scrape_configs: enable_http2: true static_configs: - targets: - - kwok--kwok-controller:10247 -- job_name: "prometheus" - scheme: http + - kwok--etcd:2379 +- job_name: "kube-apiserver" + scheme: https honor_timestamps: true metrics_path: /metrics follow_redirects: true enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true static_configs: - targets: - - 127.0.0.1:9090 + - kwok--kube-apiserver:6443 EOF -chmod 0644 /workdir/clusters//prometheus.yaml +podman pull docker.io/jaegertracing/all-in-one:1.45.0 +podman pull docker.io/kubernetesui/dashboard:v2.7.0 +podman pull docker.io/prom/prometheus:v2.44.0 +podman pull localhost/kwok:test podman pull registry.k8s.io/etcd:3.5.9-0 podman pull registry.k8s.io/kube-apiserver:v1.28.0 -podman pull localhost/kwok:test podman pull registry.k8s.io/kube-controller-manager:v1.28.0 podman pull registry.k8s.io/kube-scheduler:v1.28.0 -podman pull docker.io/kubernetesui/dashboard:v2.7.0 -podman pull docker.io/prom/prometheus:v2.44.0 -podman pull docker.io/jaegertracing/all-in-one:1.45.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: @@ -158,17 +150,17 @@ podman network create kwok- --label=io.podman.compose.project=kwok podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.9-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug podman create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.45.0 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --log-level=debug podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --requires=kwok--jaeger --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.28.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml --v=4 -podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 -podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.28.0 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 -podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml:ro localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 --v=DEBUG -podman create --name=kwok--dashboard --pull=never --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config --insecure-port=8000 -podman create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --requires=kwok--etcd --requires=kwok--kube-apiserver --requires=kwok--kube-controller-manager --requires=kwok--kube-scheduler --requires=kwok--kwok-controller --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug +podman create --name=kwok--kwok-controller --pull=never --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//kwok.yaml:~/.kwok/kwok.yaml localhost/kwok:test --manage-all-nodes=true --kubeconfig=~/.kube/config --config=~/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=1200 +podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-controller-manager:v1.28.0 --node-monitor-period=600000ms --node-monitor-grace-period=3600000ms --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ registry.k8s.io/kube-scheduler:v1.28.0 --kubeconfig=~/.kube/config --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=1000 --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics +podman create --name=kwok--dashboard --pull=never --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=8000:8000/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --insecure-port=8000 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --metrics-provider=none --system-banner=Welcome to kwok- --kubeconfig=~/.kube/config +podman create --name=kwok--prometheus --pull=never --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config --volume=/workdir/clusters//pki:/etc/kubernetes/pki/ --volume=/workdir/clusters//files/prometheus/etc/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml docker.io/prom/prometheus:v2.44.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 # Add context kwok- to ~/.kube/config podman start kwok--etcd podman start kwok--jaeger podman start kwok--kube-apiserver +podman start kwok--kwok-controller podman start kwok--kube-controller-manager podman start kwok--kube-scheduler -podman start kwok--kwok-controller podman start kwok--dashboard podman start kwok--prometheus