From 63d0e9f31a77b6e3967f2290008354aa30e4fd8d Mon Sep 17 00:00:00 2001 From: Shiming Zhang Date: Fri, 24 Nov 2023 14:25:22 +0800 Subject: [PATCH] Add metrics for components --- .../v1alpha1/kwokctl_configuration_types.go | 25 +++ .../config/v1alpha1/zz_generated.deepcopy.go | 26 +++ .../kwokctl_configuration_types.go | 25 +++ .../zz_generated.conversion.go | 46 +++++ .../internalversion/zz_generated.deepcopy.go | 26 +++ pkg/kwokctl/components/etcd.go | 17 ++ pkg/kwokctl/components/kube_apiserver.go | 30 +++ .../components/kube_controller_manager.go | 30 +++ pkg/kwokctl/components/kube_scheduler.go | 30 +++ pkg/kwokctl/components/kwok_controller.go | 47 ++++- pkg/kwokctl/components/prometheus.go | 38 ++++ pkg/kwokctl/components/prometheus.yaml.tpl | 47 +++++ pkg/kwokctl/runtime/binary/cluster.go | 42 +++-- pkg/kwokctl/runtime/binary/prometheus.go | 54 ------ .../runtime/binary/prometheus.yaml.tpl | 113 ----------- pkg/kwokctl/runtime/compose/cluster.go | 39 ++-- pkg/kwokctl/runtime/compose/prometheus.go | 48 ----- .../runtime/compose/prometheus.yaml.tpl | 113 ----------- pkg/kwokctl/runtime/kind/cluster.go | 64 ++++++- pkg/kwokctl/runtime/kind/prometheus.go | 24 --- pkg/kwokctl/runtime/kind/prometheus.yaml.tpl | 87 --------- site/content/en/docs/generated/apis.md | 124 ++++++++++++ .../binary/create_cluster_with_extra.txt | 50 ++--- .../binary/create_cluster_with_verbosity.txt | 50 ++--- .../docker/create_cluster_with_extra.txt | 62 +++--- .../docker/create_cluster_with_verbosity.txt | 66 +++---- .../kind-podman/create_cluster_with_extra.txt | 178 +++++++++--------- .../create_cluster_with_verbosity.txt | 178 +++++++++--------- .../kind/create_cluster_with_extra.txt | 178 +++++++++--------- .../kind/create_cluster_with_verbosity.txt | 178 +++++++++--------- .../nerdctl/create_cluster_with_extra.txt | 62 +++--- .../nerdctl/create_cluster_with_verbosity.txt | 66 +++---- .../podman/create_cluster_with_extra.txt | 62 +++--- .../podman/create_cluster_with_verbosity.txt | 66 +++---- 34 files changed, 1215 insertions(+), 1076 deletions(-) create mode 100644 pkg/kwokctl/components/prometheus.yaml.tpl delete mode 100644 pkg/kwokctl/runtime/binary/prometheus.go delete mode 100644 pkg/kwokctl/runtime/binary/prometheus.yaml.tpl delete mode 100644 pkg/kwokctl/runtime/compose/prometheus.go delete mode 100644 pkg/kwokctl/runtime/compose/prometheus.yaml.tpl delete mode 100644 pkg/kwokctl/runtime/kind/prometheus.go delete mode 100644 pkg/kwokctl/runtime/kind/prometheus.yaml.tpl diff --git a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go index 41789568f7..1d48f6ce2a 100644 --- a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go +++ b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go @@ -460,6 +460,12 @@ type Component struct { // +optional Volumes []Volume `json:"volumes,omitempty"` + // Metric is the metric of the component. + Metric *ComponentMetric `json:"metric,omitempty"` + + // MetricsDiscoveries is the metrics discovery of the component. + MetricsDiscoveries []ComponentMetric `json:"metricsDiscoveries,omitempty"` + // Version is the version of the component. // +optional Version string `json:"version,omitempty"` @@ -494,6 +500,25 @@ type Port struct { Protocol Protocol `json:"protocol,omitempty"` } +// ComponentMetric represents a metric of a component. +type ComponentMetric struct { + // Name is the name of the metrics discovery. + Name string `json:"name,omitempty"` + // Scheme is the scheme of the metric. + Scheme string `json:"scheme"` + // Host is the host of the metric. + Host string `json:"host"` + // Path is the path of the metric. + Path string `json:"path"` + + // CertPath is the cert path of the metric. + CertPath string `json:"certPath,omitempty"` + // KeyPath is the key path of the metric. + KeyPath string `json:"keyPath,omitempty"` + // InsecureSkipVerify is the flag to skip verify the metric. + InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` +} + // Protocol defines network protocols supported for things like component ports. // +enum type Protocol string diff --git a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index 443fc8af2b..74a2f23339 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -62,6 +62,16 @@ func (in *Component) DeepCopyInto(out *Component) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Metric != nil { + in, out := &in.Metric, &out.Metric + *out = new(ComponentMetric) + **out = **in + } + if in.MetricsDiscoveries != nil { + in, out := &in.MetricsDiscoveries, &out.MetricsDiscoveries + *out = make([]ComponentMetric, len(*in)) + copy(*out, *in) + } return } @@ -75,6 +85,22 @@ func (in *Component) DeepCopy() *Component { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ComponentMetric) DeepCopyInto(out *ComponentMetric) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentMetric. +func (in *ComponentMetric) DeepCopy() *ComponentMetric { + if in == nil { + return nil + } + out := new(ComponentMetric) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ComponentPatches) DeepCopyInto(out *ComponentPatches) { *out = *in diff --git a/pkg/apis/internalversion/kwokctl_configuration_types.go b/pkg/apis/internalversion/kwokctl_configuration_types.go index 5aad5fd08b..4cf78a2eca 100644 --- a/pkg/apis/internalversion/kwokctl_configuration_types.go +++ b/pkg/apis/internalversion/kwokctl_configuration_types.go @@ -304,6 +304,12 @@ type Component struct { // Volumes is a list of named volumes that can be mounted by containers belonging to the component. Volumes []Volume + // Metric is the metric of the component. + Metric *ComponentMetric + + // MetricsDiscoveries is the metrics discovery of the component. + MetricsDiscoveries []ComponentMetric + // Version is the version of the component. Version string } @@ -331,6 +337,25 @@ type Port struct { Protocol Protocol } +// ComponentMetric represents a metric of a component. +type ComponentMetric struct { + // Name is the name of the metrics discovery. + Name string + // Scheme is the scheme of the metric. + Scheme string + // Host is the host of the metric. + Host string + // Path is the path of the metric. + Path string + + // CertPath is the cert path of the metric. + CertPath string + // KeyPath is the key path of the metric. + KeyPath string + // InsecureSkipVerify is the flag to skip verify the metric. + InsecureSkipVerify bool +} + // Protocol defines network protocols supported for things like component ports. type Protocol string diff --git a/pkg/apis/internalversion/zz_generated.conversion.go b/pkg/apis/internalversion/zz_generated.conversion.go index 31cf53b27b..b71eb0f361 100644 --- a/pkg/apis/internalversion/zz_generated.conversion.go +++ b/pkg/apis/internalversion/zz_generated.conversion.go @@ -159,6 +159,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*ComponentMetric)(nil), (*configv1alpha1.ComponentMetric)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_internalversion_ComponentMetric_To_v1alpha1_ComponentMetric(a.(*ComponentMetric), b.(*configv1alpha1.ComponentMetric), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*configv1alpha1.ComponentMetric)(nil), (*ComponentMetric)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_ComponentMetric_To_internalversion_ComponentMetric(a.(*configv1alpha1.ComponentMetric), b.(*ComponentMetric), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*ComponentPatches)(nil), (*configv1alpha1.ComponentPatches)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_internalversion_ComponentPatches_To_v1alpha1_ComponentPatches(a.(*ComponentPatches), b.(*configv1alpha1.ComponentPatches), scope) }); err != nil { @@ -928,6 +938,8 @@ func autoConvert_internalversion_Component_To_v1alpha1_Component(in *Component, } else { out.Volumes = nil } + out.Metric = (*configv1alpha1.ComponentMetric)(unsafe.Pointer(in.Metric)) + out.MetricsDiscoveries = *(*[]configv1alpha1.ComponentMetric)(unsafe.Pointer(&in.MetricsDiscoveries)) out.Version = in.Version return nil } @@ -959,6 +971,8 @@ func autoConvert_v1alpha1_Component_To_internalversion_Component(in *configv1alp } else { out.Volumes = nil } + out.Metric = (*ComponentMetric)(unsafe.Pointer(in.Metric)) + out.MetricsDiscoveries = *(*[]ComponentMetric)(unsafe.Pointer(&in.MetricsDiscoveries)) out.Version = in.Version return nil } @@ -968,6 +982,38 @@ func Convert_v1alpha1_Component_To_internalversion_Component(in *configv1alpha1. return autoConvert_v1alpha1_Component_To_internalversion_Component(in, out, s) } +func autoConvert_internalversion_ComponentMetric_To_v1alpha1_ComponentMetric(in *ComponentMetric, out *configv1alpha1.ComponentMetric, s conversion.Scope) error { + out.Name = in.Name + out.Scheme = in.Scheme + out.Host = in.Host + out.Path = in.Path + out.CertPath = in.CertPath + out.KeyPath = in.KeyPath + out.InsecureSkipVerify = in.InsecureSkipVerify + return nil +} + +// Convert_internalversion_ComponentMetric_To_v1alpha1_ComponentMetric is an autogenerated conversion function. +func Convert_internalversion_ComponentMetric_To_v1alpha1_ComponentMetric(in *ComponentMetric, out *configv1alpha1.ComponentMetric, s conversion.Scope) error { + return autoConvert_internalversion_ComponentMetric_To_v1alpha1_ComponentMetric(in, out, s) +} + +func autoConvert_v1alpha1_ComponentMetric_To_internalversion_ComponentMetric(in *configv1alpha1.ComponentMetric, out *ComponentMetric, s conversion.Scope) error { + out.Name = in.Name + out.Scheme = in.Scheme + out.Host = in.Host + out.Path = in.Path + out.CertPath = in.CertPath + out.KeyPath = in.KeyPath + out.InsecureSkipVerify = in.InsecureSkipVerify + return nil +} + +// Convert_v1alpha1_ComponentMetric_To_internalversion_ComponentMetric is an autogenerated conversion function. +func Convert_v1alpha1_ComponentMetric_To_internalversion_ComponentMetric(in *configv1alpha1.ComponentMetric, out *ComponentMetric, s conversion.Scope) error { + return autoConvert_v1alpha1_ComponentMetric_To_internalversion_ComponentMetric(in, out, s) +} + func autoConvert_internalversion_ComponentPatches_To_v1alpha1_ComponentPatches(in *ComponentPatches, out *configv1alpha1.ComponentPatches, s conversion.Scope) error { out.Name = in.Name out.ExtraArgs = *(*[]configv1alpha1.ExtraArgs)(unsafe.Pointer(&in.ExtraArgs)) diff --git a/pkg/apis/internalversion/zz_generated.deepcopy.go b/pkg/apis/internalversion/zz_generated.deepcopy.go index 2c03bf13fb..50de2d47f0 100644 --- a/pkg/apis/internalversion/zz_generated.deepcopy.go +++ b/pkg/apis/internalversion/zz_generated.deepcopy.go @@ -304,6 +304,16 @@ func (in *Component) DeepCopyInto(out *Component) { *out = make([]Volume, len(*in)) copy(*out, *in) } + if in.Metric != nil { + in, out := &in.Metric, &out.Metric + *out = new(ComponentMetric) + **out = **in + } + if in.MetricsDiscoveries != nil { + in, out := &in.MetricsDiscoveries, &out.MetricsDiscoveries + *out = make([]ComponentMetric, len(*in)) + copy(*out, *in) + } return } @@ -317,6 +327,22 @@ func (in *Component) DeepCopy() *Component { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ComponentMetric) DeepCopyInto(out *ComponentMetric) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentMetric. +func (in *ComponentMetric) DeepCopy() *ComponentMetric { + if in == nil { + return nil + } + out := new(ComponentMetric) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ComponentPatches) DeepCopyInto(out *ComponentPatches) { *out = *in diff --git a/pkg/kwokctl/components/etcd.go b/pkg/kwokctl/components/etcd.go index 21c551af57..ea33a1716d 100644 --- a/pkg/kwokctl/components/etcd.go +++ b/pkg/kwokctl/components/etcd.go @@ -23,6 +23,7 @@ import ( "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/version" ) @@ -31,6 +32,7 @@ type BuildEtcdComponentConfig struct { Runtime string Binary string Image string + ProjectName string Version version.Version DataPath string Workdir string @@ -67,6 +69,8 @@ func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversio } etcdArgs = append(etcdArgs, extraArgsToStrings(conf.ExtraArgs)...) + var metric *internalversion.ComponentMetric + if getRuntimeKind(conf.Runtime) != runtimeKindRaw { // TODO: use a volume for the data path // volumes = append(volumes, @@ -104,6 +108,12 @@ func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversio "--listen-client-urls=http://"+conf.BindAddress+":2379", "--initial-cluster=node0=http://"+conf.BindAddress+":2380", ) + + metric = &internalversion.ComponentMetric{ + Scheme: "http", + Host: conf.ProjectName + "-" + consts.ComponentEtcd + ":2379", + Path: "/metrics", + } } else { etcdPeerPortStr := format.String(conf.PeerPort) etcdClientPortStr := format.String(conf.Port) @@ -115,6 +125,12 @@ func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversio "--listen-client-urls=http://"+conf.BindAddress+":"+etcdClientPortStr, "--initial-cluster=node0=http://"+conf.BindAddress+":"+etcdPeerPortStr, ) + + metric = &internalversion.ComponentMetric{ + Scheme: "http", + Host: net.LocalAddress + ":" + etcdClientPortStr, + Path: "/metrics", + } } if conf.Version.GTE(version.NewVersion(3, 4, 0)) { @@ -144,6 +160,7 @@ func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversio Args: etcdArgs, Binary: conf.Binary, Ports: ports, + Metric: metric, Image: conf.Image, WorkDir: conf.Workdir, Envs: envs, diff --git a/pkg/kwokctl/components/kube_apiserver.go b/pkg/kwokctl/components/kube_apiserver.go index a79ef8d475..55e99e47b5 100644 --- a/pkg/kwokctl/components/kube_apiserver.go +++ b/pkg/kwokctl/components/kube_apiserver.go @@ -24,12 +24,14 @@ import ( "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/version" ) // BuildKubeApiserverComponentConfig is the configuration for building a kube-apiserver component. type BuildKubeApiserverComponentConfig struct { Runtime string + ProjectName string Binary string Image string Version version.Version @@ -121,6 +123,7 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon var ports []internalversion.Port var volumes []internalversion.Volume volumes = append(volumes, conf.ExtraVolumes...) + var metric *internalversion.ComponentMetric if getRuntimeKind(conf.Runtime) != runtimeKindRaw { kubeApiserverArgs = append(kubeApiserverArgs, @@ -173,6 +176,14 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon "--service-account-signing-key-file=/etc/kubernetes/pki/admin.key", "--service-account-issuer=https://kubernetes.default.svc.cluster.local", ) + metric = &internalversion.ComponentMetric{ + Scheme: "https", + Host: conf.ProjectName + "-" + consts.ComponentKubeApiserver + ":6443", + Path: "/metrics", + CertPath: "/etc/kubernetes/pki/admin.crt", + KeyPath: "/etc/kubernetes/pki/admin.key", + InsecureSkipVerify: true, + } } else { kubeApiserverArgs = append(kubeApiserverArgs, "--bind-address="+conf.BindAddress, @@ -184,6 +195,14 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon "--service-account-signing-key-file="+conf.AdminKeyPath, "--service-account-issuer=https://kubernetes.default.svc.cluster.local", ) + metric = &internalversion.ComponentMetric{ + Scheme: "https", + Host: net.LocalAddress + ":" + format.String(conf.Port), + Path: "/metrics", + CertPath: conf.AdminCertPath, + KeyPath: conf.AdminKeyPath, + InsecureSkipVerify: true, + } } } else { if getRuntimeKind(conf.Runtime) != runtimeKindRaw { @@ -198,11 +217,21 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon "--insecure-bind-address="+conf.BindAddress, "--insecure-port=8080", ) + metric = &internalversion.ComponentMetric{ + Scheme: "http", + Host: conf.ProjectName + "-" + consts.ComponentKubeApiserver + ":8080", + Path: "/metrics", + } } else { kubeApiserverArgs = append(kubeApiserverArgs, "--insecure-bind-address="+conf.BindAddress, "--insecure-port="+format.String(conf.Port), ) + metric = &internalversion.ComponentMetric{ + Scheme: "http", + Host: net.LocalAddress + ":" + format.String(conf.Port), + Path: "/metrics", + } } } @@ -273,6 +302,7 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon Args: kubeApiserverArgs, Binary: conf.Binary, Image: conf.Image, + Metric: metric, WorkDir: conf.Workdir, Envs: envs, }, nil diff --git a/pkg/kwokctl/components/kube_controller_manager.go b/pkg/kwokctl/components/kube_controller_manager.go index ac9d126929..bd30327c51 100644 --- a/pkg/kwokctl/components/kube_controller_manager.go +++ b/pkg/kwokctl/components/kube_controller_manager.go @@ -23,12 +23,14 @@ import ( "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/version" ) // BuildKubeControllerManagerComponentConfig is the configuration for building a kube-controller-manager component. type BuildKubeControllerManagerComponentConfig struct { Runtime string + ProjectName string Binary string Image string Version version.Version @@ -77,6 +79,7 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen var volumes []internalversion.Volume volumes = append(volumes, conf.ExtraVolumes...) var ports []internalversion.Port + var metric *internalversion.ComponentMetric if getRuntimeKind(conf.Runtime) != runtimeKindRaw { volumes = append(volumes, @@ -131,11 +134,27 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen }, ) } + metric = &internalversion.ComponentMetric{ + Scheme: "https", + Host: conf.ProjectName + "-" + consts.ComponentKubeControllerManager + ":10257", + Path: "/metrics", + CertPath: "/etc/kubernetes/pki/admin.crt", + KeyPath: "/etc/kubernetes/pki/admin.key", + InsecureSkipVerify: true, + } } else { kubeControllerManagerArgs = append(kubeControllerManagerArgs, "--bind-address="+conf.BindAddress, "--secure-port="+format.String(conf.Port), ) + metric = &internalversion.ComponentMetric{ + Scheme: "https", + Host: net.LocalAddress + ":" + format.String(conf.Port), + Path: "/metrics", + CertPath: conf.AdminCertPath, + KeyPath: conf.AdminKeyPath, + InsecureSkipVerify: true, + } } // TODO: Support disable insecure port @@ -157,11 +176,21 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen }, ) } + metric = &internalversion.ComponentMetric{ + Scheme: "http", + Host: conf.ProjectName + "-" + consts.ComponentKubeControllerManager + ":10252", + Path: "/metrics", + } } else { kubeControllerManagerArgs = append(kubeControllerManagerArgs, "--address="+conf.BindAddress, "--port="+format.String(conf.Port), ) + metric = &internalversion.ComponentMetric{ + Scheme: "http", + Host: net.LocalAddress + ":" + format.String(conf.Port), + Path: "/metrics", + } } kubeControllerManagerArgs = append(kubeControllerManagerArgs, @@ -210,6 +239,7 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen Binary: conf.Binary, Image: conf.Image, WorkDir: conf.Workdir, + Metric: metric, Envs: envs, }, nil } diff --git a/pkg/kwokctl/components/kube_scheduler.go b/pkg/kwokctl/components/kube_scheduler.go index 15978a4735..d5dca96116 100644 --- a/pkg/kwokctl/components/kube_scheduler.go +++ b/pkg/kwokctl/components/kube_scheduler.go @@ -21,12 +21,14 @@ import ( "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/version" ) // BuildKubeSchedulerComponentConfig is the configuration for building a kube-scheduler component. type BuildKubeSchedulerComponentConfig struct { Runtime string + ProjectName string Binary string Image string Version version.Version @@ -61,6 +63,7 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon var volumes []internalversion.Volume volumes = append(volumes, conf.ExtraVolumes...) var ports []internalversion.Port + var metric *internalversion.ComponentMetric if getRuntimeKind(conf.Runtime) != runtimeKindRaw { volumes = append(volumes, @@ -135,11 +138,27 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon }, ) } + metric = &internalversion.ComponentMetric{ + Scheme: "https", + Host: conf.ProjectName + "-" + consts.ComponentKubeScheduler + ":10259", + Path: "/metrics", + CertPath: "/etc/kubernetes/pki/admin.crt", + KeyPath: "/etc/kubernetes/pki/admin.key", + InsecureSkipVerify: true, + } } else { kubeSchedulerArgs = append(kubeSchedulerArgs, "--bind-address="+conf.BindAddress, "--secure-port="+format.String(conf.Port), ) + metric = &internalversion.ComponentMetric{ + Scheme: "https", + Host: net.LocalAddress + ":" + format.String(conf.Port), + Path: "/metrics", + CertPath: conf.AdminCertPath, + KeyPath: conf.AdminKeyPath, + InsecureSkipVerify: true, + } } // TODO: Support disable insecure port // kubeSchedulerArgs = append(kubeSchedulerArgs, @@ -160,11 +179,21 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon }, ) } + metric = &internalversion.ComponentMetric{ + Scheme: "http", + Host: conf.ProjectName + "-" + consts.ComponentKubeScheduler + ":10251", + Path: "/metrics", + } } else { kubeSchedulerArgs = append(kubeSchedulerArgs, "--address="+conf.BindAddress, "--port="+format.String(conf.Port), ) + metric = &internalversion.ComponentMetric{ + Scheme: "http", + Host: net.LocalAddress + ":" + format.String(conf.Port), + Path: "/metrics", + } } // TODO: Support disable secure port @@ -200,6 +229,7 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon Image: conf.Image, Ports: ports, WorkDir: conf.Workdir, + Metric: metric, Envs: envs, }, nil } diff --git a/pkg/kwokctl/components/kwok_controller.go b/pkg/kwokctl/components/kwok_controller.go index bb497889c0..8c888aa02b 100644 --- a/pkg/kwokctl/components/kwok_controller.go +++ b/pkg/kwokctl/components/kwok_controller.go @@ -23,12 +23,14 @@ import ( "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/version" ) // BuildKwokControllerComponentConfig is the configuration for building a kwok controller component. type BuildKwokControllerComponentConfig struct { Runtime string + ProjectName string Binary string Image string Version version.Version @@ -133,6 +135,33 @@ func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (comp ) } + var metricsHost string + switch getRuntimeKind(conf.Runtime) { + case runtimeKindRaw: + metricsHost = net.LocalAddress + ":" + format.String(conf.Port) + case runtimeKindContainer: + metricsHost = conf.ProjectName + "-" + consts.ComponentKwokController + ":10247" + case runtimeKindCluster: + metricsHost = net.LocalAddress + ":10247" + } + + var metric *internalversion.ComponentMetric + var metricsDiscoveries []internalversion.ComponentMetric + + if metricsHost != "" { + metric = &internalversion.ComponentMetric{ + Scheme: "http", + Host: metricsHost, + Path: "/metrics", + } + metricsDiscoveries = append(metricsDiscoveries, internalversion.ComponentMetric{ + Name: "kwok-service-discovery", + Scheme: "http", + Host: metricsHost, + Path: "/discovery/prometheus", + }) + } + if conf.Verbosity != log.LevelInfo { kwokControllerArgs = append(kwokControllerArgs, "--v="+format.String(conf.Verbosity)) } @@ -150,13 +179,15 @@ func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (comp Links: []string{ consts.ComponentKubeApiserver, }, - Ports: ports, - Command: []string{"kwok"}, - Volumes: volumes, - Args: kwokControllerArgs, - Binary: conf.Binary, - Image: conf.Image, - WorkDir: conf.Workdir, - Envs: envs, + Ports: ports, + Command: []string{"kwok"}, + Volumes: volumes, + Args: kwokControllerArgs, + Binary: conf.Binary, + Image: conf.Image, + Metric: metric, + MetricsDiscoveries: metricsDiscoveries, + WorkDir: conf.Workdir, + Envs: envs, } } diff --git a/pkg/kwokctl/components/prometheus.go b/pkg/kwokctl/components/prometheus.go index 0ed5e3242b..90b8c7cfa9 100644 --- a/pkg/kwokctl/components/prometheus.go +++ b/pkg/kwokctl/components/prometheus.go @@ -17,11 +17,20 @@ limitations under the License. package components import ( + "bytes" + "fmt" + "text/template" + + "github.com/Masterminds/sprig/v3" + "sigs.k8s.io/kwok/pkg/apis/internalversion" "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/version" + + _ "embed" ) // BuildPrometheusComponentConfig is the configuration for building a prometheus component. @@ -50,6 +59,8 @@ func BuildPrometheusComponent(conf BuildPrometheusComponentConfig) (component in var volumes []internalversion.Volume volumes = append(volumes, conf.ExtraVolumes...) var ports []internalversion.Port + var metric *internalversion.ComponentMetric + if getRuntimeKind(conf.Runtime) != runtimeKindRaw { volumes = append(volumes, internalversion.Volume{ @@ -85,6 +96,12 @@ func BuildPrometheusComponent(conf BuildPrometheusComponentConfig) (component in ) } + metric = &internalversion.ComponentMetric{ + Scheme: "http", + Host: net.LocalAddress + ":" + format.String(conf.Port), + Path: "/metrics", + } + if conf.Verbosity != log.LevelInfo { prometheusArgs = append(prometheusArgs, "--log.level="+log.ToLogSeverityLevel(conf.Verbosity)) } @@ -109,6 +126,27 @@ func BuildPrometheusComponent(conf BuildPrometheusComponentConfig) (component in Binary: conf.Binary, Image: conf.Image, WorkDir: conf.Workdir, + Metric: metric, Envs: envs, }, nil } + +//go:embed prometheus.yaml.tpl +var prometheusYamlTpl string + +var prometheusYamlTemplate = template.Must(template.New("prometheus_config").Funcs(sprig.TxtFuncMap()).Parse(prometheusYamlTpl)) + +// BuildPrometheus builds the prometheus yaml content. +func BuildPrometheus(conf BuildPrometheusConfig) (string, error) { + buf := bytes.NewBuffer(nil) + err := prometheusYamlTemplate.Execute(buf, conf) + if err != nil { + return "", fmt.Errorf("build prometheus error: %w", err) + } + return buf.String(), nil +} + +// BuildPrometheusConfig is the configuration for building the prometheus config +type BuildPrometheusConfig struct { + Components []internalversion.Component +} diff --git a/pkg/kwokctl/components/prometheus.yaml.tpl b/pkg/kwokctl/components/prometheus.yaml.tpl new file mode 100644 index 0000000000..64efe82161 --- /dev/null +++ b/pkg/kwokctl/components/prometheus.yaml.tpl @@ -0,0 +1,47 @@ +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +{{ range .Components }} +{{ range .MetricsDiscoveries }} +- job_name: {{ or .Name | quote }} + http_sd_configs: + - url: {{ .Scheme }}://{{ .Host }}{{ .Path }} +{{ end }} +{{ if .Metric }} +- job_name: {{ or .Metric.Name .Name | quote }} +{{ if eq .Metric.Scheme "https" }} + scheme: https + honor_timestamps: true + metrics_path: {{ .Metric.Path }} + follow_redirects: true + enable_http2: true + tls_config: + cert_file: {{ .Metric.CertPath | quote }} + key_file: {{ .Metric.KeyPath | quote }} + insecure_skip_verify: {{ .Metric.InsecureSkipVerify }} + static_configs: + - targets: + - {{ .Metric.Host }} +{{ else }} + scheme: http + honor_timestamps: true + metrics_path: {{ .Metric.Path }} + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - {{ .Metric.Host }} +{{ end }} +{{ end }} +{{ end }} diff --git a/pkg/kwokctl/runtime/binary/cluster.go b/pkg/kwokctl/runtime/binary/cluster.go index 8939d1e5ba..736b65f851 100644 --- a/pkg/kwokctl/runtime/binary/cluster.go +++ b/pkg/kwokctl/runtime/binary/cluster.go @@ -350,6 +350,11 @@ func (c *Cluster) Install(ctx context.Context) error { return err } + err = c.setupPrometheusConfig(ctx, env) + if err != nil { + return err + } + err = c.finishInstall(ctx, env) if err != nil { return err @@ -372,6 +377,7 @@ func (c *Cluster) addEtcd(ctx context.Context, env *env) (err error) { etcdComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentEtcd) etcdComponent, err := components.BuildEtcdComponent(components.BuildEtcdComponentConfig{ Runtime: conf.Runtime, + ProjectName: c.Name(), Workdir: env.workdir, Binary: etcdPath, Version: etcdVersion, @@ -429,6 +435,7 @@ func (c *Cluster) addKubeApiserver(ctx context.Context, env *env) (err error) { kubeApiserverComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeApiserver) kubeApiserverComponent, err := components.BuildKubeApiserverComponent(components.BuildKubeApiserverComponentConfig{ Runtime: conf.Runtime, + ProjectName: c.Name(), Workdir: env.workdir, Binary: kubeApiserverPath, Version: kubeApiserverVersion, @@ -483,6 +490,7 @@ func (c *Cluster) addKubeControllerManager(ctx context.Context, env *env) (err e kubeControllerManagerPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeControllerManager) kubeControllerManagerComponent, err := components.BuildKubeControllerManagerComponent(components.BuildKubeControllerManagerComponentConfig{ Runtime: conf.Runtime, + ProjectName: c.Name(), Workdir: env.workdir, Binary: kubeControllerManagerPath, Version: kubeControllerManagerVersion, @@ -543,6 +551,7 @@ func (c *Cluster) addKubeScheduler(ctx context.Context, env *env) (err error) { kubeSchedulerComponentPatches := runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentKubeScheduler) kubeSchedulerComponent, err := components.BuildKubeSchedulerComponent(components.BuildKubeSchedulerComponentConfig{ Runtime: conf.Runtime, + ProjectName: c.Name(), Workdir: env.workdir, Binary: kubeSchedulerPath, Version: kubeSchedulerVersion, @@ -584,6 +593,7 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ Runtime: conf.Runtime, + ProjectName: c.Name(), Workdir: env.workdir, Binary: kwokControllerPath, Version: kwokControllerVersion, @@ -608,24 +618,13 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { return nil } -func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { +func (c *Cluster) setupPrometheusConfig(_ context.Context, env *env) (err error) { conf := &env.kwokctlConfig.Options // Configure the prometheus if conf.PrometheusPort != 0 { - prometheusPath := c.GetBinPath(consts.ComponentPrometheus + conf.BinSuffix) - - prometheusData, err := BuildPrometheus(BuildPrometheusConfig{ - ProjectName: c.Name(), - SecurePort: conf.SecurePort, - AdminCrtPath: env.adminCertPath, - AdminKeyPath: env.adminKeyPath, - PrometheusPort: conf.PrometheusPort, - EtcdPort: conf.EtcdPort, - KubeApiserverPort: conf.KubeApiserverPort, - KubeControllerManagerPort: conf.KubeControllerManagerPort, - KubeSchedulerPort: conf.KubeSchedulerPort, - KwokControllerPort: conf.KwokControllerPort, + prometheusData, err := components.BuildPrometheus(components.BuildPrometheusConfig{ + Components: env.kwokctlConfig.Components, }) if err != nil { return fmt.Errorf("failed to generate prometheus yaml: %w", err) @@ -635,6 +634,21 @@ func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { if err != nil { return fmt.Errorf("failed to write prometheus yaml: %w", err) } + } + return nil +} + +func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options + + // Configure the prometheus + if conf.PrometheusPort != 0 { + prometheusPath := c.GetBinPath(consts.ComponentPrometheus + conf.BinSuffix) + + if err != nil { + return fmt.Errorf("failed to generate prometheus yaml: %w", err) + } + prometheusConfigPath := c.GetWorkdirPath(runtime.Prometheus) prometheusVersion, err := c.ParseVersionFromBinary(ctx, prometheusPath) if err != nil { diff --git a/pkg/kwokctl/runtime/binary/prometheus.go b/pkg/kwokctl/runtime/binary/prometheus.go deleted file mode 100644 index 4832951a68..0000000000 --- a/pkg/kwokctl/runtime/binary/prometheus.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package binary - -import ( - "bytes" - "fmt" - "text/template" - - _ "embed" -) - -//go:embed prometheus.yaml.tpl -var prometheusYamlTpl string - -var prometheusYamlTemplate = template.Must(template.New("prometheus_config").Parse(prometheusYamlTpl)) - -// BuildPrometheus builds the prometheus yaml content. -func BuildPrometheus(conf BuildPrometheusConfig) (string, error) { - buf := bytes.NewBuffer(nil) - err := prometheusYamlTemplate.Execute(buf, conf) - if err != nil { - return "", fmt.Errorf("build prometheus error: %w", err) - } - return buf.String(), nil -} - -// BuildPrometheusConfig is the configuration for building the prometheus config -type BuildPrometheusConfig struct { - ProjectName string - SecurePort bool - AdminCrtPath string - AdminKeyPath string - PrometheusPort uint32 - EtcdPort uint32 - KubeApiserverPort uint32 - KubeControllerManagerPort uint32 - KubeSchedulerPort uint32 - KwokControllerPort uint32 -} diff --git a/pkg/kwokctl/runtime/binary/prometheus.yaml.tpl b/pkg/kwokctl/runtime/binary/prometheus.yaml.tpl deleted file mode 100644 index 19700113ca..0000000000 --- a/pkg/kwokctl/runtime/binary/prometheus.yaml.tpl +++ /dev/null @@ -1,113 +0,0 @@ -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - follow_redirects: true - enable_http2: true - scheme: http - timeout: 10s - api_version: v2 - static_configs: - - targets: [] -scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://localhost:{{ .KwokControllerPort }}/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:{{ .PrometheusPort }} -- job_name: "etcd" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:{{ .EtcdPort }} -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:{{ .KwokControllerPort }} -{{ if .SecurePort }} -- job_name: "kube-apiserver" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "{{ .AdminCrtPath }}" - key_file: "{{ .AdminKeyPath }}" - insecure_skip_verify: true - static_configs: - - targets: - - localhost:{{ .KubeApiserverPort }} -- job_name: "kube-controller-manager" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "{{ .AdminCrtPath }}" - key_file: "{{ .AdminKeyPath }}" - insecure_skip_verify: true - static_configs: - - targets: - - localhost:{{ .KubeControllerManagerPort }} -- job_name: "kube-scheduler" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "{{ .AdminCrtPath }}" - key_file: "{{ .AdminKeyPath }}" - insecure_skip_verify: true - static_configs: - - targets: - - localhost:{{ .KubeSchedulerPort }} -{{ else }} -- job_name: "kube-apiserver" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:{{ .KubeApiserverPort }} -- job_name: "kube-controller-manager" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:{{ .KubeControllerManagerPort }} -- job_name: "kube-scheduler" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:{{ .KubeSchedulerPort }} -{{ end }} diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index 738439fd07..b46c2c3e46 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -339,11 +339,6 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - err = c.pullAllImages(ctx, env) - if err != nil { - return err - } - err = c.addEtcd(ctx, env) if err != nil { return err @@ -384,6 +379,16 @@ func (c *Cluster) Install(ctx context.Context) error { return err } + err = c.setupPrometheusConfig(ctx, env) + if err != nil { + return err + } + + err = c.pullAllImages(ctx, env) + if err != nil { + return err + } + err = c.finishInstall(ctx, env) if err != nil { return err @@ -408,6 +413,7 @@ func (c *Cluster) addEtcd(ctx context.Context, env *env) (err error) { } etcdComponent, err := components.BuildEtcdComponent(components.BuildEtcdComponentConfig{ Runtime: conf.Runtime, + ProjectName: c.Name(), Workdir: env.workdir, Image: conf.EtcdImage, Version: etcdVersion, @@ -459,6 +465,7 @@ func (c *Cluster) addKubeApiserver(ctx context.Context, env *env) (err error) { kubeApiserverComponent, err := components.BuildKubeApiserverComponent(components.BuildKubeApiserverComponentConfig{ Runtime: conf.Runtime, + ProjectName: c.Name(), Workdir: env.workdir, Image: conf.KubeApiserverImage, Version: kubeApiserverVersion, @@ -507,6 +514,7 @@ func (c *Cluster) addKubeControllerManager(ctx context.Context, env *env) (err e } kubeControllerManagerComponent, err := components.BuildKubeControllerManagerComponent(components.BuildKubeControllerManagerComponentConfig{ Runtime: conf.Runtime, + ProjectName: c.Name(), Workdir: env.workdir, Image: conf.KubeControllerManagerImage, Version: kubeControllerManagerVersion, @@ -561,6 +569,7 @@ func (c *Cluster) addKubeScheduler(ctx context.Context, env *env) (err error) { } kubeSchedulerComponent, err := components.BuildKubeSchedulerComponent(components.BuildKubeSchedulerComponentConfig{ Runtime: conf.Runtime, + ProjectName: c.Name(), Workdir: env.workdir, Image: conf.KubeSchedulerImage, Version: kubeSchedulerVersion, @@ -608,6 +617,7 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ Runtime: conf.Runtime, + ProjectName: c.Name(), Workdir: env.workdir, Image: conf.KwokControllerImage, Version: kwokControllerVersion, @@ -630,16 +640,13 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { return nil } -func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { +func (c *Cluster) setupPrometheusConfig(_ context.Context, env *env) (err error) { conf := &env.kwokctlConfig.Options // Configure the prometheus if conf.PrometheusPort != 0 { - prometheusData, err := BuildPrometheus(BuildPrometheusConfig{ - ProjectName: c.Name(), - SecurePort: conf.SecurePort, - AdminCrtPath: env.inClusterAdminCertPath, - AdminKeyPath: env.inClusterAdminKeyPath, + prometheusData, err := components.BuildPrometheus(components.BuildPrometheusConfig{ + Components: env.kwokctlConfig.Components, }) if err != nil { return fmt.Errorf("failed to generate prometheus yaml: %w", err) @@ -652,7 +659,17 @@ func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { if err != nil { return fmt.Errorf("failed to write prometheus yaml: %w", err) } + } + + return nil +} +func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options + + // Configure the prometheus + if conf.PrometheusPort != 0 { + prometheusConfigPath := c.GetWorkdirPath(runtime.Prometheus) prometheusVersion, err := c.ParseVersionFromImage(ctx, c.runtime, conf.PrometheusImage, "") if err != nil { return err diff --git a/pkg/kwokctl/runtime/compose/prometheus.go b/pkg/kwokctl/runtime/compose/prometheus.go deleted file mode 100644 index dfc4e4274c..0000000000 --- a/pkg/kwokctl/runtime/compose/prometheus.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package compose - -import ( - "bytes" - "fmt" - "text/template" - - _ "embed" -) - -//go:embed prometheus.yaml.tpl -var prometheusYamlTpl string - -var prometheusYamlTemplate = template.Must(template.New("prometheus_config").Parse(prometheusYamlTpl)) - -// BuildPrometheus builds the prometheus yaml content. -func BuildPrometheus(conf BuildPrometheusConfig) (string, error) { - buf := bytes.NewBuffer(nil) - err := prometheusYamlTemplate.Execute(buf, conf) - if err != nil { - return "", fmt.Errorf("build prometheus error: %w", err) - } - return buf.String(), nil -} - -// BuildPrometheusConfig is the configuration for building the prometheus -type BuildPrometheusConfig struct { - ProjectName string - SecurePort bool - AdminCrtPath string - AdminKeyPath string -} diff --git a/pkg/kwokctl/runtime/compose/prometheus.yaml.tpl b/pkg/kwokctl/runtime/compose/prometheus.yaml.tpl deleted file mode 100644 index 2901811b93..0000000000 --- a/pkg/kwokctl/runtime/compose/prometheus.yaml.tpl +++ /dev/null @@ -1,113 +0,0 @@ -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - follow_redirects: true - enable_http2: true - scheme: http - timeout: 10s - api_version: v2 - static_configs: - - targets: [] -scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://{{ .ProjectName }}-kwok-controller:10247/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:9090 -- job_name: "etcd" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "{{ .ProjectName }}-etcd:2379" -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "{{ .ProjectName }}-kwok-controller:10247" -{{ if .SecurePort }} -- job_name: "kube-apiserver" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "{{ .AdminCrtPath }}" - key_file: "{{ .AdminKeyPath }}" - insecure_skip_verify: true - static_configs: - - targets: - - "{{ .ProjectName }}-kube-apiserver:6443" -- job_name: "kube-controller-manager" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "{{ .AdminCrtPath }}" - key_file: "{{ .AdminKeyPath }}" - insecure_skip_verify: true - static_configs: - - targets: - - "{{ .ProjectName }}-kube-controller-manager:10257" -- job_name: "kube-scheduler" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: "{{ .AdminCrtPath }}" - key_file: "{{ .AdminKeyPath }}" - insecure_skip_verify: true - static_configs: - - targets: - - "{{ .ProjectName }}-kube-scheduler:10259" -{{ else }} -- job_name: "kube-apiserver" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "{{ .ProjectName }}-kube-apiserver:8080" -- job_name: "kube-controller-manager" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "{{ .ProjectName }}-kube-controller-manager:10252" -- job_name: "kube-scheduler" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "{{ .ProjectName }}-kube-scheduler:10251" -{{ end }} diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 31b4a9579f..814a3ed950 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -241,6 +241,11 @@ func (c *Cluster) Install(ctx context.Context) error { return err } + err = c.setupPrometheusConfig(ctx, env) + if err != nil { + return err + } + images, err := c.listAllImages(ctx) if err != nil { return err @@ -323,14 +328,8 @@ func (c *Cluster) addKind(ctx context.Context, env *env) (err error) { var prometheusPatches internalversion.ComponentPatches if conf.PrometheusPort != 0 { prometheusPatches = runtime.GetComponentPatches(env.kwokctlConfig, consts.ComponentPrometheus) - prometheusConfigPath := c.GetWorkdirPath(runtime.Prometheus) - err = c.WriteFile(prometheusConfigPath, []byte(prometheusYamlTpl)) - if err != nil { - return fmt.Errorf("failed to write prometheus yaml: %w", err) - } - prometheusPatches.ExtraVolumes = append(prometheusPatches.ExtraVolumes, internalversion.Volume{ Name: "prometheus-config", HostPath: prometheusConfigPath, @@ -400,6 +399,14 @@ func (c *Cluster) addKind(ctx context.Context, env *env) (err error) { func (c *Cluster) addEtcd(_ context.Context, env *env) (err error) { env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, internalversion.Component{ Name: consts.ComponentEtcd, + Metric: &internalversion.ComponentMetric{ + Scheme: "https", + Host: "127.0.0.1:2379", + Path: "/metrics", + CertPath: "/etc/kubernetes/pki/apiserver-etcd-client.crt", + KeyPath: "/etc/kubernetes/pki/apiserver-etcd-client.key", + InsecureSkipVerify: true, + }, }) return nil } @@ -407,6 +414,14 @@ func (c *Cluster) addEtcd(_ context.Context, env *env) (err error) { func (c *Cluster) addKubeApiserver(_ context.Context, env *env) (err error) { env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, internalversion.Component{ Name: consts.ComponentKubeApiserver, + Metric: &internalversion.ComponentMetric{ + Scheme: "https", + Host: "127.0.0.1:6443", + Path: "/metrics", + CertPath: "/etc/kubernetes/pki/admin.crt", + KeyPath: "/etc/kubernetes/pki/admin.key", + InsecureSkipVerify: true, + }, }) return nil } @@ -416,6 +431,14 @@ func (c *Cluster) addKubeControllerManager(_ context.Context, env *env) (err err if !conf.DisableKubeControllerManager { env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, internalversion.Component{ Name: consts.ComponentKubeControllerManager, + Metric: &internalversion.ComponentMetric{ + Scheme: "https", + Host: "127.0.0.1:10257", + Path: "/metrics", + CertPath: "/etc/kubernetes/pki/admin.crt", + KeyPath: "/etc/kubernetes/pki/admin.key", + InsecureSkipVerify: true, + }, }) } return nil @@ -426,6 +449,14 @@ func (c *Cluster) addKubeScheduler(_ context.Context, env *env) (err error) { if !conf.DisableKubeScheduler { env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, internalversion.Component{ Name: consts.ComponentKubeScheduler, + Metric: &internalversion.ComponentMetric{ + Scheme: "https", + Host: "127.0.0.1:10259", + Path: "/metrics", + CertPath: "/etc/kubernetes/pki/admin.crt", + KeyPath: "/etc/kubernetes/pki/admin.key", + InsecureSkipVerify: true, + }, }) } return nil @@ -457,6 +488,7 @@ func (c *Cluster) addKwokController(ctx context.Context, env *env) (err error) { kwokControllerComponent := components.BuildKwokControllerComponent(components.BuildKwokControllerComponentConfig{ Runtime: conf.Runtime, + ProjectName: c.Name(), Workdir: env.workdir, Image: conf.KwokControllerImage, Version: kwokControllerVersion, @@ -539,6 +571,26 @@ func (c *Cluster) addDashboard(ctx context.Context, env *env) (err error) { return nil } +func (c *Cluster) setupPrometheusConfig(_ context.Context, env *env) (err error) { + conf := &env.kwokctlConfig.Options + + // Configure the prometheus + if conf.PrometheusPort != 0 { + prometheusData, err := components.BuildPrometheus(components.BuildPrometheusConfig{ + Components: env.kwokctlConfig.Components, + }) + if err != nil { + return fmt.Errorf("failed to generate prometheus yaml: %w", err) + } + prometheusConfigPath := c.GetWorkdirPath(runtime.Prometheus) + err = c.WriteFile(prometheusConfigPath, []byte(prometheusData)) + if err != nil { + return fmt.Errorf("failed to write prometheus yaml: %w", err) + } + } + return nil +} + func (c *Cluster) addPrometheus(ctx context.Context, env *env) (err error) { conf := &env.kwokctlConfig.Options diff --git a/pkg/kwokctl/runtime/kind/prometheus.go b/pkg/kwokctl/runtime/kind/prometheus.go deleted file mode 100644 index 066b5f8c37..0000000000 --- a/pkg/kwokctl/runtime/kind/prometheus.go +++ /dev/null @@ -1,24 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kind - -import ( - _ "embed" -) - -//go:embed prometheus.yaml.tpl -var prometheusYamlTpl string diff --git a/pkg/kwokctl/runtime/kind/prometheus.yaml.tpl b/pkg/kwokctl/runtime/kind/prometheus.yaml.tpl deleted file mode 100644 index 70ad9947d9..0000000000 --- a/pkg/kwokctl/runtime/kind/prometheus.yaml.tpl +++ /dev/null @@ -1,87 +0,0 @@ -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - follow_redirects: true - enable_http2: true - scheme: http - timeout: 10s - api_version: v2 - static_configs: - - targets: [ ] -scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://localhost:10247/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "localhost:9090" -- job_name: "etcd" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/apiserver-etcd-client.crt - key_file: /etc/kubernetes/pki/apiserver-etcd-client.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:2379" -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "localhost:10247" -- job_name: "kube-apiserver" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:6443" -- job_name: "kube-controller-manager" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:10257" -- job_name: "kube-scheduler" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:10259" diff --git a/site/content/en/docs/generated/apis.md b/site/content/en/docs/generated/apis.md index 1d446bca66..e8d30a071b 100644 --- a/site/content/en/docs/generated/apis.md +++ b/site/content/en/docs/generated/apis.md @@ -1518,6 +1518,32 @@ string +metric + + +ComponentMetric + + + + +

Metric is the metric of the component.

+ + + + +metricsDiscoveries + + +[]ComponentMetric + + + + +

MetricsDiscoveries is the metrics discovery of the component.

+ + + + version string @@ -1530,6 +1556,104 @@ string +

+ComponentMetric + # +

+

+Appears on: +Component +

+

+

ComponentMetric represents a metric of a component.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+name + +string + + +

Name is the name of the metrics discovery.

+
+scheme + +string + + +

Scheme is the scheme of the metric.

+
+host + +string + + +

Host is the host of the metric.

+
+path + +string + + +

Path is the path of the metric.

+
+certPath + +string + + +

CertPath is the cert path of the metric.

+
+keyPath + +string + + +

KeyPath is the key path of the metric.

+
+insecureSkipVerify + +bool + + +

InsecureSkipVerify is the flag to skip verify the metric.

+

ComponentPatches # diff --git a/test/kwokctl/testdata/binary/create_cluster_with_extra.txt b/test/kwokctl/testdata/binary/create_cluster_with_extra.txt index 4c268f92a1..8962bdcad5 100644 --- a/test/kwokctl/testdata/binary/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/binary/create_cluster_with_extra.txt @@ -26,18 +26,6 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://localhost:32763/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:9090 - job_name: "etcd" scheme: http honor_timestamps: true @@ -46,16 +34,7 @@ scrape_configs: enable_http2: true static_configs: - targets: - - localhost:32765 -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:32763 + - 127.0.0.1:32765 - job_name: "kube-apiserver" scheme: https honor_timestamps: true @@ -68,7 +47,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - localhost:32764 + - 127.0.0.1:32764 - job_name: "kube-controller-manager" scheme: https honor_timestamps: true @@ -81,7 +60,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - localhost:32762 + - 127.0.0.1:32762 - job_name: "kube-scheduler" scheme: https honor_timestamps: true @@ -94,7 +73,28 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - localhost:32761 + - 127.0.0.1:32761 +- job_name: "kwok-service-discovery" + http_sd_configs: + - url: http://127.0.0.1:32763/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:32763 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 EOF cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 diff --git a/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt index 3fcb5c627a..20a5ffeeec 100644 --- a/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/binary/create_cluster_with_verbosity.txt @@ -40,18 +40,6 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://localhost:32763/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:9090 - job_name: "etcd" scheme: http honor_timestamps: true @@ -60,16 +48,7 @@ scrape_configs: enable_http2: true static_configs: - targets: - - localhost:32765 -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:32763 + - 127.0.0.1:32765 - job_name: "kube-apiserver" scheme: https honor_timestamps: true @@ -82,7 +61,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - localhost:32764 + - 127.0.0.1:32764 - job_name: "kube-controller-manager" scheme: https honor_timestamps: true @@ -95,7 +74,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - localhost:32761 + - 127.0.0.1:32761 - job_name: "kube-scheduler" scheme: https honor_timestamps: true @@ -108,7 +87,28 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - localhost:32760 + - 127.0.0.1:32760 +- job_name: "kwok-service-discovery" + http_sd_configs: + - url: http://127.0.0.1:32763/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:32763 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 EOF cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 diff --git a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt index 3b5342dea8..8353e51fc3 100644 --- a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt @@ -3,12 +3,6 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd -docker pull registry.k8s.io/etcd:3.5.9-0 -docker pull registry.k8s.io/kube-apiserver:v1.28.0 -docker pull localhost/kwok:test -docker pull registry.k8s.io/kube-controller-manager:v1.28.0 -docker pull registry.k8s.io/kube-scheduler:v1.28.0 -docker pull docker.io/prom/prometheus:v2.44.0 cat </workdir/clusters//prometheus.yaml global: scrape_interval: 15s @@ -24,18 +18,6 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://kwok--kwok-controller:10247/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:9090 - job_name: "etcd" scheme: http honor_timestamps: true @@ -44,16 +26,7 @@ scrape_configs: enable_http2: true static_configs: - targets: - - "kwok--etcd:2379" -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "kwok--kwok-controller:10247" + - kwok--etcd:2379 - job_name: "kube-apiserver" scheme: https honor_timestamps: true @@ -66,7 +39,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-apiserver:6443" + - kwok--kube-apiserver:6443 - job_name: "kube-controller-manager" scheme: https honor_timestamps: true @@ -79,7 +52,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-controller-manager:10257" + - kwok--kube-controller-manager:10257 - job_name: "kube-scheduler" scheme: https honor_timestamps: true @@ -92,9 +65,36 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-scheduler:10259" + - kwok--kube-scheduler:10259 +- job_name: "kwok-service-discovery" + http_sd_configs: + - url: http://kwok--kwok-controller:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--kwok-controller:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 EOF chmod 0644 /workdir/clusters//prometheus.yaml +docker pull registry.k8s.io/etcd:3.5.9-0 +docker pull registry.k8s.io/kube-apiserver:v1.28.0 +docker pull localhost/kwok:test +docker pull registry.k8s.io/kube-controller-manager:v1.28.0 +docker pull registry.k8s.io/kube-scheduler:v1.28.0 +docker pull docker.io/prom/prometheus:v2.44.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: diff --git a/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt index 23391014db..226e5e738e 100644 --- a/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/docker/create_cluster_with_verbosity.txt @@ -6,14 +6,6 @@ mkdir -p /workdir/clusters//logs touch /workdir/clusters//logs/audit.log cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml mkdir -p /workdir/clusters//etcd -docker pull registry.k8s.io/etcd:3.5.9-0 -docker pull registry.k8s.io/kube-apiserver:v1.28.0 -docker pull localhost/kwok:test -docker pull registry.k8s.io/kube-controller-manager:v1.28.0 -docker pull registry.k8s.io/kube-scheduler:v1.28.0 -docker pull docker.io/kubernetesui/dashboard:v2.7.0 -docker pull docker.io/prom/prometheus:v2.44.0 -docker pull docker.io/jaegertracing/all-in-one:1.45.0 cat </workdir/clusters//apiserver-tracing-config.yaml apiVersion: apiserver.config.k8s.io/v1alpha1 kind: TracingConfiguration @@ -40,18 +32,6 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://kwok--kwok-controller:10247/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:9090 - job_name: "etcd" scheme: http honor_timestamps: true @@ -60,16 +40,7 @@ scrape_configs: enable_http2: true static_configs: - targets: - - "kwok--etcd:2379" -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "kwok--kwok-controller:10247" + - kwok--etcd:2379 - job_name: "kube-apiserver" scheme: https honor_timestamps: true @@ -82,7 +53,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-apiserver:6443" + - kwok--kube-apiserver:6443 - job_name: "kube-controller-manager" scheme: https honor_timestamps: true @@ -95,7 +66,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-controller-manager:10257" + - kwok--kube-controller-manager:10257 - job_name: "kube-scheduler" scheme: https honor_timestamps: true @@ -108,9 +79,38 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-scheduler:10259" + - kwok--kube-scheduler:10259 +- job_name: "kwok-service-discovery" + http_sd_configs: + - url: http://kwok--kwok-controller:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--kwok-controller:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 EOF chmod 0644 /workdir/clusters//prometheus.yaml +docker pull registry.k8s.io/etcd:3.5.9-0 +docker pull registry.k8s.io/kube-apiserver:v1.28.0 +docker pull localhost/kwok:test +docker pull registry.k8s.io/kube-controller-manager:v1.28.0 +docker pull registry.k8s.io/kube-scheduler:v1.28.0 +docker pull docker.io/kubernetesui/dashboard:v2.7.0 +docker pull docker.io/prom/prometheus:v2.44.0 +docker pull docker.io/jaegertracing/all-in-one:1.45.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: diff --git a/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt b/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt index a78b9b8a7b..66d4a40393 100644 --- a/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt @@ -5,95 +5,6 @@ mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//pki/etcd mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//manifests -cat </workdir/clusters//prometheus.yaml -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - follow_redirects: true - enable_http2: true - scheme: http - timeout: 10s - api_version: v2 - static_configs: - - targets: [ ] -scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://localhost:10247/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "localhost:9090" -- job_name: "etcd" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/apiserver-etcd-client.crt - key_file: /etc/kubernetes/pki/apiserver-etcd-client.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:2379" -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "localhost:10247" -- job_name: "kube-apiserver" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:6443" -- job_name: "kube-controller-manager" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:10257" -- job_name: "kube-scheduler" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:10259" -EOF cat </workdir/clusters//kind.yaml kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 @@ -337,6 +248,95 @@ spec: name: volume-5 status: {} EOF +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10259 +- job_name: "kwok-service-discovery" + http_sd_configs: + - url: http://127.0.0.1:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF podman pull docker.io/kindest/node:v1.28.0 podman pull localhost/kwok:test podman pull docker.io/prom/prometheus:v2.44.0 diff --git a/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt index cd86793c9e..1fb66c56d8 100644 --- a/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt @@ -19,95 +19,6 @@ kind: TracingConfiguration endpoint: 0.0.0.0:4317 samplingRatePerMillion: 1000000 EOF -cat </workdir/clusters//prometheus.yaml -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - follow_redirects: true - enable_http2: true - scheme: http - timeout: 10s - api_version: v2 - static_configs: - - targets: [ ] -scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://localhost:10247/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "localhost:9090" -- job_name: "etcd" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/apiserver-etcd-client.crt - key_file: /etc/kubernetes/pki/apiserver-etcd-client.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:2379" -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "localhost:10247" -- job_name: "kube-apiserver" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:6443" -- job_name: "kube-controller-manager" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:10257" -- job_name: "kube-scheduler" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:10259" -EOF cat </workdir/clusters//kind.yaml kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 @@ -425,6 +336,95 @@ spec: runAsUser: 0 status: {} EOF +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10259 +- job_name: "kwok-service-discovery" + http_sd_configs: + - url: http://127.0.0.1:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF podman pull docker.io/kindest/node:v1.28.0 podman pull localhost/kwok:test podman pull docker.io/kubernetesui/dashboard:v2.7.0 diff --git a/test/kwokctl/testdata/kind/create_cluster_with_extra.txt b/test/kwokctl/testdata/kind/create_cluster_with_extra.txt index 67fed93285..4a6745dfd5 100644 --- a/test/kwokctl/testdata/kind/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/kind/create_cluster_with_extra.txt @@ -5,95 +5,6 @@ mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//pki/etcd mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//manifests -cat </workdir/clusters//prometheus.yaml -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - follow_redirects: true - enable_http2: true - scheme: http - timeout: 10s - api_version: v2 - static_configs: - - targets: [ ] -scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://localhost:10247/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "localhost:9090" -- job_name: "etcd" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/apiserver-etcd-client.crt - key_file: /etc/kubernetes/pki/apiserver-etcd-client.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:2379" -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "localhost:10247" -- job_name: "kube-apiserver" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:6443" -- job_name: "kube-controller-manager" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:10257" -- job_name: "kube-scheduler" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:10259" -EOF cat </workdir/clusters//kind.yaml kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 @@ -337,6 +248,95 @@ spec: name: volume-5 status: {} EOF +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10259 +- job_name: "kwok-service-discovery" + http_sd_configs: + - url: http://127.0.0.1:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF docker pull docker.io/kindest/node:v1.28.0 docker pull localhost/kwok:test docker pull docker.io/prom/prometheus:v2.44.0 diff --git a/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt index 98fb31d328..f65bc55228 100644 --- a/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt @@ -19,95 +19,6 @@ kind: TracingConfiguration endpoint: 0.0.0.0:4317 samplingRatePerMillion: 1000000 EOF -cat </workdir/clusters//prometheus.yaml -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - follow_redirects: true - enable_http2: true - scheme: http - timeout: 10s - api_version: v2 - static_configs: - - targets: [ ] -scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://localhost:10247/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "localhost:9090" -- job_name: "etcd" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/apiserver-etcd-client.crt - key_file: /etc/kubernetes/pki/apiserver-etcd-client.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:2379" -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "localhost:10247" -- job_name: "kube-apiserver" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:6443" -- job_name: "kube-controller-manager" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:10257" -- job_name: "kube-scheduler" - scheme: https - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - tls_config: - cert_file: /etc/kubernetes/pki/admin.crt - key_file: /etc/kubernetes/pki/admin.key - insecure_skip_verify: true - static_configs: - - targets: - - "localhost:10259" -EOF cat </workdir/clusters//kind.yaml kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 @@ -425,6 +336,95 @@ spec: runAsUser: 0 status: {} EOF +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10259 +- job_name: "kwok-service-discovery" + http_sd_configs: + - url: http://127.0.0.1:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF docker pull docker.io/kindest/node:v1.28.0 docker pull localhost/kwok:test docker pull docker.io/kubernetesui/dashboard:v2.7.0 diff --git a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt index 76f94cf567..53ec45a9e2 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt @@ -3,12 +3,6 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd -nerdctl pull registry.k8s.io/etcd:3.5.9-0 -nerdctl pull registry.k8s.io/kube-apiserver:v1.28.0 -nerdctl pull localhost/kwok:test -nerdctl pull registry.k8s.io/kube-controller-manager:v1.28.0 -nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 -nerdctl pull docker.io/prom/prometheus:v2.44.0 cat </workdir/clusters//prometheus.yaml global: scrape_interval: 15s @@ -24,18 +18,6 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://kwok--kwok-controller:10247/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:9090 - job_name: "etcd" scheme: http honor_timestamps: true @@ -44,16 +26,7 @@ scrape_configs: enable_http2: true static_configs: - targets: - - "kwok--etcd:2379" -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "kwok--kwok-controller:10247" + - kwok--etcd:2379 - job_name: "kube-apiserver" scheme: https honor_timestamps: true @@ -66,7 +39,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-apiserver:6443" + - kwok--kube-apiserver:6443 - job_name: "kube-controller-manager" scheme: https honor_timestamps: true @@ -79,7 +52,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-controller-manager:10257" + - kwok--kube-controller-manager:10257 - job_name: "kube-scheduler" scheme: https honor_timestamps: true @@ -92,9 +65,36 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-scheduler:10259" + - kwok--kube-scheduler:10259 +- job_name: "kwok-service-discovery" + http_sd_configs: + - url: http://kwok--kwok-controller:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--kwok-controller:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 EOF chmod 0644 /workdir/clusters//prometheus.yaml +nerdctl pull registry.k8s.io/etcd:3.5.9-0 +nerdctl pull registry.k8s.io/kube-apiserver:v1.28.0 +nerdctl pull localhost/kwok:test +nerdctl pull registry.k8s.io/kube-controller-manager:v1.28.0 +nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 +nerdctl pull docker.io/prom/prometheus:v2.44.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: diff --git a/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt index 837920ff8a..aca7572508 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster_with_verbosity.txt @@ -6,14 +6,6 @@ mkdir -p /workdir/clusters//logs touch /workdir/clusters//logs/audit.log cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml mkdir -p /workdir/clusters//etcd -nerdctl pull registry.k8s.io/etcd:3.5.9-0 -nerdctl pull registry.k8s.io/kube-apiserver:v1.28.0 -nerdctl pull localhost/kwok:test -nerdctl pull registry.k8s.io/kube-controller-manager:v1.28.0 -nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 -nerdctl pull docker.io/kubernetesui/dashboard:v2.7.0 -nerdctl pull docker.io/prom/prometheus:v2.44.0 -nerdctl pull docker.io/jaegertracing/all-in-one:1.45.0 cat </workdir/clusters//apiserver-tracing-config.yaml apiVersion: apiserver.config.k8s.io/v1alpha1 kind: TracingConfiguration @@ -40,18 +32,6 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://kwok--kwok-controller:10247/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:9090 - job_name: "etcd" scheme: http honor_timestamps: true @@ -60,16 +40,7 @@ scrape_configs: enable_http2: true static_configs: - targets: - - "kwok--etcd:2379" -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "kwok--kwok-controller:10247" + - kwok--etcd:2379 - job_name: "kube-apiserver" scheme: https honor_timestamps: true @@ -82,7 +53,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-apiserver:6443" + - kwok--kube-apiserver:6443 - job_name: "kube-controller-manager" scheme: https honor_timestamps: true @@ -95,7 +66,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-controller-manager:10257" + - kwok--kube-controller-manager:10257 - job_name: "kube-scheduler" scheme: https honor_timestamps: true @@ -108,9 +79,38 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-scheduler:10259" + - kwok--kube-scheduler:10259 +- job_name: "kwok-service-discovery" + http_sd_configs: + - url: http://kwok--kwok-controller:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--kwok-controller:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 EOF chmod 0644 /workdir/clusters//prometheus.yaml +nerdctl pull registry.k8s.io/etcd:3.5.9-0 +nerdctl pull registry.k8s.io/kube-apiserver:v1.28.0 +nerdctl pull localhost/kwok:test +nerdctl pull registry.k8s.io/kube-controller-manager:v1.28.0 +nerdctl pull registry.k8s.io/kube-scheduler:v1.28.0 +nerdctl pull docker.io/kubernetesui/dashboard:v2.7.0 +nerdctl pull docker.io/prom/prometheus:v2.44.0 +nerdctl pull docker.io/jaegertracing/all-in-one:1.45.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: diff --git a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt index 23ddd4c3d7..dafd31cfc1 100644 --- a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt @@ -3,12 +3,6 @@ mkdir -p /workdir/clusters/ mkdir -p /workdir/clusters//pki # Generate PKI to /workdir/clusters//pki mkdir -p /workdir/clusters//etcd -podman pull registry.k8s.io/etcd:3.5.9-0 -podman pull registry.k8s.io/kube-apiserver:v1.28.0 -podman pull localhost/kwok:test -podman pull registry.k8s.io/kube-controller-manager:v1.28.0 -podman pull registry.k8s.io/kube-scheduler:v1.28.0 -podman pull docker.io/prom/prometheus:v2.44.0 cat </workdir/clusters//prometheus.yaml global: scrape_interval: 15s @@ -24,18 +18,6 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://kwok--kwok-controller:10247/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:9090 - job_name: "etcd" scheme: http honor_timestamps: true @@ -44,16 +26,7 @@ scrape_configs: enable_http2: true static_configs: - targets: - - "kwok--etcd:2379" -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "kwok--kwok-controller:10247" + - kwok--etcd:2379 - job_name: "kube-apiserver" scheme: https honor_timestamps: true @@ -66,7 +39,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-apiserver:6443" + - kwok--kube-apiserver:6443 - job_name: "kube-controller-manager" scheme: https honor_timestamps: true @@ -79,7 +52,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-controller-manager:10257" + - kwok--kube-controller-manager:10257 - job_name: "kube-scheduler" scheme: https honor_timestamps: true @@ -92,9 +65,36 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-scheduler:10259" + - kwok--kube-scheduler:10259 +- job_name: "kwok-service-discovery" + http_sd_configs: + - url: http://kwok--kwok-controller:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--kwok-controller:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 EOF chmod 0644 /workdir/clusters//prometheus.yaml +podman pull registry.k8s.io/etcd:3.5.9-0 +podman pull registry.k8s.io/kube-apiserver:v1.28.0 +podman pull localhost/kwok:test +podman pull registry.k8s.io/kube-controller-manager:v1.28.0 +podman pull registry.k8s.io/kube-scheduler:v1.28.0 +podman pull docker.io/prom/prometheus:v2.44.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: diff --git a/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt index 4c48f92a8a..f130bbdb28 100644 --- a/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/podman/create_cluster_with_verbosity.txt @@ -6,14 +6,6 @@ mkdir -p /workdir/clusters//logs touch /workdir/clusters//logs/audit.log cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml mkdir -p /workdir/clusters//etcd -podman pull registry.k8s.io/etcd:3.5.9-0 -podman pull registry.k8s.io/kube-apiserver:v1.28.0 -podman pull localhost/kwok:test -podman pull registry.k8s.io/kube-controller-manager:v1.28.0 -podman pull registry.k8s.io/kube-scheduler:v1.28.0 -podman pull docker.io/kubernetesui/dashboard:v2.7.0 -podman pull docker.io/prom/prometheus:v2.44.0 -podman pull docker.io/jaegertracing/all-in-one:1.45.0 cat </workdir/clusters//apiserver-tracing-config.yaml apiVersion: apiserver.config.k8s.io/v1alpha1 kind: TracingConfiguration @@ -40,18 +32,6 @@ alerting: static_configs: - targets: [] scrape_configs: -- job_name: "kwok-service-discovery" - http_sd_configs: - - url: http://kwok--kwok-controller:10247/discovery/prometheus -- job_name: "prometheus" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - localhost:9090 - job_name: "etcd" scheme: http honor_timestamps: true @@ -60,16 +40,7 @@ scrape_configs: enable_http2: true static_configs: - targets: - - "kwok--etcd:2379" -- job_name: "kwok-controller" - scheme: http - honor_timestamps: true - metrics_path: /metrics - follow_redirects: true - enable_http2: true - static_configs: - - targets: - - "kwok--kwok-controller:10247" + - kwok--etcd:2379 - job_name: "kube-apiserver" scheme: https honor_timestamps: true @@ -82,7 +53,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-apiserver:6443" + - kwok--kube-apiserver:6443 - job_name: "kube-controller-manager" scheme: https honor_timestamps: true @@ -95,7 +66,7 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-controller-manager:10257" + - kwok--kube-controller-manager:10257 - job_name: "kube-scheduler" scheme: https honor_timestamps: true @@ -108,9 +79,38 @@ scrape_configs: insecure_skip_verify: true static_configs: - targets: - - "kwok--kube-scheduler:10259" + - kwok--kube-scheduler:10259 +- job_name: "kwok-service-discovery" + http_sd_configs: + - url: http://kwok--kwok-controller:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--kwok-controller:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 EOF chmod 0644 /workdir/clusters//prometheus.yaml +podman pull registry.k8s.io/etcd:3.5.9-0 +podman pull registry.k8s.io/kube-apiserver:v1.28.0 +podman pull localhost/kwok:test +podman pull registry.k8s.io/kube-controller-manager:v1.28.0 +podman pull registry.k8s.io/kube-scheduler:v1.28.0 +podman pull docker.io/kubernetesui/dashboard:v2.7.0 +podman pull docker.io/prom/prometheus:v2.44.0 +podman pull docker.io/jaegertracing/all-in-one:1.45.0 cat </workdir/clusters//kubeconfig.yaml apiVersion: v1 clusters: