[kwokctl] Structured Prometheus configuration

pkg/kwokctl/components/config/prometheus/types.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package prometheus copy from https://github.com/prometheus/prometheus/blob/919648cafc2c07ed5c1d5dd657b8080bee331aaf/config/config.go#L243
+package prometheus
+
+type GlobalConfig struct {
+	ScrapeInterval     string `json:"scrape_interval,omitempty"`
+	ScrapeTimeout      string `json:"scrape_timeout,omitempty"`
+	EvaluationInterval string `json:"evaluation_interval,omitempty"`
+}
+
+type AlertingConfig struct {
+	AlertmanagerConfigs []AlertmanagerConfig `json:"alertmanagers,omitempty"`
+}
+
+type AlertmanagerConfig struct {
+	FollowRedirects bool   `json:"follow_redirects,omitempty"`
+	EnableHTTP2     bool   `json:"enable_http2,omitempty"`
+	Scheme          string `json:"scheme,omitempty"`
+	Timeout         string `json:"timeout,omitempty"`
+	APIVersion      string `json:"api_version,omitempty"`
+}
+
+type MetricsDiscovery struct {
+	Scheme string `json:"scheme,omitempty"`
+	Host   string `json:"host,omitempty"`
+	Path   string `json:"path,omitempty"`
+}
+
+type Metric struct {
+	Scheme             string `json:"scheme,omitempty"`
+	Path               string `json:"metrics_path,omitempty"`
+	CertPath           string `json:"cert_file,omitempty"`
+	KeyPath            string `json:"key_file,omitempty"`
+	InsecureSkipVerify bool   `json:"insecure_skip_verify,omitempty"`
+	Host               string `json:"host,omitempty"`
+}
+
+type ScrapeConfig struct {
+	JobName         string         `json:"job_name"`
+	HttpSdConfigs   []HTTPSDConfig `json:"http_sd_configs,omitempty"`
+	Scheme          string         `json:"scheme,omitempty"`
+	HonorTimestamps bool           `json:"honor_timestamps,omitempty"`
+	MetricsPath     string         `json:"metrics_path,omitempty"`
+	FollowRedirects bool           `json:"follow_redirects,omitempty"`
+	EnableHttp2     bool           `json:"enable_http2,omitempty"`
+	TLSConfig       *TLSConfig     `json:"tls_config,omitempty"`
+	StaticConfigs   []StaticConfig `json:"static_configs,omitempty"`
+}
+
+type HTTPSDConfig struct {
+	URL string `json:"url"`
+}
+
+type TLSConfig struct {
+	CertFile           string `json:"cert_file"`
+	KeyFile            string `json:"key_file"`
+	InsecureSkipVerify bool   `json:"insecure_skip_verify,omitempty"`
+}
+
+type StaticConfig struct {
+	Targets []string `json:"targets"`
+}
+
+// Config is the top-level configuration for Prometheus's config files.
+type Config struct {
+	GlobalConfig   GlobalConfig    `json:"global"`
+	AlertingConfig AlertingConfig  `json:"alerting"`
+	ScrapeConfigs  []*ScrapeConfig `json:"scrape_configs,omitempty"`
+}

pkg/kwokctl/components/prometheus_config.go

@@ -17,30 +17,90 @@ limitations under the License.
 package components
 
 import (
-	"bytes"
 	"fmt"
-	"text/template"
-
-	"github.com/Masterminds/sprig/v3"
+	"net/url"
 
 	"sigs.k8s.io/kwok/pkg/apis/internalversion"
-
-	_ "embed"
+	"sigs.k8s.io/kwok/pkg/kwokctl/components/config/prometheus"
+	"sigs.k8s.io/kwok/pkg/utils/yaml"
 )
 
-//go:embed prometheus_config.yaml.tpl
-var prometheusYamlTpl string
-
-var prometheusYamlTemplate = template.Must(template.New("prometheus_config").Funcs(sprig.TxtFuncMap()).Parse(prometheusYamlTpl))
-
-// BuildPrometheus builds the prometheus yaml content.
+// BuildPrometheus builds the Prometheus JSON configuration.
 func BuildPrometheus(conf BuildPrometheusConfig) (string, error) {
-	buf := bytes.NewBuffer(nil)
-	err := prometheusYamlTemplate.Execute(buf, conf)
+	config := prometheus.Config{
+		GlobalConfig: prometheus.GlobalConfig{
+			ScrapeInterval:     "15s",
+			ScrapeTimeout:      "10s",
+			EvaluationInterval: "15s",
+		},
+		AlertingConfig: prometheus.AlertingConfig{
+			AlertmanagerConfigs: []prometheus.AlertmanagerConfig{
+				{
+					FollowRedirects: true,
+					EnableHTTP2:     true,
+					Scheme:          "http",
+					Timeout:         "10s",
+					APIVersion:      "v2",
+				},
+			},
+		},
+		ScrapeConfigs: convertToScrapeConfigs(conf.Components),
+	}
+
+	configJSON, err := yaml.Marshal(config)
 	if err != nil {
 		return "", fmt.Errorf("build prometheus error: %w", err)
 	}
-	return buf.String(), nil
+	return string(configJSON), nil
+}
+
+// convertToScrapeConfigs converts internalversion.Component to prometheus.ScrapeConfig.
+func convertToScrapeConfigs(components []internalversion.Component) []*prometheus.ScrapeConfig {
+	var scrapeConfigs []*prometheus.ScrapeConfig
+	for _, c := range components {
+		if md := c.MetricsDiscovery; md != nil {
+			scrapeConfig := &prometheus.ScrapeConfig{}
+			scrapeConfig.JobName = fmt.Sprintf("%s-metrics-discovery", c.Name)
+			u := url.URL{
+				Scheme: md.Scheme,
+				Host:   md.Host,
+				Path:   md.Path,
+			}
+			scrapeConfig.HttpSdConfigs = []prometheus.HTTPSDConfig{
+				{
+					URL: u.String(),
+				},
+			}
+			scrapeConfigs = append(scrapeConfigs, scrapeConfig)
+		}
+
+		if m := c.Metric; m != nil {
+			scrapeConfig := &prometheus.ScrapeConfig{}
+			scrapeConfig.JobName = c.Name
+			scrapeConfig.Scheme = m.Scheme
+			scrapeConfig.MetricsPath = m.Path
+			scrapeConfig.HonorTimestamps = true
+			scrapeConfig.EnableHttp2 = true
+			scrapeConfig.FollowRedirects = true
+
+			if scrapeConfig.Scheme == "https" {
+				scrapeConfig.TLSConfig = &prometheus.TLSConfig{}
+				scrapeConfig.TLSConfig.CertFile = m.CertPath
+				scrapeConfig.TLSConfig.KeyFile = m.KeyPath
+				scrapeConfig.TLSConfig.InsecureSkipVerify = true
+			}
+
+			scrapeConfig.StaticConfigs = []prometheus.StaticConfig{
+				{
+					Targets: []string{
+						m.Host,
+					},
+				},
+			}
+			scrapeConfigs = append(scrapeConfigs, scrapeConfig)
+		}
+	}
+	return scrapeConfigs
 }
 
 // BuildPrometheusConfig is the configuration for building the prometheus config

test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_extra.txt

@@ -12,86 +12,84 @@ mkdir -p <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/etcd
 # Download https://github.com/kubernetes-sigs/kwok/releases/download/v0.7.0/kwok-<OS>-<ARCH> to <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/bin/kwok-controller
 # Download https://github.com/prometheus/prometheus/releases/download/v2.53.0/prometheus-2.53.0.<OS>-<ARCH>.<TAR> and extract prometheus to <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/bin/prometheus
 cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/prometheus.yaml
-global:
-  scrape_interval: 15s
-  scrape_timeout: 10s
-  evaluation_interval: 15s
 alerting:
   alertmanagers:
-  - follow_redirects: true
+  - api_version: v2
     enable_http2: true
+    follow_redirects: true
     scheme: http
     timeout: 10s
-    api_version: v2
-    static_configs:
-    - targets: []
+global:
+  evaluation_interval: 15s
+  scrape_interval: 15s
+  scrape_timeout: 10s
 scrape_configs:
-- job_name: "etcd"
-  scheme: http
+- enable_http2: true
+  follow_redirects: true
   honor_timestamps: true
+  job_name: etcd
   metrics_path: /metrics
-  follow_redirects: true
-  enable_http2: true
+  scheme: http
   static_configs:
   - targets:
     - 127.0.0.1:32765
-- job_name: "kube-apiserver"
-  scheme: https
+- enable_http2: true
+  follow_redirects: true
   honor_timestamps: true
+  job_name: kube-apiserver
   metrics_path: /metrics
-  follow_redirects: true
-  enable_http2: true
-  tls_config:
-    cert_file: "<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt"
-    key_file: "<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key"
-    insecure_skip_verify: true
+  scheme: https
   static_configs:
   - targets:
     - 127.0.0.1:32764
-- job_name: "kube-controller-manager"
-  scheme: https
-  honor_timestamps: true
-  metrics_path: /metrics
-  follow_redirects: true
-  enable_http2: true
   tls_config:
-    cert_file: "<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt"
-    key_file: "<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key"
+    cert_file: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt
     insecure_skip_verify: true
+    key_file: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key
+- enable_http2: true
+  follow_redirects: true
+  honor_timestamps: true
+  job_name: kube-controller-manager
+  metrics_path: /metrics
+  scheme: https
   static_configs:
   - targets:
     - 127.0.0.1:32762
-- job_name: "kube-scheduler"
-  scheme: https
-  honor_timestamps: true
-  metrics_path: /metrics
-  follow_redirects: true
-  enable_http2: true
   tls_config:
-    cert_file: "<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt"
-    key_file: "<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key"
+    cert_file: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt
     insecure_skip_verify: true
+    key_file: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key
+- enable_http2: true
+  follow_redirects: true
+  honor_timestamps: true
+  job_name: kube-scheduler
+  metrics_path: /metrics
+  scheme: https
   static_configs:
   - targets:
     - 127.0.0.1:32761
-- job_name: "kwok-controller-metrics-discovery"
-  http_sd_configs:
+  tls_config:
+    cert_file: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt
+    insecure_skip_verify: true
+    key_file: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key
+- http_sd_configs:
   - url: http://127.0.0.1:32763/discovery/prometheus
-- job_name: "kwok-controller"
-  scheme: http
+  job_name: kwok-controller-metrics-discovery
+- enable_http2: true
+  follow_redirects: true
   honor_timestamps: true
+  job_name: kwok-controller
   metrics_path: /metrics
-  follow_redirects: true
-  enable_http2: true
+  scheme: http
   static_configs:
   - targets:
     - 127.0.0.1:32763
-- job_name: "prometheus"
-  scheme: http
+- enable_http2: true
+  follow_redirects: true
   honor_timestamps: true
+  job_name: prometheus
   metrics_path: /metrics
-  follow_redirects: true
-  enable_http2: true
+  scheme: http
   static_configs:
   - targets:
     - 127.0.0.1:9090