From dc49cc5f5154cf35dc1a5023b7313bfabb8a92a8 Mon Sep 17 00:00:00 2001 From: LiZhenCheng9527 Date: Tue, 16 Jul 2024 16:11:48 +0800 Subject: [PATCH 1/2] add configuration to collect kmesh metrics using Prometheus Signed-off-by: LiZhenCheng9527 --- deploy/yaml/kmesh.yaml | 4 + samples/addons/prometheus.yaml | 553 ++++++++++++++++++++++++++++ samples/tcp-echo/tcp-echo-ipv4.yaml | 44 +++ samples/tcp-echo/tcp-echo-ipv6.yaml | 44 +++ samples/tcp-echo/tcp-echo.yaml | 41 +++ 5 files changed, 686 insertions(+) create mode 100644 samples/addons/prometheus.yaml create mode 100644 samples/tcp-echo/tcp-echo-ipv4.yaml create mode 100644 samples/tcp-echo/tcp-echo-ipv6.yaml create mode 100644 samples/tcp-echo/tcp-echo.yaml diff --git a/deploy/yaml/kmesh.yaml b/deploy/yaml/kmesh.yaml index 8033ba384..f35ade59c 100644 --- a/deploy/yaml/kmesh.yaml +++ b/deploy/yaml/kmesh.yaml @@ -18,6 +18,10 @@ spec: metadata: labels: app: kmesh + annotations: + prometheus.io/path: "status/metric" + prometheus.io/port: "15020" + prometheus.io/scrape: "true" spec: tolerations: - effect: NoSchedule diff --git a/samples/addons/prometheus.yaml b/samples/addons/prometheus.yaml new file mode 100644 index 000000000..221391745 --- /dev/null +++ b/samples/addons/prometheus.yaml @@ -0,0 +1,553 @@ +--- +# Source: prometheus/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v2.51.1 + helm.sh/chart: prometheus-25.19.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: kmesh-system + annotations: + {} +--- +# Source: prometheus/templates/cm.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v2.51.1 + helm.sh/chart: prometheus-25.19.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: kmesh-system +data: + allow-snippet-annotations: "false" + alerting_rules.yml: | + {} + alerts: | + {} + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 15s + scrape_timeout: 10s + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - /etc/config/rules + - /etc/config/alerts + scrape_configs: + - job_name: prometheus + metrics_path: /status/metric + static_configs: + - targets: + - localhost:15020 + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: default;kubernetes;https + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes-cadvisor + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + - honor_labels: true + job_name: kubernetes-service-endpoints + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-service-endpoints-slow + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + - honor_labels: true + job_name: prometheus-pushgateway + kubernetes_sd_configs: + - role: service + relabel_configs: + - action: keep + regex: pushgateway + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - honor_labels: true + job_name: kubernetes-services + kubernetes_sd_configs: + - role: service + metrics_path: /probe + params: + module: + - http_2xx + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - source_labels: + - __address__ + target_label: __param_target + - replacement: blackbox + target_label: __address__ + - source_labels: + - __param_target + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - honor_labels: true + job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-pods-slow + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + recording_rules.yml: | + {} + rules: | + {} +--- +# Source: prometheus/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v2.51.1 + helm.sh/chart: prometheus-25.19.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + - ingresses + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "discovery.k8s.io" + resources: + - endpointslices + verbs: + - get + - list + - watch + - nonResourceURLs: + - "/metrics" + verbs: + - get +--- +# Source: prometheus/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v2.51.1 + helm.sh/chart: prometheus-25.19.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: kmesh-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +--- +# Source: prometheus/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v2.51.1 + helm.sh/chart: prometheus-25.19.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: kmesh-system +spec: + ports: + - name: http + port: 9090 + protocol: TCP + targetPort: 9090 + selector: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + sessionAffinity: None + type: "ClusterIP" +--- +# Source: prometheus/templates/deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v2.51.1 + helm.sh/chart: prometheus-25.19.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + name: prometheus + namespace: kmesh-system +spec: + selector: + matchLabels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + replicas: 1 + revisionHistoryLimit: 10 + strategy: + type: Recreate + rollingUpdate: null + template: + metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v2.51.1 + helm.sh/chart: prometheus-25.19.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: prometheus + # sidecar.kmesh.io/inject: "false" + spec: + enableServiceLinks: true + serviceAccountName: prometheus + containers: + - name: prometheus-server-configmap-reload + image: "ghcr.io/prometheus-operator/prometheus-config-reloader:v0.72.0" + imagePullPolicy: "IfNotPresent" + args: + - --watched-dir=/etc/config + - --reload-url=http://127.0.0.1:9090/-/reload + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + + - name: prometheus-server + image: "prom/prometheus:v2.51.1" + imagePullPolicy: "IfNotPresent" + args: + - --storage.tsdb.retention.time=15d + - --config.file=/etc/config/prometheus.yml + - --storage.tsdb.path=/data + - --web.console.libraries=/etc/prometheus/console_libraries + - --web.console.templates=/etc/prometheus/consoles + - --web.enable-lifecycle + ports: + - containerPort: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 4 + failureThreshold: 3 + successThreshold: 1 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: storage-volume + mountPath: /data + subPath: "" + dnsPolicy: ClusterFirst + terminationGracePeriodSeconds: 300 + volumes: + - name: config-volume + configMap: + name: prometheus + - name: storage-volume + emptyDir: + {} diff --git a/samples/tcp-echo/tcp-echo-ipv4.yaml b/samples/tcp-echo/tcp-echo-ipv4.yaml new file mode 100644 index 000000000..44e8a143f --- /dev/null +++ b/samples/tcp-echo/tcp-echo-ipv4.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: + name: tcp-echo + labels: + app: tcp-echo + service: tcp-echo +spec: + ipFamilyPolicy: SingleStack + ipFamilies: + - IPv4 + ports: + - name: tcp + port: 9000 + - name: tcp-other + port: 9001 + # Port 9002 is omitted intentionally for testing the pass through filter chain. + selector: + app: tcp-echo +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tcp-echo +spec: + replicas: 1 + selector: + matchLabels: + app: tcp-echo + version: v1 + template: + metadata: + labels: + app: tcp-echo + version: v1 + spec: + containers: + - name: tcp-echo + image: docker.io/istio/tcp-echo-server:1.3 + imagePullPolicy: IfNotPresent + args: [ "9000,9001,9002", "hello" ] + ports: + - containerPort: 9000 + - containerPort: 9001 diff --git a/samples/tcp-echo/tcp-echo-ipv6.yaml b/samples/tcp-echo/tcp-echo-ipv6.yaml new file mode 100644 index 000000000..9f2954749 --- /dev/null +++ b/samples/tcp-echo/tcp-echo-ipv6.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: + name: tcp-echo + labels: + app: tcp-echo + service: tcp-echo +spec: + ipFamilyPolicy: SingleStack + ipFamilies: + - IPv6 + ports: + - name: tcp + port: 9000 + - name: tcp-other + port: 9001 + # Port 9002 is omitted intentionally for testing the pass through filter chain. + selector: + app: tcp-echo +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tcp-echo +spec: + replicas: 1 + selector: + matchLabels: + app: tcp-echo + version: v1 + template: + metadata: + labels: + app: tcp-echo + version: v1 + spec: + containers: + - name: tcp-echo + image: docker.io/istio/tcp-echo-server:1.3 + imagePullPolicy: IfNotPresent + args: [ "9000,9001,9002", "hello" ] + ports: + - containerPort: 9000 + - containerPort: 9001 diff --git a/samples/tcp-echo/tcp-echo.yaml b/samples/tcp-echo/tcp-echo.yaml new file mode 100644 index 000000000..cd2e70d3c --- /dev/null +++ b/samples/tcp-echo/tcp-echo.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: tcp-echo + labels: + app: tcp-echo + service: tcp-echo +spec: + ports: + - name: tcp + port: 9000 + - name: tcp-other + port: 9001 + # Port 9002 is omitted intentionally for testing the pass through filter chain. + selector: + app: tcp-echo +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tcp-echo +spec: + replicas: 1 + selector: + matchLabels: + app: tcp-echo + version: v1 + template: + metadata: + labels: + app: tcp-echo + version: v1 + spec: + containers: + - name: tcp-echo + image: docker.io/istio/tcp-echo-server:1.3 + imagePullPolicy: IfNotPresent + args: [ "9000,9001,9002", "hello" ] + ports: + - containerPort: 9000 + - containerPort: 9001 From eb828cee7b260074c7f1439fa3a80b43b78e7ba8 Mon Sep 17 00:00:00 2001 From: LiZhenCheng9527 Date: Wed, 17 Jul 2024 17:16:43 +0800 Subject: [PATCH 2/2] put annotation about prometheus in kmesh helm daemonset.yaml Signed-off-by: LiZhenCheng9527 --- deploy/helm/templates/daemonset.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deploy/helm/templates/daemonset.yaml b/deploy/helm/templates/daemonset.yaml index a5742b718..56908898c 100644 --- a/deploy/helm/templates/daemonset.yaml +++ b/deploy/helm/templates/daemonset.yaml @@ -16,6 +16,10 @@ spec: labels: app: kmesh {{- include "kmesh.selectorLabels" . | nindent 8 }} + annotations: + prometheus.io/path: "status/metric" + prometheus.io/port: "15020" + prometheus.io/scrape: "true" spec: containers: - args: ["./start_kmesh.sh {{ .Values.deploy.kmesh.containers.kmeshDaemonArgs }}"]