Fix: enhance loki & prom addon

Signed-off-by: Yin Da <[email protected]>
kubevela · Mar 16, 2023 · 33e01aa · 33e01aa
1 parent 9edcdae
commit 33e01aa
Show file tree

Hide file tree

Showing 12 changed files with 183 additions and 158 deletions.
diff --git a/addons/loki/definitions/stdout-logs.cue b/addons/loki/definitions/stdout-logs.cue
@@ -25,7 +25,7 @@ template: {
 	parameter: {
 		grafana:      *"default" | string
 		parser?:      "nginx" | "apache" | "customize"
-		VRL?: string
+		VRL?:         string
 		lokiEndpoint: *"http://loki:3100/" | string
 	}
 

diff --git a/addons/loki/metadata.yaml b/addons/loki/metadata.yaml
@@ -1,5 +1,5 @@
 name: loki
-version: v0.3.0
+version: v0.3.1
 description: Loki is a log aggregation system designed to store and query logs from all your applications and infrastructure.
 icon: https://grafana.com/static/img/logos/logo-loki.svg
 url: https://grafana.com/oss/loki/

diff --git a/addons/loki/parameter.cue b/addons/loki/parameter.cue
@@ -30,12 +30,16 @@ parameter: {
 	// agent parameters
 
 	// +usage=Specify the type of log agents, if empty, no agent will be installed
-	agent: *"" | "vector" | "promtail"
+	agent: *"vector" | "" | "promtail"
 	// +usage=Specify the image of promtail
 	promtailImage: *"grafana/promtail" | string
 	// +usage=Specify the image of vector
 	vectorImage:           *"timberio/vector:0.25.2-distroless-libc" | string
 	vectorControllerImage: *"oamdev/vector-controller:0.2.2" | string
 	// +usage=collect all pods' stdout log
 	stdout: *"" | "all"
+	// +usage=The loki URL for vector agent to sink. If not specified, the in cluster loki endpoint will be used.
+	lokiURL?: string
+	// +usage=If set to true, only agent will be installed.
+	agentOnly: *false | bool
 }
diff --git a/addons/loki/resources/agent-promtail.cue b/addons/loki/resources/agent-promtail.cue
@@ -76,52 +76,54 @@ promtailConfig: {
 				  http_listen_port: 3101
 
 				clients:
-				  - url: http://loki:3100/loki/api/v1/push
-				
-				positions:
-				  filename: /run/promtail/positions.yaml
-				
-				scrape_configs:
-				  # See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference
-				  - job_name: kubernetes-pods
-				    pipeline_stages:
-				      - cri: {}
-				      - static_labels:
-				          agent: promtail
-				          cluster: $CLUSTER
-				          forward: daemon
-				    kubernetes_sd_configs:
-				      - role: pod
-				    relabel_configs:
-				      - action: replace
-				        source_labels:
-				        - __meta_kubernetes_namespace
-				        target_label: namespace
-				      - action: replace
-				        source_labels:
-				        - __meta_kubernetes_pod_name
-				        target_label: pod
-				      - action: replace
-				        source_labels:
-				        - __meta_kubernetes_pod_container_name
-				        target_label: container
-				      - action: replace
-				        replacement: /var/log/pods/*$1/*.log
-				        separator: /
-				        source_labels:
-				        - __meta_kubernetes_pod_uid
-				        - __meta_kubernetes_pod_container_name
-				        target_label: __path__
-				      - action: replace
-				        regex: true/(.*)
-				        replacement: /var/log/pods/*$1/*.log
-				        separator: /
-				        source_labels:
-				        - __meta_kubernetes_pod_annotationpresent_kubernetes_io_config_hash
-				        - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
-				        - __meta_kubernetes_pod_container_name
-				        target_label: __path__
-				"""#
+				"""# +
+				"\n  - url: " + lokiURL + "loki/api/v1/push\n" +
+				#"""
+					
+					positions:
+					  filename: /run/promtail/positions.yaml
+					
+					scrape_configs:
+					  # See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference
+					  - job_name: kubernetes-pods
+					    pipeline_stages:
+					      - cri: {}
+					      - static_labels:
+					          agent: promtail
+					          cluster: $CLUSTER
+					          forward: daemon
+					    kubernetes_sd_configs:
+					      - role: pod
+					    relabel_configs:
+					      - action: replace
+					        source_labels:
+					        - __meta_kubernetes_namespace
+					        target_label: namespace
+					      - action: replace
+					        source_labels:
+					        - __meta_kubernetes_pod_name
+					        target_label: pod
+					      - action: replace
+					        source_labels:
+					        - __meta_kubernetes_pod_container_name
+					        target_label: container
+					      - action: replace
+					        replacement: /var/log/pods/*$1/*.log
+					        separator: /
+					        source_labels:
+					        - __meta_kubernetes_pod_uid
+					        - __meta_kubernetes_pod_container_name
+					        target_label: __path__
+					      - action: replace
+					        regex: true/(.*)
+					        replacement: /var/log/pods/*$1/*.log
+					        separator: /
+					        source_labels:
+					        - __meta_kubernetes_pod_annotationpresent_kubernetes_io_config_hash
+					        - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
+					        - __meta_kubernetes_pod_container_name
+					        target_label: __path__
+					"""#
 		}
 	}]
 }
diff --git a/addons/loki/resources/agent-vector.cue b/addons/loki/resources/agent-vector.cue
@@ -67,15 +67,15 @@ vector: {
 		}
 	}
 	traits: [{
-		type: "command"
+		type:       "command"
 		_configDir: *"/etc/config/" | string
 		if parameter.agent == "vector" && parameter.stdout == "" {
 			_configDir: "/etc/config/,/etc/vector-controller-config/"
-		},
+		}
 		properties: args: [
 			"--config-dir",
 			_configDir,
-			"-w"
+			"-w",
 		]
 	}, {
 		agentServiceAccount
@@ -103,22 +103,24 @@ vectorConfig: {
 					    type: loki
 					    inputs:
 					      - kubernetes-logs
-					    endpoint: http://loki:3100/
-					    compression: none
-					    request:
-					      concurrency: 10
-					    labels:
-					      agent: vector
-					      cluster: $CLUSTER
-					      stream: "{{ stream }}"
-					      forward: daemon
-					      filename: "{{ file }}"
-					      pod: "{{ kubernetes.pod_name }}"
-					      namespace: "{{ kubernetes.pod_namespace }}"
-					      container: "{{ kubernetes.container_name }}"
-					    encoding:
-					      codec: json
-					"""#
+					"""# +
+					"\n    endpoint: " + lokiURL + "\n" +
+					#"""
+						    compression: none
+						    request:
+						      concurrency: 10
+						    labels:
+						      agent: vector
+						      cluster: $CLUSTER
+						      stream: "{{ stream }}"
+						      forward: daemon
+						      filename: "{{ file }}"
+						      pod: "{{ kubernetes.pod_name }}"
+						      namespace: "{{ kubernetes.pod_namespace }}"
+						      container: "{{ kubernetes.container_name }}"
+						    encoding:
+						      codec: json
+						"""#
 			}
 		}
 		if parameter.agent == "vector" && parameter.stdout == "" {
@@ -134,22 +136,24 @@ vectorConfig: {
 					    type: loki
 					    inputs:
 					      - kubernetes-logs
-					    endpoint: http://loki:3100/
-					    compression: none
-					    request:
-					      concurrency: 10
-					    labels:
-					      agent: vector
-					      cluster: $CLUSTER
-					      stream: "{{ stream }}"
-					      forward: daemon
-					      filename: "{{ file }}"
-					      pod: "{{ kubernetes.pod_name }}"
-					      namespace: "{{ kubernetes.pod_namespace }}"
-					      container: "{{ kubernetes.container_name }}"
-					    encoding:
-					      codec: json
-					"""#
+					"""# +
+					"\n    endpoint: " + lokiURL + "\n" +
+					#"""
+						    compression: none
+						    request:
+						      concurrency: 10
+						    labels:
+						      agent: vector
+						      cluster: $CLUSTER
+						      stream: "{{ stream }}"
+						      forward: daemon
+						      filename: "{{ file }}"
+						      pod: "{{ kubernetes.pod_name }}"
+						      namespace: "{{ kubernetes.pod_namespace }}"
+						      container: "{{ kubernetes.container_name }}"
+						    encoding:
+						      codec: json
+						"""#
 			}
 		}
 	}]

diff --git a/addons/loki/resources/agent.cue b/addons/loki/resources/agent.cue
@@ -56,7 +56,7 @@ if parameter.agent == "vector" {
 		agentComponents: [vector, vectorConfig, eventLogger]
 	}
 	if parameter.stdout == "" {
-	  agentComponents: [vector, vectorConfig, vectorController, vectorControllerExtraResources, eventLogger]
+		agentComponents: [vector, vectorConfig, vectorController, vectorControllerExtraResources, eventLogger]
 	}
 }
 

diff --git a/addons/loki/resources/loki.cue b/addons/loki/resources/loki.cue
@@ -48,3 +48,65 @@ loki: {
 		properties: args: ["-config.file=/etc/loki/loki.yaml"]
 	}]
 }
+
+lokiSteps: *[] | [...{...}]
+if !parameter.agentOnly {
+	lokiSteps: [{
+		type: "deploy"
+		name: "deploy-loki"
+		properties: policies: ["topology-centralized", "loki-components"]
+	}, {
+		type: "collect-service-endpoints"
+		name: "get-loki-endpoint"
+		properties: {
+			name:      const.name
+			namespace: "vela-system"
+			components: [loki.name]
+			portName: "http"
+			outer:    parameter.serviceType != "ClusterIP"
+		}
+		outputs: [{
+			name:      "host"
+			valueFrom: "value.endpoint.host"
+		}, {
+			name:      "port"
+			valueFrom: "value.endpoint.port"
+		}, {
+			name:      "url"
+			valueFrom: "value.url"
+		}]
+	}, {
+		type: "create-config"
+		name: "loki-server-register"
+		properties: {
+			name:     "loki-vela"
+			template: "loki"
+			config: {}
+		}
+		inputs: [{
+			from:         "url"
+			parameterKey: "config.url"
+		}]
+	}, {
+		type: "export-service"
+		name: "export-service"
+		properties: {
+			name:      "loki"
+			namespace: parameter.namespace
+			topology:  "topology-distributed-exclude-local"
+			port:      3100
+		}
+		inputs: [{
+			from:         "host"
+			parameterKey: "ip"
+		}, {
+			from:         "port"
+			parameterKey: "targetPort"
+		}]
+	}]
+}
+
+lokiURL: *"http://loki:3100/" | string
+if parameter.lokiURL != _|_ {
+	lokiURL: parameter.lokiURL
+}
diff --git a/addons/loki/template.cue b/addons/loki/template.cue
@@ -30,58 +30,6 @@ output: {
 			type: "deploy"
 			name: "deploy-ns"
 			properties: policies: ["topology-distributed", "o11y-namespace"]
-		}, {
-			type: "deploy"
-			name: "deploy-loki"
-			properties: policies: ["topology-centralized", "loki-components"]
-		}, {
-			type: "collect-service-endpoints"
-			name: "get-loki-endpoint"
-			properties: {
-				name:      const.name
-				namespace: "vela-system"
-				components: [loki.name]
-				portName: "http"
-				outer:    parameter.serviceType != "ClusterIP"
-			}
-			outputs: [{
-				name:      "host"
-				valueFrom: "value.endpoint.host"
-			}, {
-				name:      "port"
-				valueFrom: "value.endpoint.port"
-			}, {
-				name:      "url"
-				valueFrom: "value.url"
-			}]
-		}, {
-			type: "create-config"
-			name: "loki-server-register"
-			properties: {
-				name:     "loki-vela"
-				template: "loki"
-				config: {}
-			}
-			inputs: [{
-				from:         "url"
-				parameterKey: "config.url"
-			}]
-		}, {
-			type: "export-service"
-			name: "export-service"
-			properties: {
-				name:      "loki"
-				namespace: parameter.namespace
-				topology:  "topology-distributed-exclude-local"
-				port:      3100
-			}
-			inputs: [{
-				from:         "host"
-				parameterKey: "ip"
-			}, {
-				from:         "port"
-				parameterKey: "targetPort"
-			}]
-		}] + agentWorkflowSteps
+		}] + lokiSteps + agentWorkflowSteps
 	}
 }
diff --git a/addons/prometheus-server/metadata.yaml b/addons/prometheus-server/metadata.yaml
@@ -1,5 +1,5 @@
 name: prometheus-server
-version: v0.3.1
+version: v0.3.2
 description: An open-source monitoring system with a dimensional data model, flexible query language, efficient time series database and modern alerting approach.
 icon: https://artifacthub.io/image/0503add5-3fce-4b63-bbf3-b9f649512a86@1x
 url: https://prometheus.io/

diff --git a/addons/prometheus-server/parameter.cue b/addons/prometheus-server/parameter.cue
@@ -34,4 +34,7 @@ parameter: {
 	customConfig?: string
 	// +usage=If specified, thanos sidecar will be attached and ports will be exposed
 	thanos: *false | bool
+
+	// +usage=The interval for scraping pods, services, endpoints.
+	scrapeInterval: *"15s" | =~"^([1-9][0-9]?)(s|m)$"
 }