Skip to content

Commit a3f1fcb

Browse files
zyy17zyy17
authored
refactor: add logging and monitoring configurations (#171)
* refactor: add logging and monitoring configurations * refactor: add example of standalone
1 parent c6ff85f commit a3f1fcb

File tree

4 files changed

+303
-11
lines changed

4 files changed

+303
-11
lines changed

‎charts/greptimedb-cluster/Chart.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ apiVersion: v2
22
name: greptimedb-cluster
33
description: A Helm chart for deploying GreptimeDB cluster in Kubernetes.
44
type: application
5-
version: 0.2.16
5+
version: 0.2.17
66
appVersion: 0.9.4
77
home: https://github.com/GreptimeTeam/greptimedb
88
sources:

‎charts/greptimedb-cluster/README.md

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
A Helm chart for deploying GreptimeDB cluster in Kubernetes.
44

5-
![Version: 0.2.16](https://img.shields.io/badge/Version-0.2.16-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.9.4](https://img.shields.io/badge/AppVersion-0.9.4-informational?style=flat-square)
5+
![Version: 0.2.17](https://img.shields.io/badge/Version-0.2.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.9.4](https://img.shields.io/badge/AppVersion-0.9.4-informational?style=flat-square)
66

77
## Source Code
88

@@ -92,9 +92,10 @@ helm uninstall mycluster -n default
9292
| base.podTemplate.nodeSelector | object | `{}` | The pod node selector |
9393
| base.podTemplate.serviceAccountName | string | `""` | The global service account |
9494
| base.podTemplate.tolerations | list | `[]` | The pod tolerations |
95-
| datanode | object | `{"configData":"","configFile":"","podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":3,"storage":{"dataHome":"/data/greptimedb","mountPath":"/data/greptimedb","storageClassName":null,"storageRetainPolicy":"Retain","storageSize":"10Gi","walDir":"/data/greptimedb/wal"}}` | Datanode configure |
95+
| datanode | object | `{"configData":"","configFile":"","logging":{},"podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1,"storage":{"dataHome":"/data/greptimedb","mountPath":"/data/greptimedb","storageClassName":null,"storageRetainPolicy":"Retain","storageSize":"10Gi","walDir":"/data/greptimedb/wal"}}` | Datanode configure |
9696
| datanode.configData | string | `""` | Extra raw toml config data of datanode. Skip if the `configFile` is used. |
9797
| datanode.configFile | string | `""` | Extra toml file of datanode. |
98+
| datanode.logging | object | `{}` | Logging configuration for datanode, if not set, it will use the global logging configuration. |
9899
| datanode.podTemplate | object | `{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]}` | The pod template for datanode |
99100
| datanode.podTemplate.affinity | object | `{}` | The pod affinity |
100101
| datanode.podTemplate.annotations | object | `{}` | The annotations to be created to the pod. |
@@ -113,7 +114,7 @@ helm uninstall mycluster -n default
113114
| datanode.podTemplate.serviceAccount.create | bool | `false` | Create a service account |
114115
| datanode.podTemplate.tolerations | list | `[]` | The pod tolerations |
115116
| datanode.podTemplate.volumes | list | `[]` | The pod volumes |
116-
| datanode.replicas | int | `3` | Datanode replicas |
117+
| datanode.replicas | int | `1` | Datanode replicas |
117118
| datanode.storage.dataHome | string | `"/data/greptimedb"` | The dataHome directory, default is "/data/greptimedb/" |
118119
| datanode.storage.mountPath | string | `"/data/greptimedb"` | The data directory of the storage, default is "/data/greptimedb" |
119120
| datanode.storage.storageClassName | string | `nil` | Storage class for datanode persistent volume |
@@ -123,10 +124,11 @@ helm uninstall mycluster -n default
123124
| debugPod.enabled | bool | `false` | Enable debug pod |
124125
| debugPod.image | object | `{"registry":"docker.io","repository":"greptime/greptime-tool","tag":"20240905-67eaa147"}` | The debug pod image |
125126
| debugPod.resources | object | `{"limits":{"cpu":"200m","memory":"256Mi"},"requests":{"cpu":"50m","memory":"64Mi"}}` | The debug pod resource |
126-
| flownode | object | `{"configData":"","configFile":"","enabled":false,"podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1}` | Flownode configure. **It's NOT READY YET** |
127+
| flownode | object | `{"configData":"","configFile":"","enabled":false,"logging":{},"podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1}` | Flownode configure. **It's NOT READY YET** |
127128
| flownode.configData | string | `""` | Extra raw toml config data of flownode. Skip if the `configFile` is used. |
128129
| flownode.configFile | string | `""` | Extra toml file of flownode. |
129130
| flownode.enabled | bool | `false` | Enable flownode |
131+
| flownode.logging | object | `{}` | Logging configuration for flownode, if not set, it will use the global logging configuration. |
130132
| flownode.podTemplate | object | `{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]}` | The pod template for frontend |
131133
| flownode.podTemplate.affinity | object | `{}` | The pod affinity |
132134
| flownode.podTemplate.annotations | object | `{}` | The annotations to be created to the pod. |
@@ -146,9 +148,10 @@ helm uninstall mycluster -n default
146148
| flownode.podTemplate.tolerations | list | `[]` | The pod tolerations |
147149
| flownode.podTemplate.volumes | list | `[]` | The pod volumes |
148150
| flownode.replicas | int | `1` | Flownode replicas |
149-
| frontend | object | `{"configData":"","configFile":"","podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1,"service":{},"tls":{}}` | Frontend configure |
151+
| frontend | object | `{"configData":"","configFile":"","logging":{},"podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1,"service":{},"tls":{}}` | Frontend configure |
150152
| frontend.configData | string | `""` | Extra raw toml config data of frontend. Skip if the `configFile` is used. |
151153
| frontend.configFile | string | `""` | Extra toml file of frontend. |
154+
| frontend.logging | object | `{}` | Logging configuration for frontend, if not set, it will use the global logging configuration. |
152155
| frontend.podTemplate | object | `{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","readinessProbe":{},"resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]}` | The pod template for frontend |
153156
| frontend.podTemplate.affinity | object | `{}` | The pod affinity |
154157
| frontend.podTemplate.annotations | object | `{}` | The annotations to be created to the pod. |
@@ -178,12 +181,19 @@ helm uninstall mycluster -n default
178181
| image.tag | string | `"v0.9.4"` | The image tag |
179182
| initializer.registry | string | `"docker.io"` | Initializer image registry |
180183
| initializer.repository | string | `"greptime/greptimedb-initializer"` | Initializer image repository |
181-
| initializer.tag | string | `"v0.1.0"` | Initializer image tag |
182-
| meta | object | `{"configData":"","configFile":"","enableRegionFailover":false,"etcdEndpoints":"etcd.etcd-cluster.svc.cluster.local:2379","podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"readinessProbe":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1,"storeKeyPrefix":""}` | Meta configure |
184+
| initializer.tag | string | `"v0.1.1"` | Initializer image tag |
185+
| logging | object | `{"format":"text","level":"info","logsDir":"/data/greptimedb/logs","onlyLogToStdout":false,"persistentWithData":false}` | Global logging configuration |
186+
| logging.format | string | `"text"` | The log format for greptimedb, only support "json" and "text" |
187+
| logging.level | string | `"info"` | The log level for greptimedb, only support "debug", "info", "warn", "debug" |
188+
| logging.logsDir | string | `"/data/greptimedb/logs"` | The logs directory for greptimedb |
189+
| logging.onlyLogToStdout | bool | `false` | Whether to log to stdout only |
190+
| logging.persistentWithData | bool | `false` | indicates whether to persist the log with the datanode data storage. It **ONLY** works for the datanode component. |
191+
| meta | object | `{"configData":"","configFile":"","enableRegionFailover":false,"etcdEndpoints":"etcd.etcd-cluster.svc.cluster.local:2379","logging":{},"podTemplate":{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"readinessProbe":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]},"replicas":1,"storeKeyPrefix":""}` | Meta configure |
183192
| meta.configData | string | `""` | Extra raw toml config data of meta. Skip if the `configFile` is used. |
184193
| meta.configFile | string | `""` | Extra toml file of meta. |
185194
| meta.enableRegionFailover | bool | `false` | Whether to enable region failover |
186195
| meta.etcdEndpoints | string | `"etcd.etcd-cluster.svc.cluster.local:2379"` | Meta etcd endpoints |
196+
| meta.logging | object | `{}` | Logging configuration for meta, if not set, it will use the global logging configuration. |
187197
| meta.podTemplate | object | `{"affinity":{},"annotations":{},"labels":{},"main":{"args":[],"command":[],"env":[],"image":"","resources":{"limits":{},"requests":{}},"volumeMounts":[]},"nodeSelector":{},"readinessProbe":{},"serviceAccount":{"annotations":{},"create":false},"tolerations":[],"volumes":[]}` | The pod template for meta |
188198
| meta.podTemplate.affinity | object | `{}` | The pod affinity |
189199
| meta.podTemplate.annotations | object | `{}` | The annotations to be created to the pod. |
@@ -204,6 +214,16 @@ helm uninstall mycluster -n default
204214
| meta.podTemplate.volumes | list | `[]` | The pod volumes |
205215
| meta.replicas | int | `1` | Meta replicas |
206216
| meta.storeKeyPrefix | string | `""` | Meta will store data with this key prefix |
217+
| monitoring | object | `{"enabled":true,"logsCollection":{"pipeline":{"data":""}},"standalone":{},"vector":{"registry":"docker.io","repository":"timberio/vector","resource":{"limits":{"cpu":"50m","memory":"64Mi"},"requests":{"cpu":"50m","memory":"64Mi"}},"tag":"nightly-alpine"}}` | The monitoring bootstrap configuration |
218+
| monitoring.enabled | bool | `true` | Enable monitoring |
219+
| monitoring.logsCollection | object | `{"pipeline":{"data":""}}` | Configure the logs collection |
220+
| monitoring.logsCollection.pipeline | object | `{"data":""}` | The greptimedb pipeline for logs collection |
221+
| monitoring.standalone | object | `{}` | Configure the standalone instance for storing monitoring data |
222+
| monitoring.vector | object | `{"registry":"docker.io","repository":"timberio/vector","resource":{"limits":{"cpu":"50m","memory":"64Mi"},"requests":{"cpu":"50m","memory":"64Mi"}},"tag":"nightly-alpine"}` | Configure vector for logs and metrics collection. |
223+
| monitoring.vector.registry | string | `"docker.io"` | vector image registry |
224+
| monitoring.vector.repository | string | `"timberio/vector"` | vector image repository |
225+
| monitoring.vector.resource | object | `{"limits":{"cpu":"50m","memory":"64Mi"},"requests":{"cpu":"50m","memory":"64Mi"}}` | vector resource |
226+
| monitoring.vector.tag | string | `"nightly-alpine"` | vector image tag |
207227
| mysqlServicePort | int | `4002` | GreptimeDB mysql service port |
208228
| objectStorage | object | `{"gcs":{},"oss":{},"s3":{}}` | Configure to object storage |
209229
| postgresServicePort | int | `4003` | GreptimeDB postgres service port |

‎charts/greptimedb-cluster/templates/cluster.yaml

Lines changed: 110 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,24 @@ spec:
129129
secretName: {{ .Release.Name }}-users-auth
130130
{{- end }}
131131
{{- end }}
132+
{{- if .Values.frontend.logging }}
133+
logging:
134+
{{- if .Values.frontend.logging.level }}
135+
level: {{ .Values.frontend.logging.level }}
136+
{{- end }}
137+
{{- if .Values.frontend.logging.format }}
138+
format: {{ .Values.frontend.logging.format }}
139+
{{- end }}
140+
{{- if .Values.frontend.logging.logsDir }}
141+
logsDir: {{ .Values.frontend.logging.logsDir }}
142+
{{- end }}
143+
{{- if .Values.frontend.logging.onlyLogToStdout }}
144+
onlyLogToStdout: {{ .Values.frontend.logging.onlyLogToStdout }}
145+
{{- end }}
146+
{{- if .Values.frontend.logging.persistentWithData }}
147+
persistentWithData: {{ .Values.frontend.logging.persistentWithData }}
148+
{{- end }}
149+
{{- end }}
132150
meta:
133151
replicas: {{ .Values.meta.replicas }}
134152
{{- if or .Values.meta.configFile .Values.meta.configData }}
@@ -149,6 +167,24 @@ spec:
149167
{{- if .Values.meta.enableRegionFailover }}
150168
enableRegionFailover: {{ .Values.meta.enableRegionFailover }}
151169
{{- end }}
170+
{{- if .Values.meta.logging }}
171+
logging:
172+
{{- if .Values.meta.logging.level }}
173+
level: {{ .Values.meta.logging.level }}
174+
{{- end }}
175+
{{- if .Values.meta.logging.format }}
176+
format: {{ .Values.meta.logging.format }}
177+
{{- end }}
178+
{{- if .Values.meta.logging.logsDir }}
179+
logsDir: {{ .Values.meta.logging.logsDir }}
180+
{{- end }}
181+
{{- if .Values.meta.logging.onlyLogToStdout }}
182+
onlyLogToStdout: {{ .Values.meta.logging.onlyLogToStdout }}
183+
{{- end }}
184+
{{- if .Values.meta.logging.persistentWithData }}
185+
persistentWithData: {{ .Values.meta.logging.persistentWithData }}
186+
{{- end }}
187+
{{- end }}
152188
template:
153189
main:
154190
{{- if .Values.meta.podTemplate.main.image }}
@@ -247,6 +283,24 @@ spec:
247283
{{- if .Values.datanode.podTemplate.volumes}}
248284
volumes: {{ .Values.datanode.podTemplate.volumes | toYaml | nindent 8 }}
249285
{{- end }}
286+
{{- if .Values.datanode.logging }}
287+
logging:
288+
{{- if .Values.datanode.logging.level }}
289+
level: {{ .Values.datanode.logging.level }}
290+
{{- end }}
291+
{{- if .Values.datanode.logging.format }}
292+
format: {{ .Values.datanode.logging.format }}
293+
{{- end }}
294+
{{- if .Values.datanode.logging.logsDir }}
295+
logsDir: {{ .Values.datanode.logging.logsDir }}
296+
{{- end }}
297+
{{- if .Values.datanode.logging.onlyLogToStdout }}
298+
onlyLogToStdout: {{ .Values.datanode.logging.onlyLogToStdout }}
299+
{{- end }}
300+
{{- if .Values.datanode.logging.persistentWithData }}
301+
persistentWithData: {{ .Values.datanode.logging.persistentWithData }}
302+
{{- end }}
303+
{{- end }}
250304
storage:
251305
dataHome: {{ .Values.datanode.storage.dataHome }}
252306
fs:
@@ -306,9 +360,27 @@ spec:
306360
{{- if .Values.flownode.podTemplate.nodeSelector }}
307361
nodeSelector: {{ .Values.flownode.podTemplate.nodeSelector | toYaml | nindent 8 }}
308362
{{- end }}
309-
{{- if .Values.flownode.podTemplate.volumes}}
363+
{{- if .Values.flownode.podTemplate.volumes }}
310364
volumes: {{ .Values.flownode.podTemplate.volumes | toYaml | nindent 8 }}
311365
{{- end }}
366+
{{- if .Values.flownode.logging }}
367+
logging:
368+
{{- if .Values.flownode.logging.level }}
369+
level: {{ .Values.flownode.logging.level }}
370+
{{- end }}
371+
{{- if .Values.flownode.logging.format }}
372+
format: {{ .Values.flownode.logging.format }}
373+
{{- end }}
374+
{{- if .Values.flownode.logging.logsDir }}
375+
logsDir: {{ .Values.flownode.logging.logsDir }}
376+
{{- end }}
377+
{{- if .Values.flownode.logging.onlyLogToStdout }}
378+
onlyLogToStdout: {{ .Values.flownode.logging.onlyLogToStdout }}
379+
{{- end }}
380+
{{- if .Values.flownode.logging.persistentWithData }}
381+
persistentWithData: {{ .Values.flownode.logging.persistentWithData }}
382+
{{- end }}
383+
{{- end }}
312384
{{- end }}
313385
{{- if (and .Values.prometheusMonitor (eq .Values.prometheusMonitor.enabled true ))}}
314386
prometheusMonitor: {{- toYaml .Values.prometheusMonitor | nindent 4 }}
@@ -362,3 +434,40 @@ spec:
362434
kafka: {{- toYaml .Values.remoteWal.kafka | nindent 6 }}
363435
{{- end }}
364436
{{- end }}
437+
{{- if .Values.monitoring.enabled }}
438+
monitoring:
439+
enabled: {{ .Values.monitoring.enabled }}
440+
{{- if .Values.monitoring.standalone }}
441+
standalone: {{ .Values.monitoring.standalone | toYaml | nindent 6 }}
442+
{{- end }}
443+
{{- if .Values.monitoring.logsCollection.pipeline.data }}
444+
logsCollection:
445+
pipeline:
446+
data: {{ .Values.monitoring.logsCollection.pipeline.data }}
447+
{{- end }}
448+
{{- if .Values.monitoring.vector }}
449+
vector:
450+
image: '{{ .Values.monitoring.vector.registry }}/{{ .Values.monitoring.vector.repository }}:{{ .Values.monitoring.vector.tag }}'
451+
{{- if .Values.monitoring.vector.resource }}
452+
resource: {{- toYaml .Values.monitoring.vector.resource | nindent 8 }}
453+
{{- end }}
454+
{{- end }}
455+
{{- end }}
456+
{{- if .Values.logging}}
457+
logging:
458+
{{- if .Values.logging.level }}
459+
level: {{ .Values.logging.level }}
460+
{{- end }}
461+
{{- if .Values.logging.format }}
462+
format: {{ .Values.logging.format }}
463+
{{- end }}
464+
{{- if .Values.logging.logsDir }}
465+
logsDir: {{ .Values.logging.logsDir }}
466+
{{- end }}
467+
{{- if .Values.logging.onlyLogToStdout }}
468+
onlyLogToStdout: {{ .Values.logging.onlyLogToStdout }}
469+
{{- end }}
470+
{{- if .Values.logging.persistentWithData }}
471+
persistentWithData: {{ .Values.logging.persistentWithData }}
472+
{{- end }}
473+
{{- end }}

0 commit comments

Comments
 (0)