From effefbb316805d778a9a0ea147c74512701350ca Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 20 Oct 2024 18:51:39 +0100 Subject: [PATCH 1/4] Update Loki to v3.2.1 Config based on https://github.com/grafana/loki/blob/5d78a3a3fd1f/cmd/loki/loki-local-config.yaml Signed-off-by: Bryan Boreham --- .../cluster-infra/6b_loki_stateful_set.yaml | 65 +++++++++++-------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml b/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml index 59e7a59e7..6ed7cbe37 100644 --- a/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml +++ b/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml @@ -7,39 +7,50 @@ metadata: data: loki.yaml: | auth_enabled: false - ingester: - chunk_idle_period: 15m - chunk_block_size: 262144 - lifecycler: - ring: - kvstore: - store: inmemory - replication_factor: 1 - limits_config: - enforce_metric_name: false - reject_old_samples: true - reject_old_samples_max_age: 168h + + common: + instance_addr: 127.0.0.1 + path_prefix: /data/loki + storage: + filesystem: + chunks_directory: /data/loki/chunks + rules_directory: /data/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + + query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + schema_config: configs: - - from: 2018-04-15 - store: boltdb + - from: 2024-10-20 + store: tsdb object_store: filesystem - schema: v9 + schema: v13 index: prefix: index_ - period: 168h + period: 24h + + pattern_ingester: + enabled: true + metric_aggregation: + enabled: true + loki_address: localhost:3100 + + frontend: + encoding: protobuf + server: http_listen_port: 3100 - storage_config: - boltdb: - directory: /data/loki/index - filesystem: - directory: /data/loki/chunks - chunk_store_config: - max_look_back_period: 0s - table_manager: - retention_deletes_enabled: true - retention_period: 2184h + grpc_listen_port: 9096 + log_level: debug + grpc_server_max_concurrent_streams: 1000 --- apiVersion: v1 kind: Service @@ -86,7 +97,7 @@ spec: runAsUser: 10001 containers: - name: loki - image: grafana/loki:1.4.1 + image: grafana/loki:3.2.1 imagePullPolicy: IfNotPresent args: - "-config.file=/etc/loki/loki.yaml" From d2f381877ba701558e70381ecee60d0ff374dad1 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 20 Oct 2024 18:58:20 +0100 Subject: [PATCH 2/4] Add resource requests Let's be polite. Signed-off-by: Bryan Boreham --- prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml b/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml index 6ed7cbe37..ce747c2e4 100644 --- a/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml +++ b/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml @@ -101,6 +101,9 @@ spec: imagePullPolicy: IfNotPresent args: - "-config.file=/etc/loki/loki.yaml" + requests: + cpu: 100m + memory: 100MB volumeMounts: - name: config mountPath: /etc/loki From 60d99c0162d384690bb55309ac6ac075e28c7603 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 20 Oct 2024 19:10:05 +0100 Subject: [PATCH 3/4] Remove footgun Killing loki is very rarely going to make anything better. Signed-off-by: Bryan Boreham --- prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml b/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml index ce747c2e4..09dd722ad 100644 --- a/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml +++ b/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml @@ -114,11 +114,6 @@ spec: - name: http-metrics containerPort: 3100 protocol: TCP - livenessProbe: - httpGet: - path: /ready - port: http-metrics - initialDelaySeconds: 45 readinessProbe: httpGet: path: /ready From f20f1cd00b5d5a5691f108cd92dcbe10f7abf65a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 2 Nov 2024 12:18:24 +0000 Subject: [PATCH 4/4] Loki: re-enable retention and reduce logging More like what we had before. Signed-off-by: Bryan Boreham --- .../manifests/cluster-infra/6b_loki_stateful_set.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml b/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml index 09dd722ad..7422b3f66 100644 --- a/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml +++ b/prombench/manifests/cluster-infra/6b_loki_stateful_set.yaml @@ -49,8 +49,13 @@ data: server: http_listen_port: 3100 grpc_listen_port: 9096 - log_level: debug + log_level: info grpc_server_max_concurrent_streams: 1000 + + compactor: + retention_enabled: true + limits_config: + retention_period: 2184h --- apiVersion: v1 kind: Service