Integrate with Loki and add alert rules based on logs (#662)

canonical · Jan 21, 2025 · eb3a4d3 · eb3a4d3
1 parent b21668a
commit eb3a4d3
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 11 deletions.
diff --git a/src/loki_alert_rules/synapse_error_found.rule b/src/loki_alert_rules/synapse_error_found.rule
@@ -0,0 +1,8 @@
+alert: SynapseErrorFound
+expr: rate({%%juju_topology%%, pebble_service="synapse"} |= " - ERROR - "[5m]) > 0
+for: 5m
+labels:
+  severity: error
+annotations:
+  summary: Synapse errors detected.
+  description: This alert is triggered when Synapse logs an error level message in the last 5 minutes.
diff --git a/src/loki_alert_rules/synapse_mjolnir_error_fond.rule b/src/loki_alert_rules/synapse_mjolnir_error_fond.rule
@@ -0,0 +1,8 @@
+alert: SynapseMjolnirErrorFound
+expr: rate({%%juju_topology%%, pebble_service="mjolnir"} |= "[ERROR]"[5m]) > 0
+for: 5m
+labels:
+  severity: error
+annotations:
+  summary: Synapse Mjolnir error detected.
+  description: This alert is triggered when Synapse Mjolnir logs an error level message in the last 5 minutes.
diff --git a/src/loki_alert_rules/synapse_nginx_emerg_found.rule b/src/loki_alert_rules/synapse_nginx_emerg_found.rule
@@ -0,0 +1,8 @@
+alert: SynapseNGINXEmergFound
+expr: rate({%%juju_topology%%, pebble_service="synapse-nginx"} |= "[emerg]"[5m]) > 0
+for: 5m
+labels:
+  severity: critical
+annotations:
+  summary: Synapse NGINX emerg detected.
+  description: This alert is triggered when Synapse NGINX logs an emerg level message in the last 5 minutes.
diff --git a/src/observability.py b/src/observability.py
@@ -6,13 +6,12 @@
 
 import ops
 from charms.grafana_k8s.v0.grafana_dashboard import GrafanaDashboardProvider
-from charms.loki_k8s.v1.loki_push_api import LogProxyConsumer
+from charms.loki_k8s.v1.loki_push_api import LogForwarder
 from charms.prometheus_k8s.v0.prometheus_scrape import MetricsEndpointProvider
 
 import synapse
 
 CONTAINER_NAME = "synapse"
-LOG_PATHS = ["/debug.log*", "/errors.log*"]
 
 
 class Observability:  # pylint: disable=too-few-public-methods
@@ -47,12 +46,4 @@ def __init__(self, charm: ops.CharmBase):
                 },
             ],
         )
-        self._logging = LogProxyConsumer(
-            charm,
-            relation_name="logging",
-            logs_scheme={
-                f"{CONTAINER_NAME}": {
-                    "log-files": LOG_PATHS,
-                },
-            },
-        )
+        self._logging = LogForwarder(charm, relation_name="logging")