Add metrics for unhandled errors

lucko · Feb 29, 2024 · 7800bec · 7800bec
1 parent d696a40
commit 7800bec
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 0 deletions.
diff --git a/src/main/java/me/lucko/bytebin/content/ContentIndexDatabase.java b/src/main/java/me/lucko/bytebin/content/ContentIndexDatabase.java
@@ -32,6 +32,7 @@
 import com.j256.ormlite.jdbc.JdbcConnectionSource;
 import com.j256.ormlite.support.ConnectionSource;
 import com.j256.ormlite.table.TableUtils;
+import io.prometheus.client.Counter;
 import io.prometheus.client.Gauge;
 import me.lucko.bytebin.content.storage.StorageBackend;
 import org.apache.logging.log4j.LogManager;
@@ -72,6 +73,12 @@ public class ContentIndexDatabase implements AutoCloseable {
             .labelNames("type", "backend")
             .register();
 
+    private static final Counter DB_ERROR_COUNTER = Counter.build()
+            .name("bytebin_db_error_total")
+            .labelNames("operation")
+            .help("Counts the number of errors that have occurred when interacting with the index database")
+            .register();
+
     public static ContentIndexDatabase initialise(Collection<StorageBackend> backends) throws SQLException {
         // ensure the db directory exists, sqlite won't create it
         try {
@@ -110,6 +117,7 @@ public void put(Content content) {
             this.dao.createOrUpdate(content);
         } catch (SQLException e) {
             LOGGER.error("[INDEX DB] Error performing sql operation", e);
+            DB_ERROR_COUNTER.labels("put").inc();
         }
     }
 
@@ -118,6 +126,7 @@ public Content get(String key) {
             return this.dao.queryForId(key);
         } catch (SQLException e) {
             LOGGER.error("[INDEX DB] Error performing sql operation", e);
+            DB_ERROR_COUNTER.labels("get").inc();
             return null;
         }
     }
@@ -127,6 +136,7 @@ public void putAll(Collection<Content> content) {
             this.dao.create(content);
         } catch (Exception e) {
             LOGGER.error("[INDEX DB] Error performing sql operation", e);
+            DB_ERROR_COUNTER.labels("putAll").inc();
         }
     }
 
@@ -135,6 +145,7 @@ public void remove(String key) {
             this.dao.deleteById(key);
         } catch (SQLException e) {
             LOGGER.error("[INDEX DB] Error performing sql operation", e);
+            DB_ERROR_COUNTER.labels("remove").inc();
         }
     }
 
@@ -147,6 +158,7 @@ public Collection<Content> getExpired() {
                     .query();
         } catch (SQLException e) {
             LOGGER.error("[INDEX DB] Error performing sql operation", e);
+            DB_ERROR_COUNTER.labels("getExpired").inc();
             return Collections.emptyList();
         }
     }
@@ -170,6 +182,7 @@ private Map<ContentStorageKeys, Long> queryStringToIntMap(String returnExpr) {
             return map;
         } catch (Exception e) {
             LOGGER.error("[INDEX DB] Error performing sql operation", e);
+            DB_ERROR_COUNTER.labels("queryStringToIntMap").inc();
             return Collections.emptyMap();
         }
     }

diff --git a/src/main/java/me/lucko/bytebin/content/ContentStorageHandler.java b/src/main/java/me/lucko/bytebin/content/ContentStorageHandler.java
@@ -59,6 +59,12 @@ public class ContentStorageHandler implements CacheLoader<String, Content> {
             .help("Counts the number of times content was written to the backend")
             .register();
 
+    private static final Counter BACKEND_ERROR_COUNTER = Counter.build()
+            .name("bytebin_backend_error_total")
+            .labelNames("backend", "operation")
+            .help("Counts the number of errors that have occurred when interacting with the backend")
+            .register();
+
     /** An index of all stored content */
     private final ContentIndexDatabase index;
 
@@ -100,6 +106,7 @@ public ContentStorageHandler(ContentIndexDatabase contentIndex, Collection<Stora
         StorageBackend backend = this.backends.get(backendId);
         if (backend == null) {
             LOGGER.error("Unable to load " + key + " - no such backend '" + backendId + "'");
+            BACKEND_ERROR_COUNTER.labels(backendId, "load").inc();
             return Content.EMPTY_CONTENT;
         }
 
@@ -115,6 +122,7 @@ public ContentStorageHandler(ContentIndexDatabase contentIndex, Collection<Stora
             }
         } catch (Exception e) {
             LOGGER.warn("[STORAGE] Unable to load '" + key + "' from the '" + backendId + "' backend", e);
+            BACKEND_ERROR_COUNTER.labels(backendId, "load").inc();
         }
 
         return Content.EMPTY_CONTENT;
@@ -142,6 +150,7 @@ public void save(Content content) {
             backend.save(content);
         } catch (Exception e) {
             LOGGER.warn("[STORAGE] Unable to save '" + content.getKey() + "' to the '" + backendId + "' backend", e);
+            BACKEND_ERROR_COUNTER.labels(backendId, "save").inc();
         }
     }
 
@@ -158,6 +167,7 @@ public void delete(Content content) {
         StorageBackend backend = this.backends.get(backendId);
         if (backend == null) {
             LOGGER.error("[STORAGE] Unable to delete " + key + " - no such backend '" + backendId + "'");
+            BACKEND_ERROR_COUNTER.labels(backendId, "delete").inc();
             return;
         }
 
@@ -166,6 +176,7 @@ public void delete(Content content) {
             backend.delete(key);
         } catch (Exception e) {
             LOGGER.warn("[STORAGE] Unable to delete '" + key + "' from the '" + backend.getBackendId() + "' backend", e);
+            BACKEND_ERROR_COUNTER.labels(backendId, "delete").inc();
         }
 
         // remove the entry from the index

diff --git a/src/main/java/me/lucko/bytebin/http/BytebinServer.java b/src/main/java/me/lucko/bytebin/http/BytebinServer.java
@@ -41,6 +41,7 @@
 import me.lucko.bytebin.content.ContentLoader;
 import me.lucko.bytebin.content.ContentStorageHandler;
 import me.lucko.bytebin.http.admin.BulkDeleteHandler;
+import me.lucko.bytebin.util.ExceptionHandler;
 import me.lucko.bytebin.util.ExpiryHandler;
 import me.lucko.bytebin.util.RateLimitHandler;
 import me.lucko.bytebin.util.RateLimiter;
@@ -90,6 +91,7 @@ public BytebinServer(ContentStorageHandler storageHandler, ContentLoader content
             } else {
                 // handle unexpected errors: log stack trace and send a generic response
                 LOGGER.error("Error thrown by handler", cause);
+                ExceptionHandler.UNCAUGHT_ERROR_COUNTER.labels(cause.getClass().getSimpleName()).inc();
                 ctx.setResponseCode(StatusCode.NOT_FOUND)
                         .setResponseType(MediaType.TEXT)
                         .send("Invalid path");

diff --git a/src/main/java/me/lucko/bytebin/util/ExceptionHandler.java b/src/main/java/me/lucko/bytebin/util/ExceptionHandler.java
@@ -25,6 +25,7 @@
 
 package me.lucko.bytebin.util;
 
+import io.prometheus.client.Counter;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -34,8 +35,15 @@ public class ExceptionHandler implements Thread.UncaughtExceptionHandler {
     /** Logger instance */
     private static final Logger LOGGER = LogManager.getLogger(ExceptionHandler.class);
 
+    public static final Counter UNCAUGHT_ERROR_COUNTER = Counter.build()
+            .name("bytebin_uncaught_error_total")
+            .labelNames("type")
+            .help("Counts the number of uncaught errors that have occurred")
+            .register();
+
     @Override
     public void uncaughtException(Thread t, Throwable e) {
         LOGGER.error("Uncaught exception thrown by thread " + t.getName(), e);
+        ExceptionHandler.UNCAUGHT_ERROR_COUNTER.labels(e.getClass().getSimpleName()).inc();
     }
 }