Add bulk delete endpoint

lucko · Jan 25, 2024 · b92d0bb · b92d0bb
1 parent a820389
commit b92d0bb
Show file tree

Hide file tree

Showing 15 changed files with 192 additions and 83 deletions.
diff --git a/src/main/java/me/lucko/bytebin/Bytebin.java b/src/main/java/me/lucko/bytebin/Bytebin.java
@@ -25,8 +25,11 @@
 
 package me.lucko.bytebin;
 
+import com.google.common.collect.ImmutableSet;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
+import io.jooby.ExecutionMode;
+import io.jooby.Jooby;
+import io.prometheus.client.hotspot.DefaultExports;
 import me.lucko.bytebin.content.Content;
 import me.lucko.bytebin.content.ContentIndexDatabase;
 import me.lucko.bytebin.content.ContentLoader;
@@ -45,16 +48,11 @@
 import me.lucko.bytebin.util.RateLimitHandler;
 import me.lucko.bytebin.util.RateLimiter;
 import me.lucko.bytebin.util.TokenGenerator;
-
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.io.IoBuilder;
 
-import io.jooby.ExecutionMode;
-import io.jooby.Jooby;
-import io.prometheus.client.hotspot.DefaultExports;
-
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
@@ -161,7 +159,7 @@ public Bytebin(Configuration config) throws Exception {
                 config.getString(Option.HOST, "0.0.0.0"),
                 config.getInt(Option.PORT, 8080),
                 metrics,
-                new RateLimitHandler(config.getStringList(Option.API_KEYS)),
+                new RateLimitHandler(config.getStringList(Option.RATELIMIT_API_KEYS)),
                 new RateLimiter(
                         // by default, allow posts at a rate of 30 times every 10 minutes (every 20s)
                         config.getInt(Option.POST_RATE_LIMIT_PERIOD, 10),
@@ -180,7 +178,8 @@ public Bytebin(Configuration config) throws Exception {
                 new TokenGenerator(config.getInt(Option.KEY_LENGTH, 7)),
                 (Content.MEGABYTE_LENGTH * config.getInt(Option.MAX_CONTENT_LENGTH, 10)),
                 expiryHandler,
-                config.getStringMap(Option.HTTP_HOST_ALIASES)
+                config.getStringMap(Option.HTTP_HOST_ALIASES),
+                ImmutableSet.copyOf(config.getStringList(Option.ADMIN_API_KEYS))
         ));
         this.server.start();
 

diff --git a/src/main/java/me/lucko/bytebin/content/ContentIndexDatabase.java b/src/main/java/me/lucko/bytebin/content/ContentIndexDatabase.java
@@ -32,14 +32,11 @@
 import com.j256.ormlite.jdbc.JdbcConnectionSource;
 import com.j256.ormlite.support.ConnectionSource;
 import com.j256.ormlite.table.TableUtils;
-
+import io.prometheus.client.Gauge;
 import me.lucko.bytebin.content.storage.StorageBackend;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import io.prometheus.client.Gauge;
-
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;

diff --git a/src/main/java/me/lucko/bytebin/content/ContentStorageHandler.java b/src/main/java/me/lucko/bytebin/content/ContentStorageHandler.java
@@ -27,16 +27,14 @@
 
 import com.github.benmanes.caffeine.cache.CacheLoader;
 import com.google.common.collect.ImmutableMap;
-
+import io.prometheus.client.Counter;
 import me.lucko.bytebin.content.storage.StorageBackend;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.checkerframework.checker.nullness.qual.NonNull;
 
-import io.prometheus.client.Counter;
-
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ScheduledExecutorService;
@@ -147,6 +145,35 @@ public void save(Content content) {
         }
     }
 
+    /**
+     * Delete content.
+     *
+     * @param content the content to delete
+     */
+    public void delete(Content content) {
+        String key = content.getKey();
+
+        // find the backend that the content is stored in
+        String backendId = content.getBackendId();
+        StorageBackend backend = this.backends.get(backendId);
+        if (backend == null) {
+            LOGGER.error("[STORAGE] Unable to delete " + key + " - no such backend '" + backendId + "'");
+            return;
+        }
+
+        // delete the data from the backend
+        try {
+            backend.delete(key);
+        } catch (Exception e) {
+            LOGGER.warn("[STORAGE] Unable to delete '" + key + "' from the '" + backend.getBackendId() + "' backend", e);
+        }
+
+        // remove the entry from the index
+        this.index.remove(key);
+
+        LOGGER.info("[STORAGE] Deleted '" + key + "' from the '" + backendId + "' backend");
+    }
+
     /**
      * Invalidates/deletes any expired content and updates the metrics gauges
      */
@@ -155,31 +182,35 @@ public void runInvalidationAndRecordMetrics() {
         Collection<Content> expired = this.index.getExpired();
 
         for (Content metadata : expired) {
-            String key = metadata.getKey();
+            delete(metadata);
+        }
 
-            // find the backend that the content is stored in
-            String backendId = metadata.getBackendId();
-            StorageBackend backend = this.backends.get(backendId);
-            if (backend == null) {
-                LOGGER.error("[STORAGE] Unable to delete " + key + " - no such backend '" + backendId + "'");
-                continue;
-            }
+        // update metrics
+        this.index.recordMetrics();
+    }
 
-            // delete the data from the backend
-            try {
-                backend.delete(key);
-            } catch (Exception e) {
-                LOGGER.warn("[STORAGE] Unable to delete '" + key + "' from the '" + backend.getBackendId() + "' backend", e);
+    /**
+     * Bulk deletes the provided keys
+     *
+     * @param keys the keys to delete
+     * @return how many entries were actually deleted
+     */
+    public int bulkDelete(List<String> keys) {
+        int count = 0;
+        for (String key : keys) {
+            Content content = this.index.get(key);
+            if (content == null) {
+                continue;
             }
 
-            // remove the entry from the index
-            this.index.remove(key);
-
-            LOGGER.info("[STORAGE] Deleted '" + key + "' from the '" + backendId + "' backend");
+            delete(content);
+            count++;
         }
 
         // update metrics
         this.index.recordMetrics();
+
+        return count;
     }
 
     public Executor getExecutor() {

diff --git a/src/main/java/me/lucko/bytebin/content/storage/AuditTask.java b/src/main/java/me/lucko/bytebin/content/storage/AuditTask.java
@@ -25,9 +25,7 @@
 
 package me.lucko.bytebin.content.storage;
 
-import me.lucko.bytebin.content.Content;
 import me.lucko.bytebin.content.ContentIndexDatabase;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 

diff --git a/src/main/java/me/lucko/bytebin/content/storage/LocalDiskBackend.java b/src/main/java/me/lucko/bytebin/content/storage/LocalDiskBackend.java
@@ -27,7 +27,6 @@
 
 import me.lucko.bytebin.content.Content;
 import me.lucko.bytebin.util.ContentEncoding;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 

diff --git a/src/main/java/me/lucko/bytebin/content/storage/S3Backend.java b/src/main/java/me/lucko/bytebin/content/storage/S3Backend.java
@@ -26,10 +26,8 @@
 package me.lucko.bytebin.content.storage;
 
 import me.lucko.bytebin.content.Content;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-
 import software.amazon.awssdk.core.ResponseInputStream;
 import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.services.s3.S3Client;

diff --git a/src/main/java/me/lucko/bytebin/http/BytebinServer.java b/src/main/java/me/lucko/bytebin/http/BytebinServer.java
@@ -25,17 +25,6 @@
 
 package me.lucko.bytebin.http;
 
-import me.lucko.bytebin.Bytebin;
-import me.lucko.bytebin.content.ContentLoader;
-import me.lucko.bytebin.content.ContentStorageHandler;
-import me.lucko.bytebin.util.ExpiryHandler;
-import me.lucko.bytebin.util.RateLimitHandler;
-import me.lucko.bytebin.util.RateLimiter;
-import me.lucko.bytebin.util.TokenGenerator;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
 import io.jooby.AssetHandler;
 import io.jooby.AssetSource;
 import io.jooby.Context;
@@ -48,9 +37,20 @@
 import io.jooby.StatusCode;
 import io.jooby.exception.StatusCodeException;
 import io.prometheus.client.Counter;
+import me.lucko.bytebin.Bytebin;
+import me.lucko.bytebin.content.ContentLoader;
+import me.lucko.bytebin.content.ContentStorageHandler;
+import me.lucko.bytebin.http.admin.BulkDeleteHandler;
+import me.lucko.bytebin.util.ExpiryHandler;
+import me.lucko.bytebin.util.RateLimitHandler;
+import me.lucko.bytebin.util.RateLimiter;
+import me.lucko.bytebin.util.TokenGenerator;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.time.Duration;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CompletionException;
 
 public class BytebinServer extends Jooby {
@@ -64,7 +64,7 @@ public class BytebinServer extends Jooby {
             .labelNames("method", "useragent")
             .register();
 
-    public BytebinServer(ContentStorageHandler storageHandler, ContentLoader contentLoader, String host, int port, boolean metrics, RateLimitHandler rateLimitHandler, RateLimiter postRateLimiter, RateLimiter putRateLimiter, RateLimiter readRateLimiter, TokenGenerator contentTokenGenerator, long maxContentLength, ExpiryHandler expiryHandler, Map<String, String> hostAliases) {
+    public BytebinServer(ContentStorageHandler storageHandler, ContentLoader contentLoader, String host, int port, boolean metrics, RateLimitHandler rateLimitHandler, RateLimiter postRateLimiter, RateLimiter putRateLimiter, RateLimiter readRateLimiter, TokenGenerator contentTokenGenerator, long maxContentLength, ExpiryHandler expiryHandler, Map<String, String> hostAliases, Set<String> adminApiKeys) {
         ServerOptions serverOpts = new ServerOptions();
         serverOpts.setHost(host);
         serverOpts.setPort(port);
@@ -136,6 +136,10 @@ public BytebinServer(ContentStorageHandler storageHandler, ContentLoader content
             get("/{id:[a-zA-Z0-9]+}", new GetHandler(this, readRateLimiter, rateLimitHandler, contentLoader));
             put("/{id:[a-zA-Z0-9]+}", new PutHandler(this, putRateLimiter, rateLimitHandler, storageHandler, contentLoader, maxContentLength, expiryHandler));
         });
+
+        routes(() -> {
+            post("/admin/bulkdelete", new BulkDeleteHandler(this, storageHandler, adminApiKeys));
+        });
     }
 
     public static String getMetricsLabel(Context ctx) {

diff --git a/src/main/java/me/lucko/bytebin/http/GetHandler.java b/src/main/java/me/lucko/bytebin/http/GetHandler.java
@@ -25,30 +25,26 @@
 
 package me.lucko.bytebin.http;
 
+import io.jooby.Context;
+import io.jooby.MediaType;
+import io.jooby.Route;
+import io.jooby.StatusCode;
+import io.jooby.exception.StatusCodeException;
 import me.lucko.bytebin.content.ContentLoader;
 import me.lucko.bytebin.util.ContentEncoding;
 import me.lucko.bytebin.util.Gzip;
 import me.lucko.bytebin.util.RateLimitHandler;
 import me.lucko.bytebin.util.RateLimiter;
 import me.lucko.bytebin.util.TokenGenerator;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import io.jooby.Context;
-import io.jooby.MediaType;
-import io.jooby.Route;
-import io.jooby.StatusCode;
-import io.jooby.exception.StatusCodeException;
-
+import javax.annotation.Nonnull;
 import java.io.IOException;
 import java.time.Instant;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
-import java.util.stream.Collectors;
-
-import javax.annotation.Nonnull;
 
 public final class GetHandler implements Route.Handler {
 

diff --git a/src/main/java/me/lucko/bytebin/http/MetricsHandler.java b/src/main/java/me/lucko/bytebin/http/MetricsHandler.java
@@ -32,9 +32,8 @@
 import io.prometheus.client.CollectorRegistry;
 import io.prometheus.client.exporter.common.TextFormat;
 
-import java.io.OutputStreamWriter;
-
 import javax.annotation.Nonnull;
+import java.io.OutputStreamWriter;
 
 public final class MetricsHandler implements Route.Handler {
 

diff --git a/src/main/java/me/lucko/bytebin/http/PostHandler.java b/src/main/java/me/lucko/bytebin/http/PostHandler.java
@@ -25,6 +25,12 @@
 
 package me.lucko.bytebin.http;
 
+import io.jooby.Context;
+import io.jooby.MediaType;
+import io.jooby.Route;
+import io.jooby.StatusCode;
+import io.jooby.exception.StatusCodeException;
+import io.prometheus.client.Summary;
 import me.lucko.bytebin.content.Content;
 import me.lucko.bytebin.content.ContentLoader;
 import me.lucko.bytebin.content.ContentStorageHandler;
@@ -34,24 +40,15 @@
 import me.lucko.bytebin.util.RateLimitHandler;
 import me.lucko.bytebin.util.RateLimiter;
 import me.lucko.bytebin.util.TokenGenerator;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import io.jooby.Context;
-import io.jooby.MediaType;
-import io.jooby.Route;
-import io.jooby.StatusCode;
-import io.jooby.exception.StatusCodeException;
-import io.prometheus.client.Summary;
-
+import javax.annotation.Nonnull;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 
-import javax.annotation.Nonnull;
-
 public final class PostHandler implements Route.Handler {
 
     /** Logger instance */

diff --git a/src/main/java/me/lucko/bytebin/http/PutHandler.java b/src/main/java/me/lucko/bytebin/http/PutHandler.java
@@ -25,6 +25,10 @@
 
 package me.lucko.bytebin.http;
 
+import io.jooby.Context;
+import io.jooby.Route;
+import io.jooby.StatusCode;
+import io.jooby.exception.StatusCodeException;
 import me.lucko.bytebin.content.ContentLoader;
 import me.lucko.bytebin.content.ContentStorageHandler;
 import me.lucko.bytebin.util.ContentEncoding;
@@ -33,22 +37,15 @@
 import me.lucko.bytebin.util.RateLimitHandler;
 import me.lucko.bytebin.util.RateLimiter;
 import me.lucko.bytebin.util.TokenGenerator;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import io.jooby.Context;
-import io.jooby.Route;
-import io.jooby.StatusCode;
-import io.jooby.exception.StatusCodeException;
-
+import javax.annotation.Nonnull;
 import java.util.Date;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicReference;
 
-import javax.annotation.Nonnull;
-
 public final class PutHandler implements Route.Handler {
 
     /** Logger instance */