From b92d0bbc37bdbfb2c4d5e23a86cd916ce993a342 Mon Sep 17 00:00:00 2001 From: Luck Date: Thu, 25 Jan 2024 23:36:33 +0000 Subject: [PATCH] Add bulk delete endpoint --- src/main/java/me/lucko/bytebin/Bytebin.java | 15 ++- .../bytebin/content/ContentIndexDatabase.java | 5 +- .../content/ContentStorageHandler.java | 73 ++++++++++---- .../bytebin/content/storage/AuditTask.java | 2 - .../content/storage/LocalDiskBackend.java | 1 - .../bytebin/content/storage/S3Backend.java | 2 - .../me/lucko/bytebin/http/BytebinServer.java | 28 +++--- .../me/lucko/bytebin/http/GetHandler.java | 16 ++-- .../me/lucko/bytebin/http/MetricsHandler.java | 3 +- .../me/lucko/bytebin/http/PostHandler.java | 17 ++-- .../me/lucko/bytebin/http/PutHandler.java | 13 +-- .../bytebin/http/admin/BulkDeleteHandler.java | 95 +++++++++++++++++++ .../me/lucko/bytebin/util/Configuration.java | 3 +- .../lucko/bytebin/util/ContentEncoding.java | 1 - .../lucko/bytebin/util/RateLimitHandler.java | 1 - 15 files changed, 192 insertions(+), 83 deletions(-) create mode 100644 src/main/java/me/lucko/bytebin/http/admin/BulkDeleteHandler.java diff --git a/src/main/java/me/lucko/bytebin/Bytebin.java b/src/main/java/me/lucko/bytebin/Bytebin.java index 55c34ae..ed250fc 100644 --- a/src/main/java/me/lucko/bytebin/Bytebin.java +++ b/src/main/java/me/lucko/bytebin/Bytebin.java @@ -25,8 +25,11 @@ package me.lucko.bytebin; +import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ThreadFactoryBuilder; - +import io.jooby.ExecutionMode; +import io.jooby.Jooby; +import io.prometheus.client.hotspot.DefaultExports; import me.lucko.bytebin.content.Content; import me.lucko.bytebin.content.ContentIndexDatabase; import me.lucko.bytebin.content.ContentLoader; @@ -45,16 +48,11 @@ import me.lucko.bytebin.util.RateLimitHandler; import me.lucko.bytebin.util.RateLimiter; import me.lucko.bytebin.util.TokenGenerator; - import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.io.IoBuilder; -import io.jooby.ExecutionMode; -import io.jooby.Jooby; -import io.prometheus.client.hotspot.DefaultExports; - import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -161,7 +159,7 @@ public Bytebin(Configuration config) throws Exception { config.getString(Option.HOST, "0.0.0.0"), config.getInt(Option.PORT, 8080), metrics, - new RateLimitHandler(config.getStringList(Option.API_KEYS)), + new RateLimitHandler(config.getStringList(Option.RATELIMIT_API_KEYS)), new RateLimiter( // by default, allow posts at a rate of 30 times every 10 minutes (every 20s) config.getInt(Option.POST_RATE_LIMIT_PERIOD, 10), @@ -180,7 +178,8 @@ public Bytebin(Configuration config) throws Exception { new TokenGenerator(config.getInt(Option.KEY_LENGTH, 7)), (Content.MEGABYTE_LENGTH * config.getInt(Option.MAX_CONTENT_LENGTH, 10)), expiryHandler, - config.getStringMap(Option.HTTP_HOST_ALIASES) + config.getStringMap(Option.HTTP_HOST_ALIASES), + ImmutableSet.copyOf(config.getStringList(Option.ADMIN_API_KEYS)) )); this.server.start(); diff --git a/src/main/java/me/lucko/bytebin/content/ContentIndexDatabase.java b/src/main/java/me/lucko/bytebin/content/ContentIndexDatabase.java index 32d1e9d..53b9c78 100644 --- a/src/main/java/me/lucko/bytebin/content/ContentIndexDatabase.java +++ b/src/main/java/me/lucko/bytebin/content/ContentIndexDatabase.java @@ -32,14 +32,11 @@ import com.j256.ormlite.jdbc.JdbcConnectionSource; import com.j256.ormlite.support.ConnectionSource; import com.j256.ormlite.table.TableUtils; - +import io.prometheus.client.Gauge; import me.lucko.bytebin.content.storage.StorageBackend; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import io.prometheus.client.Gauge; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; diff --git a/src/main/java/me/lucko/bytebin/content/ContentStorageHandler.java b/src/main/java/me/lucko/bytebin/content/ContentStorageHandler.java index 8a42ff4..2a3a3d9 100644 --- a/src/main/java/me/lucko/bytebin/content/ContentStorageHandler.java +++ b/src/main/java/me/lucko/bytebin/content/ContentStorageHandler.java @@ -27,16 +27,14 @@ import com.github.benmanes.caffeine.cache.CacheLoader; import com.google.common.collect.ImmutableMap; - +import io.prometheus.client.Counter; import me.lucko.bytebin.content.storage.StorageBackend; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.checkerframework.checker.nullness.qual.NonNull; -import io.prometheus.client.Counter; - import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.ScheduledExecutorService; @@ -147,6 +145,35 @@ public void save(Content content) { } } + /** + * Delete content. + * + * @param content the content to delete + */ + public void delete(Content content) { + String key = content.getKey(); + + // find the backend that the content is stored in + String backendId = content.getBackendId(); + StorageBackend backend = this.backends.get(backendId); + if (backend == null) { + LOGGER.error("[STORAGE] Unable to delete " + key + " - no such backend '" + backendId + "'"); + return; + } + + // delete the data from the backend + try { + backend.delete(key); + } catch (Exception e) { + LOGGER.warn("[STORAGE] Unable to delete '" + key + "' from the '" + backend.getBackendId() + "' backend", e); + } + + // remove the entry from the index + this.index.remove(key); + + LOGGER.info("[STORAGE] Deleted '" + key + "' from the '" + backendId + "' backend"); + } + /** * Invalidates/deletes any expired content and updates the metrics gauges */ @@ -155,31 +182,35 @@ public void runInvalidationAndRecordMetrics() { Collection expired = this.index.getExpired(); for (Content metadata : expired) { - String key = metadata.getKey(); + delete(metadata); + } - // find the backend that the content is stored in - String backendId = metadata.getBackendId(); - StorageBackend backend = this.backends.get(backendId); - if (backend == null) { - LOGGER.error("[STORAGE] Unable to delete " + key + " - no such backend '" + backendId + "'"); - continue; - } + // update metrics + this.index.recordMetrics(); + } - // delete the data from the backend - try { - backend.delete(key); - } catch (Exception e) { - LOGGER.warn("[STORAGE] Unable to delete '" + key + "' from the '" + backend.getBackendId() + "' backend", e); + /** + * Bulk deletes the provided keys + * + * @param keys the keys to delete + * @return how many entries were actually deleted + */ + public int bulkDelete(List keys) { + int count = 0; + for (String key : keys) { + Content content = this.index.get(key); + if (content == null) { + continue; } - // remove the entry from the index - this.index.remove(key); - - LOGGER.info("[STORAGE] Deleted '" + key + "' from the '" + backendId + "' backend"); + delete(content); + count++; } // update metrics this.index.recordMetrics(); + + return count; } public Executor getExecutor() { diff --git a/src/main/java/me/lucko/bytebin/content/storage/AuditTask.java b/src/main/java/me/lucko/bytebin/content/storage/AuditTask.java index 6a90366..62c48c2 100644 --- a/src/main/java/me/lucko/bytebin/content/storage/AuditTask.java +++ b/src/main/java/me/lucko/bytebin/content/storage/AuditTask.java @@ -25,9 +25,7 @@ package me.lucko.bytebin.content.storage; -import me.lucko.bytebin.content.Content; import me.lucko.bytebin.content.ContentIndexDatabase; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/me/lucko/bytebin/content/storage/LocalDiskBackend.java b/src/main/java/me/lucko/bytebin/content/storage/LocalDiskBackend.java index c29165b..ffb1e62 100644 --- a/src/main/java/me/lucko/bytebin/content/storage/LocalDiskBackend.java +++ b/src/main/java/me/lucko/bytebin/content/storage/LocalDiskBackend.java @@ -27,7 +27,6 @@ import me.lucko.bytebin.content.Content; import me.lucko.bytebin.util.ContentEncoding; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/me/lucko/bytebin/content/storage/S3Backend.java b/src/main/java/me/lucko/bytebin/content/storage/S3Backend.java index 2d07f65..4e740b6 100644 --- a/src/main/java/me/lucko/bytebin/content/storage/S3Backend.java +++ b/src/main/java/me/lucko/bytebin/content/storage/S3Backend.java @@ -26,10 +26,8 @@ package me.lucko.bytebin.content.storage; import me.lucko.bytebin.content.Content; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; diff --git a/src/main/java/me/lucko/bytebin/http/BytebinServer.java b/src/main/java/me/lucko/bytebin/http/BytebinServer.java index 2a5ac09..e850377 100644 --- a/src/main/java/me/lucko/bytebin/http/BytebinServer.java +++ b/src/main/java/me/lucko/bytebin/http/BytebinServer.java @@ -25,17 +25,6 @@ package me.lucko.bytebin.http; -import me.lucko.bytebin.Bytebin; -import me.lucko.bytebin.content.ContentLoader; -import me.lucko.bytebin.content.ContentStorageHandler; -import me.lucko.bytebin.util.ExpiryHandler; -import me.lucko.bytebin.util.RateLimitHandler; -import me.lucko.bytebin.util.RateLimiter; -import me.lucko.bytebin.util.TokenGenerator; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import io.jooby.AssetHandler; import io.jooby.AssetSource; import io.jooby.Context; @@ -48,9 +37,20 @@ import io.jooby.StatusCode; import io.jooby.exception.StatusCodeException; import io.prometheus.client.Counter; +import me.lucko.bytebin.Bytebin; +import me.lucko.bytebin.content.ContentLoader; +import me.lucko.bytebin.content.ContentStorageHandler; +import me.lucko.bytebin.http.admin.BulkDeleteHandler; +import me.lucko.bytebin.util.ExpiryHandler; +import me.lucko.bytebin.util.RateLimitHandler; +import me.lucko.bytebin.util.RateLimiter; +import me.lucko.bytebin.util.TokenGenerator; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.time.Duration; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletionException; public class BytebinServer extends Jooby { @@ -64,7 +64,7 @@ public class BytebinServer extends Jooby { .labelNames("method", "useragent") .register(); - public BytebinServer(ContentStorageHandler storageHandler, ContentLoader contentLoader, String host, int port, boolean metrics, RateLimitHandler rateLimitHandler, RateLimiter postRateLimiter, RateLimiter putRateLimiter, RateLimiter readRateLimiter, TokenGenerator contentTokenGenerator, long maxContentLength, ExpiryHandler expiryHandler, Map hostAliases) { + public BytebinServer(ContentStorageHandler storageHandler, ContentLoader contentLoader, String host, int port, boolean metrics, RateLimitHandler rateLimitHandler, RateLimiter postRateLimiter, RateLimiter putRateLimiter, RateLimiter readRateLimiter, TokenGenerator contentTokenGenerator, long maxContentLength, ExpiryHandler expiryHandler, Map hostAliases, Set adminApiKeys) { ServerOptions serverOpts = new ServerOptions(); serverOpts.setHost(host); serverOpts.setPort(port); @@ -136,6 +136,10 @@ public BytebinServer(ContentStorageHandler storageHandler, ContentLoader content get("/{id:[a-zA-Z0-9]+}", new GetHandler(this, readRateLimiter, rateLimitHandler, contentLoader)); put("/{id:[a-zA-Z0-9]+}", new PutHandler(this, putRateLimiter, rateLimitHandler, storageHandler, contentLoader, maxContentLength, expiryHandler)); }); + + routes(() -> { + post("/admin/bulkdelete", new BulkDeleteHandler(this, storageHandler, adminApiKeys)); + }); } public static String getMetricsLabel(Context ctx) { diff --git a/src/main/java/me/lucko/bytebin/http/GetHandler.java b/src/main/java/me/lucko/bytebin/http/GetHandler.java index aac8f31..a76fe78 100644 --- a/src/main/java/me/lucko/bytebin/http/GetHandler.java +++ b/src/main/java/me/lucko/bytebin/http/GetHandler.java @@ -25,30 +25,26 @@ package me.lucko.bytebin.http; +import io.jooby.Context; +import io.jooby.MediaType; +import io.jooby.Route; +import io.jooby.StatusCode; +import io.jooby.exception.StatusCodeException; import me.lucko.bytebin.content.ContentLoader; import me.lucko.bytebin.util.ContentEncoding; import me.lucko.bytebin.util.Gzip; import me.lucko.bytebin.util.RateLimitHandler; import me.lucko.bytebin.util.RateLimiter; import me.lucko.bytebin.util.TokenGenerator; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import io.jooby.Context; -import io.jooby.MediaType; -import io.jooby.Route; -import io.jooby.StatusCode; -import io.jooby.exception.StatusCodeException; - +import javax.annotation.Nonnull; import java.io.IOException; import java.time.Instant; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; - -import javax.annotation.Nonnull; public final class GetHandler implements Route.Handler { diff --git a/src/main/java/me/lucko/bytebin/http/MetricsHandler.java b/src/main/java/me/lucko/bytebin/http/MetricsHandler.java index 527089c..72a616a 100644 --- a/src/main/java/me/lucko/bytebin/http/MetricsHandler.java +++ b/src/main/java/me/lucko/bytebin/http/MetricsHandler.java @@ -32,9 +32,8 @@ import io.prometheus.client.CollectorRegistry; import io.prometheus.client.exporter.common.TextFormat; -import java.io.OutputStreamWriter; - import javax.annotation.Nonnull; +import java.io.OutputStreamWriter; public final class MetricsHandler implements Route.Handler { diff --git a/src/main/java/me/lucko/bytebin/http/PostHandler.java b/src/main/java/me/lucko/bytebin/http/PostHandler.java index a754270..c2c19b9 100644 --- a/src/main/java/me/lucko/bytebin/http/PostHandler.java +++ b/src/main/java/me/lucko/bytebin/http/PostHandler.java @@ -25,6 +25,12 @@ package me.lucko.bytebin.http; +import io.jooby.Context; +import io.jooby.MediaType; +import io.jooby.Route; +import io.jooby.StatusCode; +import io.jooby.exception.StatusCodeException; +import io.prometheus.client.Summary; import me.lucko.bytebin.content.Content; import me.lucko.bytebin.content.ContentLoader; import me.lucko.bytebin.content.ContentStorageHandler; @@ -34,24 +40,15 @@ import me.lucko.bytebin.util.RateLimitHandler; import me.lucko.bytebin.util.RateLimiter; import me.lucko.bytebin.util.TokenGenerator; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import io.jooby.Context; -import io.jooby.MediaType; -import io.jooby.Route; -import io.jooby.StatusCode; -import io.jooby.exception.StatusCodeException; -import io.prometheus.client.Summary; - +import javax.annotation.Nonnull; import java.util.Date; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import javax.annotation.Nonnull; - public final class PostHandler implements Route.Handler { /** Logger instance */ diff --git a/src/main/java/me/lucko/bytebin/http/PutHandler.java b/src/main/java/me/lucko/bytebin/http/PutHandler.java index 44f50cc..16c76dc 100644 --- a/src/main/java/me/lucko/bytebin/http/PutHandler.java +++ b/src/main/java/me/lucko/bytebin/http/PutHandler.java @@ -25,6 +25,10 @@ package me.lucko.bytebin.http; +import io.jooby.Context; +import io.jooby.Route; +import io.jooby.StatusCode; +import io.jooby.exception.StatusCodeException; import me.lucko.bytebin.content.ContentLoader; import me.lucko.bytebin.content.ContentStorageHandler; import me.lucko.bytebin.util.ContentEncoding; @@ -33,22 +37,15 @@ import me.lucko.bytebin.util.RateLimitHandler; import me.lucko.bytebin.util.RateLimiter; import me.lucko.bytebin.util.TokenGenerator; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import io.jooby.Context; -import io.jooby.Route; -import io.jooby.StatusCode; -import io.jooby.exception.StatusCodeException; - +import javax.annotation.Nonnull; import java.util.Date; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; -import javax.annotation.Nonnull; - public final class PutHandler implements Route.Handler { /** Logger instance */ diff --git a/src/main/java/me/lucko/bytebin/http/admin/BulkDeleteHandler.java b/src/main/java/me/lucko/bytebin/http/admin/BulkDeleteHandler.java new file mode 100644 index 0000000..e9b24b7 --- /dev/null +++ b/src/main/java/me/lucko/bytebin/http/admin/BulkDeleteHandler.java @@ -0,0 +1,95 @@ +/* + * This file is part of bytebin, licensed under the MIT License. + * + * Copyright (c) lucko (Luck) + * Copyright (c) contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package me.lucko.bytebin.http.admin; + +import com.google.gson.Gson; +import io.jooby.Context; +import io.jooby.Route; +import io.jooby.StatusCode; +import io.jooby.exception.StatusCodeException; +import me.lucko.bytebin.content.ContentStorageHandler; +import me.lucko.bytebin.http.BytebinServer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.annotation.Nonnull; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +public final class BulkDeleteHandler implements Route.Handler { + + private static final String HEADER_API_KEY = "Bytebin-Api-Key"; + + /** Logger instance */ + private static final Logger LOGGER = LogManager.getLogger(BulkDeleteHandler.class); + + private final BytebinServer server; + private final ContentStorageHandler storageHandler; + private final Set apiKeys; + + public BulkDeleteHandler(BytebinServer server, ContentStorageHandler storageHandler, Set apiKeys) { + this.server = server; + this.storageHandler = storageHandler; + this.apiKeys = apiKeys; + } + + @Override + public CompletableFuture apply(@Nonnull Context ctx) { + String apiKey = ctx.header(HEADER_API_KEY).value(""); + if (apiKey.isEmpty() || !this.apiKeys.contains(apiKey)) { + throw new StatusCodeException(StatusCode.UNAUTHORIZED, "API key is invalid"); + } + + // a bit lazy but meh + List list = Arrays.asList(new Gson().fromJson(ctx.body().value(""), String[].class)); + + if (list.isEmpty()) { + throw new StatusCodeException(StatusCode.BAD_REQUEST, "Missing content"); + } + + String ipAddress = ctx.header("x-real-ip").valueOrNull(); + if (ipAddress == null) { + ipAddress = ctx.getRemoteAddress(); + } + String origin = ctx.header("Origin").valueOrNull(); + + LOGGER.info("[BULK DELETE]\n" + + " user agent = " + ctx.header("User-Agent").value("null") + "\n" + + " ip = " + ipAddress + "\n" + + (origin == null ? "" : " origin = " + origin + "\n") + ); + LOGGER.info("[BULK DELETE] keys = " + list); + + return CompletableFuture.supplyAsync(() -> { + int deleted = this.storageHandler.bulkDelete(list); + LOGGER.info("[BULK DELETE] Successfully deleted " + deleted + " entries"); + return deleted; + }); + } + +} diff --git a/src/main/java/me/lucko/bytebin/util/Configuration.java b/src/main/java/me/lucko/bytebin/util/Configuration.java index c22b5ed..f37acb1 100644 --- a/src/main/java/me/lucko/bytebin/util/Configuration.java +++ b/src/main/java/me/lucko/bytebin/util/Configuration.java @@ -167,7 +167,8 @@ public enum Option { CACHE_EXPIRY("cacheExpiryMinutes", "bytebin.cache.expiry"), // minutes CACHE_MAX_SIZE("cacheMaxSizeMb", "bytebin.cache.maxsize"), // mb - API_KEYS("apiKeys", "bytebin.ratelimit.apikeys"), // list + RATELIMIT_API_KEYS("apiKeys", "bytebin.ratelimit.apikeys"), // list + ADMIN_API_KEYS("adminApiKeys", "bytebin.admin.apikeys"), // list POST_RATE_LIMIT_PERIOD("postRateLimitPeriodMins", "bytebin.ratelimit.post.period"), // minutes POST_RATE_LIMIT("postRateLimit", "bytebin.ratelimit.post.amount"), diff --git a/src/main/java/me/lucko/bytebin/util/ContentEncoding.java b/src/main/java/me/lucko/bytebin/util/ContentEncoding.java index 5b29cb9..bf6ecae 100644 --- a/src/main/java/me/lucko/bytebin/util/ContentEncoding.java +++ b/src/main/java/me/lucko/bytebin/util/ContentEncoding.java @@ -26,7 +26,6 @@ package me.lucko.bytebin.util; import com.google.common.base.Splitter; - import io.jooby.Context; import java.util.ArrayList; diff --git a/src/main/java/me/lucko/bytebin/util/RateLimitHandler.java b/src/main/java/me/lucko/bytebin/util/RateLimitHandler.java index b9ca8ec..bb8836c 100644 --- a/src/main/java/me/lucko/bytebin/util/RateLimitHandler.java +++ b/src/main/java/me/lucko/bytebin/util/RateLimitHandler.java @@ -26,7 +26,6 @@ package me.lucko.bytebin.util; import com.google.common.collect.ImmutableSet; - import io.jooby.Context; import io.jooby.StatusCode; import io.jooby.exception.StatusCodeException;