diff --git a/nixpkgs_merge_bot/webhook/secret.py b/nixpkgs_merge_bot/webhook/secret.py
index 893a879..9139951 100644
--- a/nixpkgs_merge_bot/webhook/secret.py
+++ b/nixpkgs_merge_bot/webhook/secret.py
@@ -11,7 +11,10 @@
 
 class WebhookSecret:
     def __init__(self, secret: Path) -> None:
-        self.secret = secret.read_text().strip()
+        if secret.exists():
+            self.secret = secret.read_text().strip()
+        else:
+            raise FileNotFoundError
 
     def validate_signature(self, body: bytes, headers: Message) -> bool:
         # Get the signature from the payload