diff --git a/src/fetch/classify.test.ts b/src/fetch/classify.test.ts
index 7d4f42f7..0af98645 100644
--- a/src/fetch/classify.test.ts
+++ b/src/fetch/classify.test.ts
@@ -5,6 +5,25 @@ describe('fetch classification', () => {
it('recognizes explicit challenges but not bare forbidden responses', () => {
expect(isChallengeResponse(403, { server: 'cloudflare' }, 'Just a moment...')).toBe(true);
expect(isChallengeResponse(403, {}, 'forbidden')).toBe(false);
+ expect(isChallengeResponse(403, { server: 'cloudflare' }, 'forbidden')).toBe(true);
+ });
+ it('ignores third-party allow-lists in CSP and friends', () => {
+ const csp = "default-src 'self'; script-src https://www.google.com/recaptcha/ https://cdnjs.cloudflare.com/";
+ expect(isChallengeResponse(200, { 'content-security-policy': csp }, 'Hacker News')).toBe(false);
+ expect(isChallengeResponse(403, { 'content-security-policy': csp }, 'forbidden')).toBe(false);
+ });
+ it('does not treat a served 200 as a challenge on headers alone', () => {
+ expect(isChallengeResponse(200, { server: 'cloudflare' }, 'real page')).toBe(false);
+ expect(isChallengeResponse(200, { server: 'cloudflare' }, 'Just a moment...')).toBe(true);
+ });
+ it('treats cf-mitigated: challenge as decisive, even on a 200', () => {
+ expect(isChallengeResponse(200, { 'cf-mitigated': 'challenge' }, 'looks fine')).toBe(true);
+ expect(isChallengeResponse(403, { 'cf-mitigated': 'challenge' }, 'forbidden')).toBe(true);
+ });
+ it('reads decisive evidence from the header name, not just its value', () => {
+ expect(isChallengeResponse(403, { 'cf-chl-out': 'AAAA1111' }, 'forbidden')).toBe(true);
+ expect(isChallengeResponse(403, { 'x-datadome': 'protected' }, 'forbidden')).toBe(true);
+ expect(isChallengeResponse(403, { 'x-datadome-cid': 'abc123' }, 'forbidden')).toBe(true);
});
it('recognizes script-heavy app shells', () => expect(isJavaScriptShell('')).toBe(true));
});
diff --git a/src/fetch/classify.ts b/src/fetch/classify.ts
index 4051c13c..d874c767 100644
--- a/src/fetch/classify.ts
+++ b/src/fetch/classify.ts
@@ -1,8 +1,22 @@
const challengeMarkers = /cloudflare|cf-chl|datadome|perimeterx|px-captcha|akamai|captcha|just a moment|verify you are human/i;
+// Headers a bot-mitigation product sets *because it challenged this request*. The evidence is in
+// the name — values are opaque tokens — so these are decisive on their own, whatever the status.
+// https://developers.cloudflare.com/cloudflare-challenges/challenge-types/challenge-pages/detect-response/
+const decisiveHeaders = /^(?:cf-chl-[\w-]+|x-datadome[\w-]*)$/i;
+// Provider branding: present on every response the provider proxies, challenge or not. It can
+// support an already-blocked status but must never turn a served 200 into a challenge.
+const weakHeaders = /^(?:server|set-cookie)$/i;
+// CSP/report-to/link are allow-lists of third parties (cdnjs.cloudflare.com, google.com/recaptcha)
+// and are evidence of nothing — they are in neither list.
export function isChallengeResponse(status: number, headers: Record, body: string): boolean {
- const evidence = `${Object.entries(headers).map(([key, value]) => `${key}:${value}`).join('\n')}\n${body.slice(0, 20_000)}`;
- return challengeMarkers.test(evidence) && (status === 403 || status === 429 || status === 503 || status === 200);
+ if (status !== 403 && status !== 429 && status !== 503 && status !== 200) return false;
+ if (challengeMarkers.test(body.slice(0, 20_000))) return true;
+ const entries = Object.entries(headers).map(([key, value]) => [key.trim().toLowerCase(), value] as const);
+ if (entries.some(([key, value]) => decisiveHeaders.test(key) || (key === 'cf-mitigated' && /challenge/i.test(value)))) return true;
+ // A 200 with a real body is a served page; branding alone (server: cloudflare) never proves otherwise.
+ if (status === 200) return false;
+ return entries.some(([key, value]) => weakHeaders.test(key) && challengeMarkers.test(value));
}
export function isJavaScriptShell(body: string): boolean {