diff --git a/.env.example b/.env.example index 7c8efb9f9a..aeff4fb9b6 100644 --- a/.env.example +++ b/.env.example @@ -13,3 +13,5 @@ APPWRITE_DB_INIT_ID= APPWRITE_COL_INIT_ID= APPWRITE_API_KEY_INIT= SENTRY_AUTH_TOKEN= +PROFOUND_API_URL= +PROFOUND_API_KEY= diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e8a9cb5b54..4313a7c2e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,6 +33,8 @@ jobs: APPWRITE_COL_INIT_ID: ${{ secrets.APPWRITE_COL_INIT_ID }} APPWRITE_API_KEY_INIT: ${{ secrets.APPWRITE_API_KEY_INIT }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PROFOUND_API_URL: ${{ secrets.PROFOUND_API_URL }} + PROFOUND_API_KEY: ${{ secrets.PROFOUND_API_KEY }} run: bun run check format: runs-on: ubuntu-latest @@ -77,6 +79,8 @@ jobs: APPWRITE_COL_INIT_ID: ${{ secrets.APPWRITE_COL_INIT_ID }} APPWRITE_API_KEY_INIT: ${{ secrets.APPWRITE_API_KEY_INIT }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PROFOUND_API_URL: ${{ secrets.PROFOUND_API_URL }} + PROFOUND_API_KEY: ${{ secrets.PROFOUND_API_KEY }} run: bun run test build: runs-on: ubuntu-latest @@ -107,6 +111,8 @@ jobs: APPWRITE_COL_INIT_ID: ${{ secrets.APPWRITE_COL_INIT_ID }} APPWRITE_API_KEY_INIT: ${{ secrets.APPWRITE_API_KEY_INIT }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PROFOUND_API_URL: ${{ secrets.PROFOUND_API_URL }} + PROFOUND_API_KEY: ${{ secrets.PROFOUND_API_KEY }} run: bun run build assets: runs-on: ubuntu-latest diff --git a/.prettierignore b/.prettierignore index 9f7aef0a16..aa3cc62c5a 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,6 +7,7 @@ node_modules .env.* !.env.example deploy +src/lib/generated # Ignore files for Bun bun.lock diff --git a/log-collector/.gitignore b/log-collector/.gitignore deleted file mode 100644 index 073e02da0a..0000000000 --- a/log-collector/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -node_modules -.wrangler -.dev.vars diff --git a/log-collector/package.json b/log-collector/package.json deleted file mode 100644 index 674b2fed34..0000000000 --- a/log-collector/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "log-collector", - "version": "1.0.0", - "private": true, - "scripts": { - "dev": "wrangler dev", - "deploy": "wrangler deploy", - "cf-typegen": "wrangler types" - }, - "devDependencies": { - "@cloudflare/workers-types": "^4.20250124.0", - "typescript": "^5.5.2", - "wrangler": "^3.101.0" - } -} diff --git a/log-collector/src/index.ts b/log-collector/src/index.ts deleted file mode 100644 index 0a0de473e1..0000000000 --- a/log-collector/src/index.ts +++ /dev/null @@ -1,49 +0,0 @@ -export interface Env { - PROFOUND_API_URL: string; - PROFOUND_API_KEY: string; -} - -export default { - async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { - const response = await fetch(request); - const responseClone = response.clone(); - - ctx.waitUntil(handleRequest(request, responseClone, env)); - return response; - } -} satisfies ExportedHandler; - -async function handleRequest(request: Request, response: Response, env: Env) { - const requestUrl = new URL(request.url); - - const headerSize = Array.from(response.headers.entries()).reduce( - (total, [key, value]) => total + key.length + value.length + 4, - 0 - ); - - const contentLength = response.headers.get('content-length'); - const bodySize = contentLength ? Number.parseInt(contentLength, 10) || 0 : 0; - const totalBytesSent = headerSize + bodySize; - - const logData = { - timestamp: Date.now(), - host: requestUrl.hostname, - method: request.method, - pathname: requestUrl.pathname, - query_params: Object.fromEntries(requestUrl.searchParams), - ip: request.headers.get('cf-connecting-ip'), - userAgent: request.headers.get('user-agent'), - referer: request.headers.get('referer'), - bytes: totalBytesSent, - status: response.status - }; - - await fetch(env.PROFOUND_API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-API-Key': env.PROFOUND_API_KEY - }, - body: JSON.stringify([logData]) - }).catch((error) => console.error('Failed to send logs:', error)); -} diff --git a/log-collector/tsconfig.json b/log-collector/tsconfig.json deleted file mode 100644 index f2af70dda3..0000000000 --- a/log-collector/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "Bundler", - "lib": ["ESNext"], - "types": ["@cloudflare/workers-types/2023-07-01"], - "noEmit": true, - "skipLibCheck": true, - "strict": true, - "forceConsistentCasingInFileNames": true - }, - "include": ["src/**/*.ts"] -} diff --git a/log-collector/wrangler.json b/log-collector/wrangler.json deleted file mode 100644 index 5ba48ac535..0000000000 --- a/log-collector/wrangler.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "node_modules/wrangler/config-schema.json", - "name": "log-collector", - "main": "src/index.ts", - "compatibility_date": "2025-01-29", - "observability": { - "enabled": true - }, - "route": { - "pattern": "appwrite.io/*", - "zone_name": "appwrite.io" - }, - "vars": { - "PROFOUND_API_URL": "https://artemis.api.tryprofound.com/v1/logs/cloudflare_worker" - } -} diff --git a/package.json b/package.json index 5589fce939..b870516987 100644 --- a/package.json +++ b/package.json @@ -5,11 +5,11 @@ "type": "module", "scripts": { "build": "bun ./scripts/build.js", - "fetch:stars": "node scripts/fetch-github-stars.js", + "fetch:stars": "bun scripts/fetch-github-stars.js", "prebuild": "bun run fetch:stars", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", - "clean": "rm -rf node_modules && rm -rf .svelte_kit && bun install", + "clean": "rm -rf node_modules && rm -rf .svelte_kit", "dev": "vite dev", "format": "prettier --write --cache .", "format:check": "prettier --check .", diff --git a/src/hooks.server.ts b/src/hooks.server.ts index 20817cfa39..3ba7857639 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -6,6 +6,7 @@ import { getMarkdownContent, processMarkdownWithPartials } from '$lib/server/mar import { type GithubUser } from '$routes/(init)/init/(utils)/auth'; import { createInitSessionClient } from '$routes/(init)/init/(utils)/appwrite'; import type { AppwriteUser } from '$lib/utils/console'; +import { profoundAnalytics } from '$lib/server/profound-analytics'; const redirectMap = new Map(redirects.map(({ link, redirect }) => [link, redirect])); @@ -262,6 +263,7 @@ export const handle = sequence( wwwRedirecter, securityheaders, initSession, - seoOptimization + seoOptimization, + profoundAnalytics ); export const handleError = Sentry.handleErrorWithSentry(); diff --git a/src/lib/generated/github-stars.json b/src/lib/generated/github-stars.json index 66101ba672..9aeabee5d9 100644 --- a/src/lib/generated/github-stars.json +++ b/src/lib/generated/github-stars.json @@ -1,4 +1,4 @@ { - "stars": 55124, - "fetchedAt": "2026-03-10T14:12:30.207Z" -} + "stars": 55136, + "fetchedAt": "2026-03-18T12:04:09.106Z" +} \ No newline at end of file diff --git a/src/lib/server/profound-analytics.ts b/src/lib/server/profound-analytics.ts new file mode 100644 index 0000000000..b6620b20e4 --- /dev/null +++ b/src/lib/server/profound-analytics.ts @@ -0,0 +1,128 @@ +import type { Handle } from '@sveltejs/kit'; +import { env } from '$env/dynamic/private'; +import { building } from '$app/environment'; + +const { PROFOUND_API_URL, PROFOUND_API_KEY } = env; + +interface LogEntry { + timestamp: number; + host: string; + method: string; + pathname: string; + query_params: Record; + ip: string | null; + userAgent: string | null; + referer: string | null; + bytes: number; + status: number; +} + +class LogBatcher { + private queue: LogEntry[] = []; + private readonly MAX_SIZE = 10000; + private readonly FLUSH_INTERVAL_MS = 10000; + private flushTimer: ReturnType | null = null; + private isRunning = false; + + constructor() { + this.start(); + } + + start(): void { + if (this.isRunning) return; + + this.isRunning = true; + this.flushTimer = setInterval(() => { + void this.flush(); + }, this.FLUSH_INTERVAL_MS); + } + + stop(): void { + if (this.flushTimer) { + clearInterval(this.flushTimer); + this.flushTimer = null; + } + this.isRunning = false; + } + + add(log: LogEntry): void { + // Evict oldest entries if at capacity (FIFO) + while (this.queue.length >= this.MAX_SIZE) { + this.queue.shift(); + } + this.queue.push(log); + } + + async flush(): Promise { + if (this.queue.length === 0) return; + if (!PROFOUND_API_URL || !PROFOUND_API_KEY) return; + + // Take all logs from queue + const batch = this.queue.splice(0); + + try { + const response = await fetch(PROFOUND_API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-API-Key': PROFOUND_API_KEY + }, + body: JSON.stringify(batch) + }); + + if (!response.ok) { + console.error(`Analytics flush failed: ${response.status} ${response.statusText}`); + } + } catch (error) { + console.error('Analytics flush failed:', error); + } + } +} + +// Singleton instance +const logBatcher = new LogBatcher(); + +export const profoundAnalytics: Handle = async ({ event, resolve }) => { + // Short circuit if analytics is not configured or during pre-render + if (building || !PROFOUND_API_URL || !PROFOUND_API_KEY) { + return resolve(event); + } + + const response = await resolve(event); + + // Only track HTML pages, not API responses or static assets + const contentType = response.headers.get('content-type'); + if (!contentType?.includes('text/html')) { + return response; + } + + // Calculate header size + const headerSize = Array.from(response.headers.entries()).reduce( + (total, [key, value]) => total + key.length + value.length + 4, + 0 + ); + + // Get body size + const contentLength = response.headers.get('content-length'); + const bodySize = contentLength ? Number.parseInt(contentLength, 10) || 0 : 0; + const totalBytesSent = headerSize + bodySize; + + // Build log data + const logData: LogEntry = { + timestamp: Date.now(), + host: event.url.hostname, + method: event.request.method, + pathname: event.url.pathname, + query_params: Object.fromEntries(event.url.searchParams), + ip: event.getClientAddress(), + userAgent: event.request.headers.get('user-agent'), + referer: event.request.headers.get('referer'), + bytes: totalBytesSent, + status: response.status + }; + + // Non-blocking: add to queue and return immediately + logBatcher.add(logData); + + return response; +}; diff --git a/src/routes/(init)/init/+page.server.ts b/src/routes/(init)/init/+page.server.ts index 8f91c3fd3b..a22f825dc6 100644 --- a/src/routes/(init)/init/+page.server.ts +++ b/src/routes/(init)/init/+page.server.ts @@ -2,8 +2,6 @@ import { redirect, type Actions } from '@sveltejs/kit'; import { getTicketByUser } from './(utils)/tickets'; import { loginGithub } from './(utils)/auth'; -export const prerender = false; - export const load = async ({ locals }) => { const ticket = await getTicketByUser(locals.initUser); diff --git a/src/routes/(marketing)/+page.ts b/src/routes/(marketing)/+page.ts deleted file mode 100644 index 189f71e2e1..0000000000 --- a/src/routes/(marketing)/+page.ts +++ /dev/null @@ -1 +0,0 @@ -export const prerender = true; diff --git a/src/routes/+layout.ts b/src/routes/+layout.ts index acbfaad11f..7f13d07240 100644 --- a/src/routes/+layout.ts +++ b/src/routes/+layout.ts @@ -1,5 +1,7 @@ import { changelogCount } from './changelog/utils'; +export const prerender = false; + export const trailingSlash = 'never'; export const load = () => ({ diff --git a/src/routes/blog/[[page]]/+page.ts b/src/routes/blog/[[page]]/+page.ts index 3fb5ac71b0..6ecb430467 100644 --- a/src/routes/blog/[[page]]/+page.ts +++ b/src/routes/blog/[[page]]/+page.ts @@ -2,8 +2,6 @@ import { getBlogEntries, normalizeCategory } from '../content'; import { BLOG_POSTS_PER_PAGE } from '$lib/constants'; import { error, redirect } from '@sveltejs/kit'; -export const prerender = false; - export const entries = () => { const { posts } = getBlogEntries(); const totalPages = Math.ceil(posts.length / BLOG_POSTS_PER_PAGE); diff --git a/src/routes/integrations/+page.ts b/src/routes/integrations/+page.ts index c9f17fa8b4..53cd26ad65 100644 --- a/src/routes/integrations/+page.ts +++ b/src/routes/integrations/+page.ts @@ -87,5 +87,3 @@ export const load = () => { featured: featuredIntegrationsWithCategoryHeadings }; }; - -export const prerender = false; diff --git a/src/routes/llms-full.txt/+server.ts b/src/routes/llms-full.txt/+server.ts index 9e7c33134a..cf00e5753a 100644 --- a/src/routes/llms-full.txt/+server.ts +++ b/src/routes/llms-full.txt/+server.ts @@ -85,7 +85,7 @@ function sectionWeight(href: string): number { } } -export const GET: RequestHandler = ({ request }) => { +export const GET: RequestHandler = () => { try { const base = 'https://appwrite.io'; type Item = { href: string; title: string; block: string }; diff --git a/src/routes/partners/catalog/+page.ts b/src/routes/partners/catalog/+page.ts index 5f4751b8fd..5359af6d65 100644 --- a/src/routes/partners/catalog/+page.ts +++ b/src/routes/partners/catalog/+page.ts @@ -92,5 +92,3 @@ export const load = () => { featured: featuredIntegrationsWithCategoryHeadings }; }; - -export const prerender = false; diff --git a/src/routes/threads/+layout.ts b/src/routes/threads/+layout.ts deleted file mode 100644 index d43d0cd2a5..0000000000 --- a/src/routes/threads/+layout.ts +++ /dev/null @@ -1 +0,0 @@ -export const prerender = false;