From 6c6f991be1702c09ff7f23853cbee20e7466467b Mon Sep 17 00:00:00 2001 From: Zac Pullar-Strecker Date: Thu, 11 Apr 2024 12:23:23 +1200 Subject: [PATCH 1/3] Add support for backend telemetry (tracing & metrics) This uses the opentelemetry SDK to provide support for traces and two simple metrics (a conversations count and a messages count). The telemetry entrypoint is built separately (unfortunately it does not seem like there is a method for doing this in one build as sveltekit overrides rollupOptions). As written the metrics may be too high cardinality for some use cases (as they are keyed per user), but these attributes can always be dropped in a collector. --- Dockerfile | 4 +- package.json | 8 ++- scripts/clean-opentelemetry.sh | 7 +++ src/routes/conversation/+server.ts | 10 ++++ src/routes/conversation/[id]/+server.ts | 10 ++++ src/telemetry.ts | 77 +++++++++++++++++++++++++ vite.telemetry.config.ts | 19 ++++++ 7 files changed, 132 insertions(+), 3 deletions(-) create mode 100755 scripts/clean-opentelemetry.sh create mode 100644 src/telemetry.ts create mode 100644 vite.telemetry.config.ts diff --git a/Dockerfile b/Dockerfile index a88846db80e..4d83a2a2508 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN --mount=type=cache,target=/app/.npm \ COPY --link --chown=1000 . . RUN --mount=type=secret,id=DOTENV_LOCAL,dst=.env.local \ - npm run build + npm run build && npm run build -- --config vite.telemetry.config.ts FROM node:20-slim @@ -29,4 +29,4 @@ COPY --from=builder-production /app/node_modules /app/node_modules COPY --link --chown=1000 package.json /app/package.json COPY --from=builder /app/build /app/build -CMD pm2 start /app/build/index.js -i $CPU_CORES --no-daemon +CMD pm2 start /app/build/index.js --node-args="--require /app/build/telemetry.cjs" -i $CPU_CORES --no-daemon diff --git a/package.json b/package.json index f4af3d7e41e..1b99fea4dbc 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "packageManager": "npm@9.5.0", "scripts": { "dev": "vite dev", - "build": "vite build", + "build": "bash scripts/clean-opentelemetry.sh && vite build", "preview": "vite preview", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", @@ -53,6 +53,12 @@ "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", + "@opentelemetry/api": "^1.8.0", + "@opentelemetry/auto-instrumentations-node": "^0.44.0", + "@opentelemetry/exporter-metrics-otlp-proto": "^0.50.0", + "@opentelemetry/sdk-metrics": "^1.23.0", + "@opentelemetry/sdk-node": "^0.50.0", + "@opentelemetry/sdk-trace-node": "^1.23.0", "@resvg/resvg-js": "^2.6.0", "@xenova/transformers": "^2.16.1", "autoprefixer": "^10.4.14", diff --git a/scripts/clean-opentelemetry.sh b/scripts/clean-opentelemetry.sh new file mode 100755 index 00000000000..6284a5784fc --- /dev/null +++ b/scripts/clean-opentelemetry.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +packages="./node_modules/@opentelemetry/*/package.json" + +for file in ${packages}; do + sed -i '/"module": "build\/esm\/index\.js",/d' ${file} +done diff --git a/src/routes/conversation/+server.ts b/src/routes/conversation/+server.ts index e26e8cd9c73..7feb405c8fc 100644 --- a/src/routes/conversation/+server.ts +++ b/src/routes/conversation/+server.ts @@ -10,6 +10,7 @@ import { defaultEmbeddingModel } from "$lib/server/embeddingModels"; import { v4 } from "uuid"; import { authCondition } from "$lib/server/auth"; import { usageLimits } from "$lib/server/usageLimits"; +import { metrics } from "@opentelemetry/api"; export const POST: RequestHandler = async ({ locals, request }) => { const body = await request.text(); @@ -111,6 +112,15 @@ export const POST: RequestHandler = async ({ locals, request }) => { ...(values.fromShare ? { meta: { fromShareId: values.fromShare } } : {}), }); + const meter = metrics.getMeter("chat-ui"); + const counter = meter.createCounter("chat-ui.conversations.count", { + description: "The number of conversations created", + }); + counter.add(1, { + "chat-ui.model": values.model, + "user.email": locals.user?.email || undefined, + }); + return new Response( JSON.stringify({ conversationId: res.insertedId.toString(), diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 0d6aec324a8..01cb31834f9 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -23,6 +23,7 @@ import { addSibling } from "$lib/utils/tree/addSibling.js"; import { preprocessMessages } from "$lib/server/preprocessMessages.js"; import { usageLimits } from "$lib/server/usageLimits"; import { isURLLocal } from "$lib/server/isURLLocal.js"; +import { metrics } from "@opentelemetry/api"; export async function POST({ request, locals, params, getClientAddress }) { const id = z.string().parse(params.id); @@ -243,6 +244,15 @@ export async function POST({ request, locals, params, getClientAddress }) { messageId ); + const meter = metrics.getMeter("chat-ui"); + const counter = meter.createCounter("chat-ui.conversations.messages.count", { + description: "The number of user messages created", + }); + counter.add(1, { + "chat-ui.model": "values.model", + "user.email": locals.user?.email || undefined, + }); + messageToWriteToId = addChildren( conv, { diff --git a/src/telemetry.ts b/src/telemetry.ts new file mode 100644 index 00000000000..c7e9ee3f4c3 --- /dev/null +++ b/src/telemetry.ts @@ -0,0 +1,77 @@ +// This file is built outside of sveltekit and cannot import from the rest of the application +// or special imports like $env/dynamic/private. +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node"; +import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto"; +import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto"; +import { SEMRESATTRS_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { AlwaysOnSampler } from "@opentelemetry/sdk-trace-base"; +import { Resource } from "@opentelemetry/resources"; + +const TRACE_URL = + process.env.OTEL_EXPORTER_OTLP_ENDPOINT + "/v1/traces" || "http://localhost:4318/v1/traces"; +const METRICS_URL = + process.env.OTEL_EXPORTER_OTLP_ENDPOINT + "/v1/metrics" || "http://localhost:4318/v1/metrics"; +const SERVICE_NAME = process.env.OTEL_SERVICE_NAME || "huggingface/chat-ui"; + +const exporter = new OTLPTraceExporter({ + url: TRACE_URL, + headers: {}, +}); + +const otelNodeSdk = new NodeSDK({ + autoDetectResources: true, + serviceName: SERVICE_NAME, + traceExporter: exporter, + metricReader: new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ + url: METRICS_URL, + headers: {}, + }), + }), + sampler: new AlwaysOnSampler(), + resource: new Resource({ + [SEMRESATTRS_SERVICE_NAME]: SERVICE_NAME, + }), + instrumentations: [ + getNodeAutoInstrumentations({ + "@opentelemetry/instrumentation-http": { + ignoreIncomingRequestHook: (request) => { + // Don't trace static asset requests + if ( + request.url?.endsWith(".js") || + request.url?.endsWith(".svg") || + request.url?.endsWith(".css") + ) { + return false; + } + return true; + }, + }, + }), + ], +}); + +export class Telemetry { + private static instance: Telemetry; + private initialized = false; + + private constructor() {} + + public static getInstance(): Telemetry { + if (!Telemetry.instance) { + Telemetry.instance = new Telemetry(); + } + return Telemetry.instance; + } + + public start() { + if (!this.initialized) { + this.initialized = true; + otelNodeSdk.start(); + } + } +} + +Telemetry.getInstance().start(); diff --git a/vite.telemetry.config.ts b/vite.telemetry.config.ts new file mode 100644 index 00000000000..01a8fa6fd4b --- /dev/null +++ b/vite.telemetry.config.ts @@ -0,0 +1,19 @@ +import { defineConfig } from "vite"; + +export default defineConfig({ + build: { + emptyOutDir: false, + ssr: true, + target: "node18", + outDir: "build", + rollupOptions: { + input: { + telemetry: "src/telemetry.ts", + }, + }, + lib: { + formats: ["cjs"], + entry: "src/telemetry.ts", + }, + }, +}); From 00ab11e4623a36c3040b1ab24796307bfa64fd16 Mon Sep 17 00:00:00 2001 From: Zac Pullar-Strecker Date: Wed, 1 May 2024 09:30:58 +1200 Subject: [PATCH 2/3] Ensure only one version of @opentelemetry/sdk-metrics is selected Resolves the issue described in https://github.com/open-telemetry/opentelemetry-js/issues/3944 --- package-lock.json | 95 ++++++----------------------------------------- package.json | 2 +- 2 files changed, 13 insertions(+), 84 deletions(-) diff --git a/package-lock.json b/package-lock.json index c20123e959b..68913835bdf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@opentelemetry/api": "^1.8.0", "@opentelemetry/auto-instrumentations-node": "^0.44.0", "@opentelemetry/exporter-metrics-otlp-proto": "^0.50.0", - "@opentelemetry/sdk-metrics": "^1.23.0", + "@opentelemetry/sdk-metrics": "~1.23.0", "@opentelemetry/sdk-node": "^0.50.0", "@opentelemetry/sdk-trace-node": "^1.23.0", "@resvg/resvg-js": "^2.6.0", @@ -1553,22 +1553,6 @@ "@opentelemetry/api": ">=1.0.0 <1.9.0" } }, - "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/sdk-metrics": { - "version": "1.23.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.23.0.tgz", - "integrity": "sha512-4OkvW6+wST4h6LFG23rXSTf6nmTf201h9dzq7bE0z5R9ESEVLERZz6WXwE7PSgg1gdjlaznm1jLJf8GttypFDg==", - "dependencies": { - "@opentelemetry/core": "1.23.0", - "@opentelemetry/resources": "1.23.0", - "lodash.merge": "^4.6.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" - } - }, "node_modules/@opentelemetry/exporter-metrics-otlp-proto": { "version": "0.50.0", "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-proto/-/exporter-metrics-otlp-proto-0.50.0.tgz", @@ -1604,22 +1588,6 @@ "@opentelemetry/api": ">=1.0.0 <1.9.0" } }, - "node_modules/@opentelemetry/exporter-metrics-otlp-proto/node_modules/@opentelemetry/sdk-metrics": { - "version": "1.23.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.23.0.tgz", - "integrity": "sha512-4OkvW6+wST4h6LFG23rXSTf6nmTf201h9dzq7bE0z5R9ESEVLERZz6WXwE7PSgg1gdjlaznm1jLJf8GttypFDg==", - "dependencies": { - "@opentelemetry/core": "1.23.0", - "@opentelemetry/resources": "1.23.0", - "lodash.merge": "^4.6.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" - } - }, "node_modules/@opentelemetry/exporter-trace-otlp-grpc": { "version": "0.50.0", "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.50.0.tgz", @@ -2493,22 +2461,6 @@ "@opentelemetry/api": ">=1.0.0 <1.9.0" } }, - "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-metrics": { - "version": "1.23.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.23.0.tgz", - "integrity": "sha512-4OkvW6+wST4h6LFG23rXSTf6nmTf201h9dzq7bE0z5R9ESEVLERZz6WXwE7PSgg1gdjlaznm1jLJf8GttypFDg==", - "dependencies": { - "@opentelemetry/core": "1.23.0", - "@opentelemetry/resources": "1.23.0", - "lodash.merge": "^4.6.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" - } - }, "node_modules/@opentelemetry/propagation-utils": { "version": "0.30.9", "resolved": "https://registry.npmjs.org/@opentelemetry/propagation-utils/-/propagation-utils-0.30.9.tgz", @@ -2768,12 +2720,12 @@ } }, "node_modules/@opentelemetry/sdk-metrics": { - "version": "1.24.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.24.0.tgz", - "integrity": "sha512-4tJ+E6N019OZVB/nUW/LoK9xHxfeh88TCoaTqHeLBE9wLYfi6irWW6J9cphMav7J8Qk0D5b7/RM4VEY4dArWOA==", + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.23.0.tgz", + "integrity": "sha512-4OkvW6+wST4h6LFG23rXSTf6nmTf201h9dzq7bE0z5R9ESEVLERZz6WXwE7PSgg1gdjlaznm1jLJf8GttypFDg==", "dependencies": { - "@opentelemetry/core": "1.24.0", - "@opentelemetry/resources": "1.24.0", + "@opentelemetry/core": "1.23.0", + "@opentelemetry/resources": "1.23.0", "lodash.merge": "^4.6.2" }, "engines": { @@ -2783,12 +2735,13 @@ "@opentelemetry/api": ">=1.3.0 <1.9.0" } }, - "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/core": { - "version": "1.24.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.24.0.tgz", - "integrity": "sha512-FP2oN7mVPqcdxJDTTnKExj4mi91EH+DNuArKfHTjPuJWe2K1JfMIVXNfahw1h3onJxQnxS8K0stKkogX05s+Aw==", + "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/resources": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.23.0.tgz", + "integrity": "sha512-iPRLfVfcEQynYGo7e4Di+ti+YQTAY0h5mQEUJcHlU9JOqpb4x965O6PZ+wMcwYVY63G96KtdS86YCM1BF1vQZg==", "dependencies": { - "@opentelemetry/semantic-conventions": "1.24.0" + "@opentelemetry/core": "1.23.0", + "@opentelemetry/semantic-conventions": "1.23.0" }, "engines": { "node": ">=14" @@ -2797,14 +2750,6 @@ "@opentelemetry/api": ">=1.0.0 <1.9.0" } }, - "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.24.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.24.0.tgz", - "integrity": "sha512-yL0jI6Ltuz8R+Opj7jClGrul6pOoYrdfVmzQS4SITXRPH7I5IRZbrwe/6/v8v4WYMa6MYZG480S1+uc/IGfqsA==", - "engines": { - "node": ">=14" - } - }, "node_modules/@opentelemetry/sdk-node": { "version": "0.50.0", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-node/-/sdk-node-0.50.0.tgz", @@ -2885,22 +2830,6 @@ "@opentelemetry/api": ">=1.0.0 <1.9.0" } }, - "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-metrics": { - "version": "1.23.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.23.0.tgz", - "integrity": "sha512-4OkvW6+wST4h6LFG23rXSTf6nmTf201h9dzq7bE0z5R9ESEVLERZz6WXwE7PSgg1gdjlaznm1jLJf8GttypFDg==", - "dependencies": { - "@opentelemetry/core": "1.23.0", - "@opentelemetry/resources": "1.23.0", - "lodash.merge": "^4.6.2" - }, - "engines": { - "node": ">=14" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.9.0" - } - }, "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-trace-node": { "version": "1.23.0", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.23.0.tgz", diff --git a/package.json b/package.json index 1c89415b40b..2740a135fdd 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "@opentelemetry/api": "^1.8.0", "@opentelemetry/auto-instrumentations-node": "^0.44.0", "@opentelemetry/exporter-metrics-otlp-proto": "^0.50.0", - "@opentelemetry/sdk-metrics": "^1.23.0", + "@opentelemetry/sdk-metrics": "~1.23.0", "@opentelemetry/sdk-node": "^0.50.0", "@opentelemetry/sdk-trace-node": "^1.23.0", "@resvg/resvg-js": "^2.6.0", From 336b892305c168aa0f5c22e1e2e7fb40fddd8d85 Mon Sep 17 00:00:00 2001 From: Zac Pullar-Strecker Date: Wed, 1 May 2024 09:32:50 +1200 Subject: [PATCH 3/3] Drop user labels As written these were always undefined, can re-add them later if desired. --- src/routes/conversation/+server.ts | 1 - src/routes/conversation/[id]/+server.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/src/routes/conversation/+server.ts b/src/routes/conversation/+server.ts index e9a08e827c9..74faa47c536 100644 --- a/src/routes/conversation/+server.ts +++ b/src/routes/conversation/+server.ts @@ -122,7 +122,6 @@ export const POST: RequestHandler = async ({ locals, request }) => { }); counter.add(1, { "chat-ui.model": values.model, - "user.email": locals.user?.email || undefined, }); return new Response( diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index feeb455fbf3..0f7209b7998 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -265,7 +265,6 @@ export async function POST({ request, locals, params, getClientAddress }) { }); counter.add(1, { "chat-ui.model": "values.model", - "user.email": locals.user?.email || undefined, }); messageToWriteToId = addChildren(