From 5fb8bb958da7f1d89a6ff042553d7b6cbf96fd51 Mon Sep 17 00:00:00 2001
From: Andreas Thomas <dev@chronark.com>
Date: Wed, 18 Sep 2024 10:10:11 +0200
Subject: [PATCH] fix: always return the raw llm response (#2101)

---
 apps/semantic-cache/src/pkg/streaming.ts  | 14 +++++++++-----
 apps/semantic-cache/src/pkg/util/index.ts | 12 ------------
 2 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/apps/semantic-cache/src/pkg/streaming.ts b/apps/semantic-cache/src/pkg/streaming.ts
index 9d673c815e..97baa1c48c 100644
--- a/apps/semantic-cache/src/pkg/streaming.ts
+++ b/apps/semantic-cache/src/pkg/streaming.ts
@@ -2,7 +2,7 @@ import { streamSSE } from "hono/streaming";
 import type { OpenAI } from "openai";
 
 import type { Context } from "./hono/app";
-import { OpenAIResponse, createCompletionChunk, parseMessagesToString } from "./util";
+import { createCompletionChunk, parseMessagesToString } from "./util";
 
 import type { CacheError } from "@unkey/cache";
 import { BaseError, Err, Ok, type Result, wrap } from "@unkey/error";
@@ -125,7 +125,7 @@ export async function handleNonStreamingRequest(
 
   // Cache hit
   if (cached.val) {
-    return c.json(OpenAIResponse(cached.val));
+    return c.json(JSON.parse(cached.val));
   }
 
   // miss
@@ -142,15 +142,19 @@ export async function handleNonStreamingRequest(
   const tokens = chatCompletion.val.usage?.completion_tokens ?? 0;
   c.set("tokens", Promise.resolve(tokens));
 
-  const response = chatCompletion.val.choices.at(0)?.message.content || "";
-  const { err: updateCacheError } = await updateCache(c, embeddings.val, response, tokens);
+  const { err: updateCacheError } = await updateCache(
+    c,
+    embeddings.val,
+    JSON.stringify(chatCompletion),
+    tokens,
+  );
   if (updateCacheError) {
     logger.error("unable to update cache", {
       error: updateCacheError.message,
     });
   }
 
-  c.set("response", Promise.resolve(response));
+  c.set("response", Promise.resolve(JSON.stringify(chatCompletion, null, 2)));
   return c.json(chatCompletion);
 }
 
diff --git a/apps/semantic-cache/src/pkg/util/index.ts b/apps/semantic-cache/src/pkg/util/index.ts
index fca8a437bd..6b7d86c9e3 100644
--- a/apps/semantic-cache/src/pkg/util/index.ts
+++ b/apps/semantic-cache/src/pkg/util/index.ts
@@ -21,18 +21,6 @@ export async function createCompletionChunk(content: string, stop = false) {
   };
 }
 
-export function OpenAIResponse(content: string) {
-  return {
-    choices: [
-      {
-        message: {
-          content,
-        },
-      },
-    ],
-  };
-}
-
 /**
  * Extracts the word enclosed in double quotes from the given chunk.
  *