From 73a5c0d56a9be3babc6e428be8f57c5fa8722085 Mon Sep 17 00:00:00 2001
From: Luke <9198896+flexchar@users.noreply.github.com>
Date: Tue, 6 Feb 2024 11:04:13 +0100
Subject: [PATCH] Several QoL contributions (#760)

* allow customizing disclaimer as `PUBLIC_APP_DISCLAIMER_MESSAGE`

* support passing `defaultHeaders` to `openai` endpoint

* add azure openai, claude, mistral examples using `defaultHeaders` & `openai` endpoint

* fix streaming being buffered behind cloudflare tunnel

might help to relieve issue #598

* support new lines in model description

* don't automatically generate modelUrl to huggingface

fixes broken links for self-hosted or custom-named model

* add `PUBLIC_APP_DISCLAIMER_MESSAGE` in `.env`

* `npm run format`

---------

Co-authored-by: Nathan Sarrazin <sarrazin.nathan@gmail.com>
---
 .env                                          |  1 +
 .env.template                                 |  1 +
 README.md                                     | 69 +++++++++++++++++++
 src/lib/components/DisclaimerModal.svelte     |  9 ++-
 .../server/endpoints/openai/endpointOai.ts    |  5 +-
 src/routes/conversation/[id]/+server.ts       |  6 +-
 src/routes/settings/[...model]/+page.svelte   | 22 +++---
 7 files changed, 98 insertions(+), 15 deletions(-)

diff --git a/.env b/.env
index 90cb6169e05..a6761d7852e 100644
--- a/.env
+++ b/.env
@@ -120,6 +120,7 @@ PUBLIC_APP_COLOR=blue # can be any of tailwind colors: https://tailwindcss.com/d
 PUBLIC_APP_DESCRIPTION=# description used throughout the app (if not set, a default one will be used)
 PUBLIC_APP_DATA_SHARING=#set to 1 to enable options & text regarding data sharing
 PUBLIC_APP_DISCLAIMER=#set to 1 to show a disclaimer on login page
+PUBLIC_APP_DISCLAIMER_MESSAGE="Disclaimer: AI is an area of active research with known problems such as biased generation and misinformation. Do not use this application for high-stakes decisions or advice."
 LLM_SUMMERIZATION=true
 
 EXPOSE_API=true
diff --git a/.env.template b/.env.template
index 7c5ce5b0c6d..deb22d5ba87 100644
--- a/.env.template
+++ b/.env.template
@@ -228,6 +228,7 @@ PUBLIC_APP_NAME=HuggingChat
 PUBLIC_APP_ASSETS=huggingchat
 PUBLIC_APP_COLOR=yellow
 PUBLIC_APP_DESCRIPTION="Making the community's best AI chat models available to everyone."
+PUBLIC_APP_DISCLAIMER_MESSAGE="Disclaimer: AI is an area of active research with known problems such as biased generation and misinformation. Do not use this application for high-stakes decisions or advice."
 PUBLIC_APP_DATA_SHARING=1
 PUBLIC_APP_DISCLAIMER=1
 
diff --git a/README.md b/README.md
index 3695d9bfba8..64af99312ae 100644
--- a/README.md
+++ b/README.md
@@ -316,6 +316,75 @@ MODELS=`[{
 }]`
 ```
 
+You may also consume any model provider that provides compatible OpenAI API endpoint. For example, you may self-host [Portkey](https://github.com/Portkey-AI/gateway) gateway and experiment with Claude or GPTs offered by Azure OpenAI. Example for Claude from Anthropic:
+
+```
+MODELS=`[{
+  "name": "claude-2.1",
+  "displayName": "Claude 2.1",
+  "description": "Anthropic has been founded by former OpenAI researchers...",
+  "parameters": {
+      "temperature": 0.5,
+      "max_new_tokens": 4096,
+  },
+  "endpoints": [
+      {
+          "type": "openai",
+          "baseURL": "https://gateway.example.com/v1",
+          "defaultHeaders": {
+              "x-portkey-config": '{"provider":"anthropic","api_key":"sk-ant-abc...xyz"}'
+          }
+      }
+  ]
+}]`
+```
+
+Example for GPT 4 deployed on Azure OpenAI:
+
+```
+MODELS=`[{
+  "id": "gpt-4-1106-preview",
+  "name": "gpt-4-1106-preview",
+  "displayName": "gpt-4-1106-preview",
+  "parameters": {
+      "temperature": 0.5,
+      "max_new_tokens": 4096,
+  },
+  "endpoints": [
+      {
+          "type": "openai",
+          "baseURL": "https://gateway.example.com/v1",
+          "defaultHeaders": {
+              "x-portkey-config": '{"provider":"azure-openai","resource_name":"abc-fr","deployment_id":"gpt-4-1106-preview","api_version":"2023-03-15-preview","api_key":"abc...xyz"}'
+          }
+      }
+  ]
+}]`
+```
+
+Or try Mistral from [Deepinfra](https://deepinfra.com/mistralai/Mistral-7B-Instruct-v0.1/api?example=openai-http):
+
+> Note, apiKey can either be set custom per endpoint, or globally using `OPENAI_API_KEY` variable.
+
+```
+MODELS=`[{
+  "name": "mistral-7b",
+  "displayName": "Mistral 7B",
+  "description": "A 7B dense Transformer, fast-deployed and easily customisable. Small, yet powerful for a variety of use cases. Supports English and code, and a 8k context window.",
+  "parameters": {
+      "temperature": 0.5,
+      "max_new_tokens": 4096,
+  },
+  "endpoints": [
+      {
+          "type": "openai",
+          "baseURL": "https://api.deepinfra.com/v1/openai",
+          "apiKey": "abc...xyz"
+      }
+  ]
+}]`
+```
+
 ##### Llama.cpp API server
 
 chat-ui also supports the llama.cpp API server directly without the need for an adapter. You can do this using the `llamacpp` endpoint type.
diff --git a/src/lib/components/DisclaimerModal.svelte b/src/lib/components/DisclaimerModal.svelte
index ecb5f22a895..590bb088b3c 100644
--- a/src/lib/components/DisclaimerModal.svelte
+++ b/src/lib/components/DisclaimerModal.svelte
@@ -1,7 +1,11 @@
 <script lang="ts">
 	import { base } from "$app/paths";
 	import { page } from "$app/stores";
-	import { PUBLIC_APP_DESCRIPTION, PUBLIC_APP_NAME } from "$env/static/public";
+	import {
+		PUBLIC_APP_DESCRIPTION,
+		PUBLIC_APP_NAME,
+		PUBLIC_APP_DISCLAIMER_MESSAGE,
+	} from "$env/static/public";
 	import LogoHuggingFaceBorderless from "$lib/components/icons/LogoHuggingFaceBorderless.svelte";
 	import Modal from "$lib/components/Modal.svelte";
 	import { useSettingsStore } from "$lib/stores/settings";
@@ -25,8 +29,7 @@
 		</p>
 
 		<p class="text-sm text-gray-500">
-			Disclaimer: AI is an area of active research with known problems such as biased generation and
-			misinformation. Do not use this application for high-stakes decisions or advice.
+			{PUBLIC_APP_DISCLAIMER_MESSAGE}
 		</p>
 
 		<div class="flex w-full flex-col items-center gap-2">
diff --git a/src/lib/server/endpoints/openai/endpointOai.ts b/src/lib/server/endpoints/openai/endpointOai.ts
index c878b30e80c..3ed70940d7d 100644
--- a/src/lib/server/endpoints/openai/endpointOai.ts
+++ b/src/lib/server/endpoints/openai/endpointOai.ts
@@ -15,12 +15,14 @@ export const endpointOAIParametersSchema = z.object({
 	completion: z
 		.union([z.literal("completions"), z.literal("chat_completions")])
 		.default("chat_completions"),
+	defaultHeaders: z.record(z.string()).optional(),
 });
 
 export async function endpointOai(
 	input: z.input<typeof endpointOAIParametersSchema>
 ): Promise<Endpoint> {
-	const { baseURL, apiKey, completion, model } = endpointOAIParametersSchema.parse(input);
+	const { baseURL, apiKey, completion, model, defaultHeaders } =
+		endpointOAIParametersSchema.parse(input);
 	let OpenAI;
 	try {
 		OpenAI = (await import("openai")).OpenAI;
@@ -31,6 +33,7 @@ export async function endpointOai(
 	const openai = new OpenAI({
 		apiKey: apiKey ?? "sk-",
 		baseURL,
+		defaultHeaders,
 	});
 
 	if (completion === "completions") {
diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts
index d576216a771..d9ce6229c81 100644
--- a/src/routes/conversation/[id]/+server.ts
+++ b/src/routes/conversation/[id]/+server.ts
@@ -381,7 +381,11 @@ export async function POST({ request, locals, params, getClientAddress }) {
 	});
 
 	// Todo: maybe we should wait for the message to be saved before ending the response - in case of errors
-	return new Response(stream);
+	return new Response(stream, {
+		headers: {
+			"Content-Type": "text/event-stream",
+		},
+	});
 }
 
 export async function DELETE({ locals, params }) {
diff --git a/src/routes/settings/[...model]/+page.svelte b/src/routes/settings/[...model]/+page.svelte
index 1bb368ccfa9..3035ecc524c 100644
--- a/src/routes/settings/[...model]/+page.svelte
+++ b/src/routes/settings/[...model]/+page.svelte
@@ -34,22 +34,24 @@
 		</h2>
 
 		{#if model.description}
-			<p class=" text-gray-600">
+			<p class="whitespace-pre-wrap text-gray-600">
 				{model.description}
 			</p>
 		{/if}
 	</div>
 
 	<div class="flex flex-wrap items-center gap-2 md:gap-4">
-		<a
-			href={model.modelUrl || "https://huggingface.co/" + model.name}
-			target="_blank"
-			rel="noreferrer"
-			class="flex items-center truncate underline underline-offset-2"
-		>
-			<CarbonArrowUpRight class="mr-1.5 shrink-0 text-xs " />
-			Model page
-		</a>
+		{#if model.modelUrl}
+			<a
+				href={model.modelUrl || "https://huggingface.co/" + model.name}
+				target="_blank"
+				rel="noreferrer"
+				class="flex items-center truncate underline underline-offset-2"
+			>
+				<CarbonArrowUpRight class="mr-1.5 shrink-0 text-xs " />
+				Model page
+			</a>
+		{/if}
 
 		{#if model.datasetName || model.datasetUrl}
 			<a