From 1785f3a343559f6d520857cabceeefa79ffcae5c Mon Sep 17 00:00:00 2001
From: Mishig Davaadorj <dmishig@gmail.com>
Date: Mon, 18 Dec 2023 13:41:50 +0100
Subject: [PATCH 01/11] Implement PDF-chat feature

---
 package-lock.json                             | 371 +++++++++++++++++-
 package.json                                  |   1 +
 src/lib/buildPrompt.ts                        |  32 +-
 .../components/OpenPdfSearchResults.svelte    | 114 ++++++
 src/lib/components/UploadBtn.svelte           |  61 ++-
 src/lib/components/chat/ChatMessage.svelte    |  24 +-
 src/lib/components/chat/ChatMessages.svelte   |  10 +-
 src/lib/components/chat/ChatWindow.svelte     |  11 +-
 src/lib/server/database.ts                    |   6 +-
 src/lib/server/embeddings.ts                  |  51 +++
 src/lib/server/endpoints/aws/endpointAws.ts   |   1 +
 .../endpoints/llamacpp/endpointLlamacpp.ts    |   1 +
 .../server/endpoints/ollama/endpointOllama.ts |   1 +
 .../server/endpoints/openai/endpointOai.ts    |   1 +
 src/lib/server/endpoints/tgi/endpointTgi.ts   |   1 +
 src/lib/server/files/downloadFile.ts          |  35 +-
 src/lib/server/files/uploadFile.ts            |  30 +-
 src/lib/server/pdfSearch.ts                   |  51 +++
 src/lib/stores/pendingMessage.ts              |   3 +-
 src/lib/types/Message.ts                      |   2 +
 src/lib/types/MessageUpdate.ts                |  10 +
 src/lib/types/PdfChat.ts                      |  14 +
 src/routes/+page.svelte                       |  30 +-
 src/routes/conversation/[id]/+page.svelte     |  44 ++-
 src/routes/conversation/[id]/+server.ts       |  14 +-
 .../message/[messageId]/prompt/+server.ts     |   1 +
 .../[id]/output/[sha256]/+server.ts           |   4 +-
 .../conversation/[id]/upload-pdf/+server.ts   |  41 ++
 28 files changed, 913 insertions(+), 52 deletions(-)
 create mode 100644 src/lib/components/OpenPdfSearchResults.svelte
 create mode 100644 src/lib/server/embeddings.ts
 create mode 100644 src/lib/server/pdfSearch.ts
 create mode 100644 src/lib/types/PdfChat.ts
 create mode 100644 src/routes/conversation/[id]/upload-pdf/+server.ts

diff --git a/package-lock.json b/package-lock.json
index 371731efac3..37bcd2b065f 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -26,6 +26,7 @@
 				"nanoid": "^4.0.2",
 				"openid-client": "^5.4.2",
 				"parquetjs": "^0.11.2",
+				"pdfjs-dist": "^4.0.269",
 				"postcss": "^8.4.31",
 				"saslprep": "^1.0.3",
 				"serpapi": "^1.1.1",
@@ -689,6 +690,26 @@
 				"@jridgewell/sourcemap-codec": "^1.4.14"
 			}
 		},
+		"node_modules/@mapbox/node-pre-gyp": {
+			"version": "1.0.11",
+			"resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz",
+			"integrity": "sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==",
+			"optional": true,
+			"dependencies": {
+				"detect-libc": "^2.0.0",
+				"https-proxy-agent": "^5.0.0",
+				"make-dir": "^3.1.0",
+				"node-fetch": "^2.6.7",
+				"nopt": "^5.0.0",
+				"npmlog": "^5.0.1",
+				"rimraf": "^3.0.2",
+				"semver": "^7.3.5",
+				"tar": "^6.1.11"
+			},
+			"bin": {
+				"node-pre-gyp": "bin/node-pre-gyp"
+			}
+		},
 		"node_modules/@mongodb-js/saslprep": {
 			"version": "1.1.0",
 			"resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.0.tgz",
@@ -1502,6 +1523,12 @@
 			"resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
 			"integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA=="
 		},
+		"node_modules/abbrev": {
+			"version": "1.1.1",
+			"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+			"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+			"optional": true
+		},
 		"node_modules/abort-controller": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
@@ -1587,7 +1614,7 @@
 			"version": "5.0.1",
 			"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
 			"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
-			"dev": true,
+			"devOptional": true,
 			"engines": {
 				"node": ">=8"
 			}
@@ -1624,6 +1651,25 @@
 				"node": ">= 8"
 			}
 		},
+		"node_modules/aproba": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz",
+			"integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==",
+			"optional": true
+		},
+		"node_modules/are-we-there-yet": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz",
+			"integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==",
+			"optional": true,
+			"dependencies": {
+				"delegates": "^1.0.0",
+				"readable-stream": "^3.6.0"
+			},
+			"engines": {
+				"node": ">=10"
+			}
+		},
 		"node_modules/arg": {
 			"version": "5.0.2",
 			"resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
@@ -1950,6 +1996,56 @@
 				}
 			]
 		},
+		"node_modules/canvas": {
+			"version": "2.11.2",
+			"resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz",
+			"integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==",
+			"hasInstallScript": true,
+			"optional": true,
+			"dependencies": {
+				"@mapbox/node-pre-gyp": "^1.0.0",
+				"nan": "^2.17.0",
+				"simple-get": "^3.0.3"
+			},
+			"engines": {
+				"node": ">=6"
+			}
+		},
+		"node_modules/canvas/node_modules/decompress-response": {
+			"version": "4.2.1",
+			"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz",
+			"integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==",
+			"optional": true,
+			"dependencies": {
+				"mimic-response": "^2.0.0"
+			},
+			"engines": {
+				"node": ">=8"
+			}
+		},
+		"node_modules/canvas/node_modules/mimic-response": {
+			"version": "2.1.0",
+			"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz",
+			"integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==",
+			"optional": true,
+			"engines": {
+				"node": ">=8"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
+		"node_modules/canvas/node_modules/simple-get": {
+			"version": "3.1.1",
+			"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz",
+			"integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==",
+			"optional": true,
+			"dependencies": {
+				"decompress-response": "^4.2.0",
+				"once": "^1.3.1",
+				"simple-concat": "^1.0.0"
+			}
+		},
 		"node_modules/chai": {
 			"version": "4.3.7",
 			"resolved": "https://registry.npmjs.org/chai/-/chai-4.3.7.tgz",
@@ -2103,6 +2199,15 @@
 				"simple-swizzle": "^0.2.2"
 			}
 		},
+		"node_modules/color-support": {
+			"version": "1.1.3",
+			"resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
+			"integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==",
+			"optional": true,
+			"bin": {
+				"color-support": "bin.js"
+			}
+		},
 		"node_modules/combined-stream": {
 			"version": "1.0.8",
 			"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -2152,6 +2257,12 @@
 				"node": ">=10.18.0 <11 || >=12.14.0 <13 || >=14"
 			}
 		},
+		"node_modules/console-control-strings": {
+			"version": "1.1.0",
+			"resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
+			"integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==",
+			"optional": true
+		},
 		"node_modules/cookie": {
 			"version": "0.5.0",
 			"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
@@ -2363,6 +2474,12 @@
 				"node": ">=0.4.0"
 			}
 		},
+		"node_modules/delegates": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
+			"integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==",
+			"optional": true
+		},
 		"node_modules/dequal": {
 			"version": "2.0.3",
 			"resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
@@ -2472,6 +2589,12 @@
 			"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.359.tgz",
 			"integrity": "sha512-OoVcngKCIuNXtZnsYoqlCvr0Cf3NIPzDIgwUfI9bdTFjXCrr79lI0kwQstLPZ7WhCezLlGksZk/BFAzoXC7GDw=="
 		},
+		"node_modules/emoji-regex": {
+			"version": "8.0.0",
+			"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+			"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+			"optional": true
+		},
 		"node_modules/end-of-stream": {
 			"version": "1.4.4",
 			"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
@@ -2986,6 +3109,30 @@
 			"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
 			"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
 		},
+		"node_modules/fs-minipass": {
+			"version": "2.1.0",
+			"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
+			"integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==",
+			"optional": true,
+			"dependencies": {
+				"minipass": "^3.0.0"
+			},
+			"engines": {
+				"node": ">= 8"
+			}
+		},
+		"node_modules/fs-minipass/node_modules/minipass": {
+			"version": "3.3.6",
+			"resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
+			"integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
+			"optional": true,
+			"dependencies": {
+				"yallist": "^4.0.0"
+			},
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/fs.realpath": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -3012,6 +3159,26 @@
 				"url": "https://github.com/sponsors/ljharb"
 			}
 		},
+		"node_modules/gauge": {
+			"version": "3.0.2",
+			"resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz",
+			"integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==",
+			"optional": true,
+			"dependencies": {
+				"aproba": "^1.0.3 || ^2.0.0",
+				"color-support": "^1.1.2",
+				"console-control-strings": "^1.0.0",
+				"has-unicode": "^2.0.1",
+				"object-assign": "^4.1.1",
+				"signal-exit": "^3.0.0",
+				"string-width": "^4.2.3",
+				"strip-ansi": "^6.0.1",
+				"wide-align": "^1.1.2"
+			},
+			"engines": {
+				"node": ">=10"
+			}
+		},
 		"node_modules/get-func-name": {
 			"version": "2.0.2",
 			"resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz",
@@ -3042,7 +3209,7 @@
 			"version": "7.2.3",
 			"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
 			"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
-			"dev": true,
+			"devOptional": true,
 			"dependencies": {
 				"fs.realpath": "^1.0.0",
 				"inflight": "^1.0.4",
@@ -3168,6 +3335,12 @@
 				"node": ">=8"
 			}
 		},
+		"node_modules/has-unicode": {
+			"version": "2.0.1",
+			"resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
+			"integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==",
+			"optional": true
+		},
 		"node_modules/hash-wasm": {
 			"version": "4.9.0",
 			"resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.9.0.tgz",
@@ -3409,6 +3582,15 @@
 				"node": ">=0.10.0"
 			}
 		},
+		"node_modules/is-fullwidth-code-point": {
+			"version": "3.0.0",
+			"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+			"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+			"optional": true,
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/is-glob": {
 			"version": "4.0.3",
 			"resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
@@ -3791,6 +3973,30 @@
 				"node": ">=12"
 			}
 		},
+		"node_modules/make-dir": {
+			"version": "3.1.0",
+			"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz",
+			"integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==",
+			"optional": true,
+			"dependencies": {
+				"semver": "^6.0.0"
+			},
+			"engines": {
+				"node": ">=8"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
+		"node_modules/make-dir/node_modules/semver": {
+			"version": "6.3.1",
+			"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
+			"integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+			"optional": true,
+			"bin": {
+				"semver": "bin/semver.js"
+			}
+		},
 		"node_modules/make-error": {
 			"version": "1.3.6",
 			"resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
@@ -3948,6 +4154,40 @@
 				"url": "https://github.com/sponsors/ljharb"
 			}
 		},
+		"node_modules/minipass": {
+			"version": "5.0.0",
+			"resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
+			"integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==",
+			"optional": true,
+			"engines": {
+				"node": ">=8"
+			}
+		},
+		"node_modules/minizlib": {
+			"version": "2.1.2",
+			"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz",
+			"integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==",
+			"optional": true,
+			"dependencies": {
+				"minipass": "^3.0.0",
+				"yallist": "^4.0.0"
+			},
+			"engines": {
+				"node": ">= 8"
+			}
+		},
+		"node_modules/minizlib/node_modules/minipass": {
+			"version": "3.3.6",
+			"resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
+			"integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
+			"optional": true,
+			"dependencies": {
+				"yallist": "^4.0.0"
+			},
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/mkdirp": {
 			"version": "0.5.6",
 			"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
@@ -4059,6 +4299,12 @@
 				"thenify-all": "^1.0.0"
 			}
 		},
+		"node_modules/nan": {
+			"version": "2.18.0",
+			"resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz",
+			"integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==",
+			"optional": true
+		},
 		"node_modules/nanoid": {
 			"version": "4.0.2",
 			"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-4.0.2.tgz",
@@ -4191,6 +4437,21 @@
 			"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.10.tgz",
 			"integrity": "sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w=="
 		},
+		"node_modules/nopt": {
+			"version": "5.0.0",
+			"resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz",
+			"integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==",
+			"optional": true,
+			"dependencies": {
+				"abbrev": "1"
+			},
+			"bin": {
+				"nopt": "bin/nopt.js"
+			},
+			"engines": {
+				"node": ">=6"
+			}
+		},
 		"node_modules/normalize-path": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -4219,6 +4480,18 @@
 				"node": ">=8"
 			}
 		},
+		"node_modules/npmlog": {
+			"version": "5.0.1",
+			"resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz",
+			"integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==",
+			"optional": true,
+			"dependencies": {
+				"are-we-there-yet": "^2.0.0",
+				"console-control-strings": "^1.1.0",
+				"gauge": "^3.0.0",
+				"set-blocking": "^2.0.0"
+			}
+		},
 		"node_modules/nwsapi": {
 			"version": "2.2.4",
 			"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.4.tgz",
@@ -4508,6 +4781,15 @@
 				"node": ">=8"
 			}
 		},
+		"node_modules/path2d-polyfill": {
+			"version": "2.0.1",
+			"resolved": "https://registry.npmjs.org/path2d-polyfill/-/path2d-polyfill-2.0.1.tgz",
+			"integrity": "sha512-ad/3bsalbbWhmBo0D6FZ4RNMwsLsPpL6gnvhuSaU5Vm7b06Kr5ubSltQQ0T7YKsiJQO+g22zJ4dJKNTXIyOXtA==",
+			"optional": true,
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/pathe": {
 			"version": "1.1.0",
 			"resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.0.tgz",
@@ -4523,6 +4805,18 @@
 				"node": "*"
 			}
 		},
+		"node_modules/pdfjs-dist": {
+			"version": "4.0.269",
+			"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-4.0.269.tgz",
+			"integrity": "sha512-jjWO56tcOjnmPqDf8PmXDeZ781AGvpHMYI3HhNtaFKTRXXPaD1ArSrhVe38/XsrIQJ0onISCND/vuXaWJkiDWw==",
+			"engines": {
+				"node": ">=18"
+			},
+			"optionalDependencies": {
+				"canvas": "^2.11.2",
+				"path2d-polyfill": "^2.0.1"
+			}
+		},
 		"node_modules/periscopic": {
 			"version": "3.1.0",
 			"resolved": "https://registry.npmjs.org/periscopic/-/periscopic-3.1.0.tgz",
@@ -5165,7 +5459,7 @@
 			"version": "3.0.2",
 			"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
 			"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
-			"dev": true,
+			"devOptional": true,
 			"dependencies": {
 				"glob": "^7.1.3"
 			},
@@ -5323,6 +5617,12 @@
 				"undici": "^5.12.0"
 			}
 		},
+		"node_modules/set-blocking": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
+			"integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==",
+			"optional": true
+		},
 		"node_modules/set-cookie-parser": {
 			"version": "2.6.0",
 			"resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.6.0.tgz",
@@ -5382,7 +5682,7 @@
 			"version": "3.0.7",
 			"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
 			"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",
-			"dev": true
+			"devOptional": true
 		},
 		"node_modules/simple-concat": {
 			"version": "1.0.1",
@@ -5553,11 +5853,25 @@
 				"safe-buffer": "~5.2.0"
 			}
 		},
+		"node_modules/string-width": {
+			"version": "4.2.3",
+			"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+			"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+			"optional": true,
+			"dependencies": {
+				"emoji-regex": "^8.0.0",
+				"is-fullwidth-code-point": "^3.0.0",
+				"strip-ansi": "^6.0.1"
+			},
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/strip-ansi": {
 			"version": "6.0.1",
 			"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
 			"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-			"dev": true,
+			"devOptional": true,
 			"dependencies": {
 				"ansi-regex": "^5.0.1"
 			},
@@ -5977,6 +6291,23 @@
 				"node": ">= 14"
 			}
 		},
+		"node_modules/tar": {
+			"version": "6.2.0",
+			"resolved": "https://registry.npmjs.org/tar/-/tar-6.2.0.tgz",
+			"integrity": "sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==",
+			"optional": true,
+			"dependencies": {
+				"chownr": "^2.0.0",
+				"fs-minipass": "^2.0.0",
+				"minipass": "^5.0.0",
+				"minizlib": "^2.1.1",
+				"mkdirp": "^1.0.3",
+				"yallist": "^4.0.0"
+			},
+			"engines": {
+				"node": ">=10"
+			}
+		},
 		"node_modules/tar-fs": {
 			"version": "3.0.4",
 			"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.4.tgz",
@@ -5997,6 +6328,27 @@
 				"streamx": "^2.15.0"
 			}
 		},
+		"node_modules/tar/node_modules/chownr": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
+			"integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==",
+			"optional": true,
+			"engines": {
+				"node": ">=10"
+			}
+		},
+		"node_modules/tar/node_modules/mkdirp": {
+			"version": "1.0.4",
+			"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
+			"integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
+			"optional": true,
+			"bin": {
+				"mkdirp": "bin/cmd.js"
+			},
+			"engines": {
+				"node": ">=10"
+			}
+		},
 		"node_modules/text-table": {
 			"version": "0.2.0",
 			"resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
@@ -6675,6 +7027,15 @@
 				"node": ">=8"
 			}
 		},
+		"node_modules/wide-align": {
+			"version": "1.1.5",
+			"resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz",
+			"integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==",
+			"optional": true,
+			"dependencies": {
+				"string-width": "^1.0.2 || 2 || 3 || 4"
+			}
+		},
 		"node_modules/word-wrap": {
 			"version": "1.2.5",
 			"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",
diff --git a/package.json b/package.json
index cf65cabaeb7..fdc445dbebb 100644
--- a/package.json
+++ b/package.json
@@ -62,6 +62,7 @@
 		"nanoid": "^4.0.2",
 		"openid-client": "^5.4.2",
 		"parquetjs": "^0.11.2",
+		"pdfjs-dist": "^4.0.269",
 		"postcss": "^8.4.31",
 		"saslprep": "^1.0.3",
 		"serpapi": "^1.1.1",
diff --git a/src/lib/buildPrompt.ts b/src/lib/buildPrompt.ts
index 15e3a450e64..a1262f91045 100644
--- a/src/lib/buildPrompt.ts
+++ b/src/lib/buildPrompt.ts
@@ -2,7 +2,8 @@ import type { BackendModel } from "./server/models";
 import type { Message } from "./types/Message";
 import { format } from "date-fns";
 import type { WebSearch } from "./types/WebSearch";
-import { downloadFile } from "./server/files/downloadFile";
+import type { PdfSearch } from "./types/PdfChat";
+import { downloadImgFile } from "./server/files/downloadFile";
 import type { Conversation } from "./types/Conversation";
 
 interface buildPromptOptions {
@@ -11,6 +12,7 @@ interface buildPromptOptions {
 	model: BackendModel;
 	locals?: App.Locals;
 	webSearch?: WebSearch;
+	pdfSearch?: PdfSearch;
 	preprompt?: string;
 	files?: File[];
 }
@@ -19,6 +21,7 @@ export async function buildPrompt({
 	messages,
 	model,
 	webSearch,
+	pdfSearch,
 	preprompt,
 	id,
 }: buildPromptOptions): Promise<string> {
@@ -47,6 +50,31 @@ export async function buildPrompt({
 				`,
 			},
 		];
+	}else if (pdfSearch && pdfSearch.context) {
+		const lastMsg = messages.slice(-1)[0];
+		const messagesWithoutLastUsrMsg = messages.slice(0, -1);
+		const previousUserMessages = messages.filter((el) => el.from === "user").slice(0, -1);
+
+		const previousQuestions =
+			previousUserMessages.length > 0
+				? `Previous questions: \n${previousUserMessages
+						.map(({ content }) => `- ${content}`)
+						.join("\n")}`
+				: "";
+		
+		messages = [
+			...messagesWithoutLastUsrMsg,
+			{
+				from: "user",
+				content: `Below are the information I extracted from a PDF file that might be useful:
+				=====================
+				${pdfSearch.context}
+				=====================
+				${previousQuestions}
+				Answer the question: ${lastMsg.content} 
+				`,
+			},
+		];
 	}
 
 	// section to handle potential files input
@@ -60,7 +88,7 @@ export async function buildPrompt({
 						const markdowns = await Promise.all(
 							el.files.map(async (hash) => {
 								try {
-									const { content: image, mime } = await downloadFile(hash, id);
+									const { content: image, mime } = await downloadImgFile(hash, id);
 									const b64 = image.toString("base64");
 									return `![](data:${mime};base64,${b64})})`;
 								} catch (e) {
diff --git a/src/lib/components/OpenPdfSearchResults.svelte b/src/lib/components/OpenPdfSearchResults.svelte
new file mode 100644
index 00000000000..71a372b0813
--- /dev/null
+++ b/src/lib/components/OpenPdfSearchResults.svelte
@@ -0,0 +1,114 @@
+<script lang="ts">
+	import type { PdfSearchUpdate } from "$lib/types/MessageUpdate";
+	import CarbonCaretRight from "~icons/carbon/caret-right";
+
+	import CarbonCheckmark from "~icons/carbon/checkmark-filled";
+	import CarbonError from "~icons/carbon/error-filled";
+
+	import EosIconsLoading from "~icons/eos-icons/loading";
+
+	export let loading = false;
+	export let classNames = "";
+	export let pdfSearchMessages: PdfSearchUpdate[] = [];
+
+	let detailsOpen: boolean;
+	let error: boolean;
+	$: error = pdfSearchMessages[pdfSearchMessages.length - 1]?.messageType === "error";
+</script>
+
+<details
+	class="flex w-fit rounded-xl border border-gray-200 bg-white shadow-sm dark:border-gray-800 dark:bg-gray-900 {classNames} max-w-full"
+	bind:open={detailsOpen}
+>
+	<summary
+		class="align-center flex cursor-pointer select-none list-none py-1 pl-2.5 pr-2 align-text-top transition-all"
+	>
+		{#if error}
+			<CarbonError class="my-auto text-red-700 dark:text-red-500" />
+		{:else if loading}
+			<EosIconsLoading class="my-auto text-gray-500" />
+		{:else}
+			<CarbonCheckmark class="my-auto text-gray-500" />
+		{/if}
+		<span class="px-2 font-medium" class:text-red-700={error} class:dark:text-red-500={error}
+			>PDF search
+		</span>
+		<div class="my-auto transition-all" class:rotate-90={detailsOpen}>
+			<CarbonCaretRight />
+		</div>
+	</summary>
+
+	<div class="content px-5 pb-5 pt-4">
+		{#if pdfSearchMessages.length === 0}
+			<div class="mx-auto w-fit">
+				<EosIconsLoading class="mb-3 h-4 w-4" />
+			</div>
+		{:else}
+			<ol>
+				{#each pdfSearchMessages as message}
+					{#if message.messageType === "update"}
+						<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
+							<div class="flex items-start">
+								<div
+									class="-ml-1.5 h-3 w-3 flex-none rounded-full bg-gray-200 dark:bg-gray-600 {loading
+										? 'group-last:animate-pulse group-last:bg-gray-300 group-last:dark:bg-gray-500'
+										: ''}"
+								/>
+								<h3 class="text-md -mt-1.5 pl-2.5 text-gray-800 dark:text-gray-100">
+									{message.message}
+								</h3>
+							</div>
+							{#if message.args}
+								<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
+									{message.args}
+								</p>
+							{/if}
+						</li>
+					{:else if message.messageType === "error"}
+						<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
+							<div class="flex items-start">
+								<CarbonError
+									class="-ml-1.5 h-3 w-3 flex-none scale-110 text-red-700 dark:text-red-500"
+								/>
+								<h3 class="text-md -mt-1.5 pl-2.5 text-red-700 dark:text-red-500">
+									{message.message}
+								</h3>
+							</div>
+							{#if message.args}
+								<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
+									{message.args}
+								</p>
+							{/if}
+						</li>
+					{/if}
+				{/each}
+			</ol>
+		{/if}
+	</div>
+</details>
+
+<style>
+	@keyframes grow {
+		0% {
+			font-size: 0;
+			opacity: 0;
+		}
+		30% {
+			font-size: 1em;
+			opacity: 0;
+		}
+		100% {
+			opacity: 1;
+		}
+	}
+
+	details[open] .content {
+		animation-name: grow;
+		animation-duration: 300ms;
+		animation-delay: 0ms;
+	}
+
+	details summary::-webkit-details-marker {
+		display: none;
+	}
+</style>
diff --git a/src/lib/components/UploadBtn.svelte b/src/lib/components/UploadBtn.svelte
index cb869443e9b..e49d895849e 100644
--- a/src/lib/components/UploadBtn.svelte
+++ b/src/lib/components/UploadBtn.svelte
@@ -1,23 +1,72 @@
 <script lang="ts">
+	import { PdfUploadStatus } from "$lib/types/PdfChat";
+	import {createEventDispatcher, onDestroy} from "svelte";
 	import CarbonUpload from "~icons/carbon/upload";
 
 	export let classNames = "";
+	export let multimodal = false;
 	export let files: File[];
-	let filelist: FileList;
+	export let uploadPdfStatus: PdfUploadStatus;
+	
+	const accept = multimodal ? "image/*,.pdf" : ".pdf";
+	const label = multimodal ? "Upload image or PDF" : "Upload PDF";
+	let fileInput: HTMLInputElement;
+	let interval: ReturnType<typeof setInterval>;
 
-	$: if (filelist) {
-		files = Array.from(filelist);
+	$: uploading = uploadPdfStatus === PdfUploadStatus.Uploading;
+	$:{
+		if(uploadPdfStatus === PdfUploadStatus.Uploaded){
+			interval = setInterval(() => {
+				uploadPdfStatus = PdfUploadStatus.Ready;
+			}, 1500);
+		}
 	}
+
+	const dispatch = createEventDispatcher<{
+		uploadpdf: File;
+	}>();
+
+	function onChange() {
+		if(!fileInput.files){
+			return;
+		}
+		
+		const file = fileInput.files?.[0];
+		if (file?.type === "application/pdf") {
+			// pdf upload
+			dispatch("uploadpdf", file);
+		}else{
+			// image files for multimodal models
+			files = Array.from(fileInput.files);
+		}
+	}
+
+	onDestroy(() => {
+		if(interval){
+			clearInterval(interval);
+		}
+	})
 </script>
 
 <button
 	class="btn relative h-8 rounded-lg border bg-white px-3 py-1 text-sm text-gray-500 shadow-sm transition-all hover:bg-gray-100 dark:border-gray-600 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600 {classNames}"
+	class:animate-pulse={uploading}
+	class:pointer-events-none={uploading}
 >
 	<input
-		bind:files={filelist}
+		bind:this={fileInput}
+		on:change={onChange}
 		class="absolute w-full cursor-pointer opacity-0"
 		type="file"
-		accept="image/*"
+		{accept}
+		disabled={uploading}
 	/>
-	<CarbonUpload class="mr-2 text-xs " /> Upload image
+	<CarbonUpload class="mr-2 text-xs " />
+	{#if uploadPdfStatus === PdfUploadStatus.Uploaded}
+		PDF Uploaded ✅
+	{:else if uploading}
+		Processing PDF file
+	{:else}
+		{label}
+	{/if}
 </button>
diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte
index 158d8728863..b0f94c57d69 100644
--- a/src/lib/components/chat/ChatMessage.svelte
+++ b/src/lib/components/chat/ChatMessage.svelte
@@ -17,7 +17,8 @@
 	import type { Model } from "$lib/types/Model";
 
 	import OpenWebSearchResults from "../OpenWebSearchResults.svelte";
-	import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+	import OpenPdfSearchResults from "../OpenPdfSearchResults.svelte";
+	import type { RAGUpdate, WebSearchUpdate, PdfSearchUpdate } from "$lib/types/MessageUpdate";
 
 	function sanitizeMd(md: string) {
 		let ret = md
@@ -49,7 +50,7 @@
 	export let readOnly = false;
 	export let isTapped = false;
 
-	export let webSearchMessages: WebSearchUpdate[];
+	export let RAGMessages: RAGUpdate[];
 
 	const dispatch = createEventDispatcher<{
 		retry: { content: string; id: Message["id"] };
@@ -108,9 +109,15 @@
 
 	let searchUpdates: WebSearchUpdate[] = [];
 
-	$: searchUpdates = ((webSearchMessages.length > 0
-		? webSearchMessages
+	$: searchUpdates = ((RAGMessages.filter(({type}) => type === "webSearch").length > 0
+		? RAGMessages.filter(({type}) => type === "webSearch")
 		: message.updates?.filter(({ type }) => type === "webSearch")) ?? []) as WebSearchUpdate[];
+	
+	let pdfUpdates: PdfSearchUpdate[] = [];
+
+	$: pdfUpdates = ((RAGMessages.filter(({type}) => type === "pdfSearch").length > 0
+		? RAGMessages.filter(({type}) => type === "pdfSearch")
+		: message.updates?.filter(({ type }) => type === "pdfSearch")) ?? []) as PdfSearchUpdate[];
 
 	$: downloadLink =
 		message.from === "user" ? `${$page.url.pathname}/message/${message.id}/prompt` : undefined;
@@ -153,7 +160,14 @@
 					loading={!(searchUpdates[searchUpdates.length - 1]?.messageType === "sources")}
 				/>
 			{/if}
-			{#if !message.content && (webSearchIsDone || (webSearchMessages && webSearchMessages.length === 0))}
+			{#if pdfUpdates && pdfUpdates.length > 0}
+				<OpenPdfSearchResults
+					classNames={tokens.length ? "mb-3.5" : ""}
+					pdfSearchMessages={pdfUpdates}
+					loading={!(pdfUpdates[pdfUpdates.length - 1]?.messageType === "done")}
+				/>
+			{/if}
+			{#if !message.content && (webSearchIsDone || (RAGMessages && RAGMessages.length === 0))}
 				<IconLoading />
 			{/if}
 
diff --git a/src/lib/components/chat/ChatMessages.svelte b/src/lib/components/chat/ChatMessages.svelte
index 9ce0b115a3b..f1a7adbda19 100644
--- a/src/lib/components/chat/ChatMessages.svelte
+++ b/src/lib/components/chat/ChatMessages.svelte
@@ -7,7 +7,7 @@
 	import type { Model } from "$lib/types/Model";
 	import ChatIntroduction from "./ChatIntroduction.svelte";
 	import ChatMessage from "./ChatMessage.svelte";
-	import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+	import type { RAGUpdate } from "$lib/types/MessageUpdate";
 	import { browser } from "$app/environment";
 	import SystemPromptModal from "../SystemPromptModal.svelte";
 
@@ -22,7 +22,7 @@
 
 	let chatContainer: HTMLElement;
 
-	export let webSearchMessages: WebSearchUpdate[] = [];
+	export let RAGMessages: RAGUpdate[] = [];
 
 	async function scrollToBottom() {
 		await tick();
@@ -37,7 +37,7 @@
 
 <div
 	class="scrollbar-custom mr-1 h-full overflow-y-auto"
-	use:snapScrollToBottom={messages.length ? [...messages, ...webSearchMessages] : false}
+	use:snapScrollToBottom={messages.length ? [...messages, ...RAGMessages] : false}
 	bind:this={chatContainer}
 >
 	<div class="mx-auto flex h-full max-w-3xl flex-col gap-6 px-5 pt-6 sm:gap-8 xl:max-w-4xl">
@@ -51,7 +51,7 @@
 				{isAuthor}
 				{readOnly}
 				model={currentModel}
-				webSearchMessages={i === messages.length - 1 ? webSearchMessages : []}
+				RAGMessages={i === messages.length - 1 ? RAGMessages : []}
 				on:retry
 				on:vote
 			/>
@@ -62,7 +62,7 @@
 			<ChatMessage
 				message={{ from: "assistant", content: "", id: randomUUID() }}
 				model={currentModel}
-				{webSearchMessages}
+				{RAGMessages}
 			/>
 		{/if}
 		<div class="h-44 flex-none" />
diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte
index 949602cf37d..6cbbfad7f13 100644
--- a/src/lib/components/chat/ChatWindow.svelte
+++ b/src/lib/components/chat/ChatWindow.svelte
@@ -16,7 +16,7 @@
 	import type { Model } from "$lib/types/Model";
 	import WebSearchToggle from "../WebSearchToggle.svelte";
 	import LoginModal from "../LoginModal.svelte";
-	import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+	import type { RAGUpdate } from "$lib/types/MessageUpdate";
 	import { page } from "$app/stores";
 	import DisclaimerModal from "../DisclaimerModal.svelte";
 	import FileDropzone from "./FileDropzone.svelte";
@@ -24,6 +24,7 @@
 	import UploadBtn from "../UploadBtn.svelte";
 	import file2base64 from "$lib/utils/file2base64";
 	import { useSettingsStore } from "$lib/stores/settings";
+	import type { PdfUploadStatus } from "$lib/types/PdfChat";
 
 	export let messages: Message[] = [];
 	export let loading = false;
@@ -31,9 +32,10 @@
 	export let shared = false;
 	export let currentModel: Model;
 	export let models: Model[];
-	export let webSearchMessages: WebSearchUpdate[] = [];
+	export let RAGMessages: RAGUpdate[] = [];
 	export let preprompt: string | undefined = undefined;
 	export let files: File[] = [];
+	export let uploadPdfStatus: PdfUploadStatus;
 
 	$: isReadOnly = !models.some((model) => model.id === currentModel.id);
 
@@ -114,7 +116,7 @@
 		{messages}
 		readOnly={isReadOnly}
 		isAuthor={!shared}
-		{webSearchMessages}
+		{RAGMessages}
 		{preprompt}
 		on:message={(ev) => {
 			if ($page.data.loginRequired) {
@@ -173,9 +175,8 @@
 								content: messages[messages.length - 1].content,
 							})}
 					/>
-				{:else if currentModel.multimodal}
-					<UploadBtn bind:files classNames="ml-auto" />
 				{/if}
+				<UploadBtn bind:files on:uploadpdf classNames="ml-auto" multimodal={currentModel.multimodal} {uploadPdfStatus} />
 			</div>
 			<form
 				on:dragover={onDragOver}
diff --git a/src/lib/server/database.ts b/src/lib/server/database.ts
index 39d0c6a5273..5b9c3e28a0f 100644
--- a/src/lib/server/database.ts
+++ b/src/lib/server/database.ts
@@ -29,7 +29,10 @@ const settings = db.collection<Settings>("settings");
 const users = db.collection<User>("users");
 const sessions = db.collection<Session>("sessions");
 const messageEvents = db.collection<MessageEvent>("messageEvents");
-const bucket = new GridFSBucket(db, { bucketName: "files" });
+
+const bucketName = "files";
+const bucket = new GridFSBucket(db, { bucketName });
+const files = db.collection(`${bucketName}.files`);
 
 export { client, db };
 export const collections = {
@@ -41,6 +44,7 @@ export const collections = {
 	sessions,
 	messageEvents,
 	bucket,
+	files,
 };
 
 client.on("open", () => {
diff --git a/src/lib/server/embeddings.ts b/src/lib/server/embeddings.ts
new file mode 100644
index 00000000000..921c00d7569
--- /dev/null
+++ b/src/lib/server/embeddings.ts
@@ -0,0 +1,51 @@
+import type { Tensor, Pipeline } from "@xenova/transformers";
+import { pipeline, dot } from "@xenova/transformers";
+
+// see here: https://github.com/nmslib/hnswlib/blob/359b2ba87358224963986f709e593d799064ace6/README.md?plain=1#L34
+function innerProduct(tensor1: Tensor, tensor2: Tensor) {
+	return 1.0 - dot(tensor1.data, tensor2.data);
+}
+
+// Use the Singleton pattern to enable lazy construction of the pipeline.
+class PipelineSingleton {
+	static modelId = "Xenova/gte-small";
+	static instance: Promise<Pipeline> | null = null;
+	static async getInstance() {
+		if (this.instance === null) {
+			this.instance = pipeline("feature-extraction", this.modelId);
+		}
+		return this.instance;
+	}
+}
+
+// see https://huggingface.co/thenlper/gte-small/blob/d8e2604cadbeeda029847d19759d219e0ce2e6d8/README.md?code=true#L2625
+export const MAX_SEQ_LEN = 512 as const;
+
+export async function createEmbeddings(input: string[]) {
+	const extractor = await PipelineSingleton.getInstance();
+	const embeddings: Tensor = await extractor(input, { pooling: "mean", normalize: true });
+	return embeddings;
+}
+
+// docstring about the first sentence being the query sentence
+export function findSimilarSentences(
+	embeddings: Tensor,
+	queryEmbedding: Tensor,
+	{ topK = 5 }: { topK: number }
+) {
+	const distancesFromQuery: { distance: number; index: number }[] = [...embeddings].map(
+		(sentenceTensor: Tensor, index: number) => {
+			return {
+				distance: innerProduct(queryEmbedding, sentenceTensor),
+				index: index,
+			};
+		}
+	);
+
+	distancesFromQuery.sort((a, b) => {
+		return a.distance - b.distance;
+	});
+
+	// Return the indexes of the closest topK sentences
+	return distancesFromQuery.slice(0, topK).map((item) => item.index);
+}
\ No newline at end of file
diff --git a/src/lib/server/endpoints/aws/endpointAws.ts b/src/lib/server/endpoints/aws/endpointAws.ts
index 0cd899be19a..0051ceea3ec 100644
--- a/src/lib/server/endpoints/aws/endpointAws.ts
+++ b/src/lib/server/endpoints/aws/endpointAws.ts
@@ -40,6 +40,7 @@ export async function endpointAws(
 		const prompt = await buildPrompt({
 			messages: conversation.messages,
 			webSearch: conversation.messages[conversation.messages.length - 1].webSearch,
+			pdfSearch: conversation.messages[conversation.messages.length - 1].pdfSearch,
 			preprompt: conversation.preprompt,
 			model,
 		});
diff --git a/src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts b/src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts
index 33a3c93460e..9d6659eb0f4 100644
--- a/src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts
+++ b/src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts
@@ -23,6 +23,7 @@ export function endpointLlamacpp(
 		const prompt = await buildPrompt({
 			messages: conversation.messages,
 			webSearch: conversation.messages[conversation.messages.length - 1].webSearch,
+			pdfSearch: conversation.messages[conversation.messages.length - 1].pdfSearch,
 			preprompt: conversation.preprompt,
 			model,
 		});
diff --git a/src/lib/server/endpoints/ollama/endpointOllama.ts b/src/lib/server/endpoints/ollama/endpointOllama.ts
index fab06a8dd17..f33600d1b7a 100644
--- a/src/lib/server/endpoints/ollama/endpointOllama.ts
+++ b/src/lib/server/endpoints/ollama/endpointOllama.ts
@@ -18,6 +18,7 @@ export function endpointOllama(input: z.input<typeof endpointOllamaParametersSch
 		const prompt = await buildPrompt({
 			messages: conversation.messages,
 			webSearch: conversation.messages[conversation.messages.length - 1].webSearch,
+			pdfSearch: conversation.messages[conversation.messages.length - 1].pdfSearch,
 			preprompt: conversation.preprompt,
 			model,
 		});
diff --git a/src/lib/server/endpoints/openai/endpointOai.ts b/src/lib/server/endpoints/openai/endpointOai.ts
index 93cf87768ae..ae67ee05008 100644
--- a/src/lib/server/endpoints/openai/endpointOai.ts
+++ b/src/lib/server/endpoints/openai/endpointOai.ts
@@ -41,6 +41,7 @@ export async function endpointOai(
 					prompt: await buildPrompt({
 						messages: conversation.messages,
 						webSearch: conversation.messages[conversation.messages.length - 1].webSearch,
+						pdfSearch: conversation.messages[conversation.messages.length - 1].pdfSearch,
 						preprompt: conversation.preprompt,
 						model,
 					}),
diff --git a/src/lib/server/endpoints/tgi/endpointTgi.ts b/src/lib/server/endpoints/tgi/endpointTgi.ts
index e0307826fb4..d181c144946 100644
--- a/src/lib/server/endpoints/tgi/endpointTgi.ts
+++ b/src/lib/server/endpoints/tgi/endpointTgi.ts
@@ -19,6 +19,7 @@ export function endpointTgi(input: z.input<typeof endpointTgiParametersSchema>):
 		const prompt = await buildPrompt({
 			messages: conversation.messages,
 			webSearch: conversation.messages[conversation.messages.length - 1].webSearch,
+			pdfSearch: conversation.messages[conversation.messages.length - 1].pdfSearch,
 			preprompt: conversation.preprompt,
 			model,
 			id: conversation._id,
diff --git a/src/lib/server/files/downloadFile.ts b/src/lib/server/files/downloadFile.ts
index 4d2bddb1c30..5ec8d704d5f 100644
--- a/src/lib/server/files/downloadFile.ts
+++ b/src/lib/server/files/downloadFile.ts
@@ -3,7 +3,7 @@ import { collections } from "../database";
 import type { Conversation } from "$lib/types/Conversation";
 import type { SharedConversation } from "$lib/types/SharedConversation";
 
-export async function downloadFile(
+export async function downloadImgFile(
 	sha256: string,
 	convId: Conversation["_id"] | SharedConversation["_id"]
 ) {
@@ -34,3 +34,36 @@ export async function downloadFile(
 
 	return { content, mime };
 }
+
+export async function downloadPdfEmbeddings(
+	convId: Conversation["_id"] | SharedConversation["_id"]
+) {
+	const fileId = collections.bucket.find({ filename: `${convId.toString()}-pdf` });
+	let textChunks: string[] = [];
+	let dims: number[] = []
+
+	const content = await fileId.next().then(async (file) => {
+		if (!file) {
+			throw error(404, "File not found");
+		}
+		if (file.metadata?.conversation !== convId.toString()) {
+			throw error(403, "You don't have access to this file.");
+		}
+
+		textChunks = file.metadata?.textChunks;
+		dims = file.metadata?.dims;
+
+		const fileStream = collections.bucket.openDownloadStream(file._id);
+
+		const fileBuffer = await new Promise<Buffer>((resolve, reject) => {
+			const chunks: Uint8Array[] = [];
+			fileStream.on("data", (chunk) => chunks.push(chunk));
+			fileStream.on("error", reject);
+			fileStream.on("end", () => resolve(Buffer.concat(chunks)));
+		});
+
+		return fileBuffer;
+	});
+
+	return { content, textChunks, dims };
+}
diff --git a/src/lib/server/files/uploadFile.ts b/src/lib/server/files/uploadFile.ts
index 1c4a59b6f44..96859cd6424 100644
--- a/src/lib/server/files/uploadFile.ts
+++ b/src/lib/server/files/uploadFile.ts
@@ -1,8 +1,9 @@
 import type { Conversation } from "$lib/types/Conversation";
+import type { Tensor } from "@xenova/transformers";
 import { sha256 } from "$lib/utils/sha256";
 import { collections } from "../database";
 
-export async function uploadFile(file: Blob, conv: Conversation): Promise<string> {
+export async function uploadImgFile(file: Blob, conv: Conversation): Promise<string> {
 	const sha = await sha256(await file.text());
 
 	const upload = collections.bucket.openUploadStream(`${conv._id}-${sha}`, {
@@ -19,3 +20,30 @@ export async function uploadFile(file: Blob, conv: Conversation): Promise<string
 		setTimeout(() => reject(new Error("Upload timed out")), 10000);
 	});
 }
+
+export async function uploadPdfEmbeddings(embeddings: Tensor, textChunks: string[], conv: Conversation): Promise<void> {
+	const filename = `${conv._id}-pdf`;
+
+	// Step 1: Check if the file exists
+	const existingFile = await collections.files.findOne({ filename });
+
+	// Step 2: Delete the existing file if it exists
+	if (existingFile) {
+		await collections.bucket.delete(existingFile._id);
+	}
+
+	// Step 3: Upload the new file
+	const upload = collections.bucket.openUploadStream(filename, {
+		metadata: { conversation: conv._id.toString(), textChunks, dims: embeddings.dims },
+	});
+
+	upload.write((await embeddings.data.buffer) as unknown as Buffer);
+	upload.end();
+
+	// only return the filename when upload throws a finish event or a 10s time out occurs
+	return new Promise((resolve, reject) => {
+		upload.once("finish", () => resolve());
+		upload.once("error", reject);
+		setTimeout(() => reject(new Error("Upload timed out")), 10000);
+	});
+}
diff --git a/src/lib/server/pdfSearch.ts b/src/lib/server/pdfSearch.ts
new file mode 100644
index 00000000000..3a573c5f42d
--- /dev/null
+++ b/src/lib/server/pdfSearch.ts
@@ -0,0 +1,51 @@
+import type { PdfSearch } from "$lib/types/PdfChat";
+import {
+	createEmbeddings,
+	findSimilarSentences,
+} from "$lib/server/embeddings";
+import type { Conversation } from "$lib/types/Conversation";
+import type { MessageUpdate } from "$lib/types/MessageUpdate";
+import { downloadPdfEmbeddings } from "./files/downloadFile";
+import { Tensor } from "@xenova/transformers";
+
+// todo: embed the prompt, download the embeddings, serialize them, and find the closest sentences, and get their texts, lets go
+export async function runPdfSearch(
+	conv: Conversation,
+	prompt: string,
+	updatePad: (upd: MessageUpdate) => void
+) {
+	const pdfSearch: PdfSearch = {
+		context: "",
+		createdAt: new Date(),
+		updatedAt: new Date(),
+	};
+
+	function appendUpdate(message: string, args?: string[], type?: "error" | "update" | "done") {
+		updatePad({ type: "pdfSearch", messageType: type ?? "update", message: message, args: args });
+	}
+
+	try {
+		appendUpdate("Extracting relevant information from PDF file");
+		const { content, textChunks, dims } = await downloadPdfEmbeddings(conv._id);
+		// reconstruct pdfEmbeddings
+		const buffer = Buffer.from(content);
+		const data = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.length / Float32Array.BYTES_PER_ELEMENT);
+		const pdfEmbeddings = new Tensor('float32', data, dims);
+		const promptEmbeddings = await createEmbeddings([prompt]);
+
+		const indices = findSimilarSentences(pdfEmbeddings, promptEmbeddings, {topK: 5});
+		pdfSearch.context = indices.map((idx) => textChunks[idx]).join(" ");
+
+		appendUpdate("Done", [], "done");
+	} catch (pdfError) {
+		if (pdfError instanceof Error) {
+			appendUpdate(
+				"An error occurred with the pdf search",
+				[JSON.stringify(pdfError.message)],
+				"error"
+			);
+		}
+	}
+
+	return pdfSearch;
+}
diff --git a/src/lib/stores/pendingMessage.ts b/src/lib/stores/pendingMessage.ts
index 2a7387f393f..410155ad659 100644
--- a/src/lib/stores/pendingMessage.ts
+++ b/src/lib/stores/pendingMessage.ts
@@ -2,8 +2,9 @@ import { writable } from "svelte/store";
 
 export const pendingMessage = writable<
 	| {
-			content: string;
+			content?: string;
 			files: File[];
+			pdfFile?: File;
 	  }
 	| undefined
 >();
diff --git a/src/lib/types/Message.ts b/src/lib/types/Message.ts
index e485dfc444f..164fe6085f6 100644
--- a/src/lib/types/Message.ts
+++ b/src/lib/types/Message.ts
@@ -1,4 +1,5 @@
 import type { MessageUpdate } from "./MessageUpdate";
+import type { PdfSearch } from "./PdfChat";
 import type { Timestamps } from "./Timestamps";
 import type { WebSearch } from "./WebSearch";
 
@@ -9,6 +10,7 @@ export type Message = Partial<Timestamps> & {
 	updates?: MessageUpdate[];
 	webSearchId?: WebSearch["_id"]; // legacy version
 	webSearch?: WebSearch;
+	pdfSearch?: PdfSearch;
 	score?: -1 | 0 | 1;
 	files?: string[]; // can contain either the hash of the file or the b64 encoded image data on the client side when uploading
 };
diff --git a/src/lib/types/MessageUpdate.ts b/src/lib/types/MessageUpdate.ts
index 9bfb25667b9..684bb4cd24c 100644
--- a/src/lib/types/MessageUpdate.ts
+++ b/src/lib/types/MessageUpdate.ts
@@ -25,6 +25,15 @@ export type WebSearchUpdate = {
 	sources?: WebSearchSource[];
 };
 
+export type PdfSearchUpdate = {
+	type: "pdfSearch";
+	messageType: "update" | "error" | "done";
+	message: string;
+	args?: string[];
+};
+
+export type RAGUpdate = WebSearchUpdate | PdfSearchUpdate;
+
 export type StatusUpdate = {
 	type: "status";
 	status: "started" | "pending" | "finished" | "error" | "title";
@@ -42,5 +51,6 @@ export type MessageUpdate =
 	| TextStreamUpdate
 	| AgentUpdate
 	| WebSearchUpdate
+	| PdfSearchUpdate
 	| StatusUpdate
 	| ErrorUpdate;
diff --git a/src/lib/types/PdfChat.ts b/src/lib/types/PdfChat.ts
new file mode 100644
index 00000000000..9294ad8ba05
--- /dev/null
+++ b/src/lib/types/PdfChat.ts
@@ -0,0 +1,14 @@
+import type { ObjectId } from "mongodb";
+import type { Timestamps } from "./Timestamps";
+
+export interface PdfSearch extends Timestamps {
+	_id?: ObjectId;
+	context: string;
+}
+
+/* eslint-disable no-shadow */
+export enum PdfUploadStatus {
+	Ready = "Ready",
+	Uploading = "Uploading",
+	Uploaded = "Uploaded",
+}
diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte
index fa458cfe80b..ebeed603dcd 100644
--- a/src/routes/+page.svelte
+++ b/src/routes/+page.svelte
@@ -14,7 +14,7 @@
 
 	const settings = useSettingsStore();
 
-	async function createConversation(message: string) {
+	async function createConversation() {
 		try {
 			loading = true;
 			const res = await fetch(`${base}/conversation`, {
@@ -36,12 +36,6 @@
 
 			const { conversationId } = await res.json();
 
-			// Ugly hack to use a store as temp storage, feel free to improve ^^
-			pendingMessage.set({
-				content: message,
-				files,
-			});
-
 			// invalidateAll to update list of conversations
 			await goto(`${base}/conversation/${conversationId}`, { invalidateAll: true });
 		} catch (err) {
@@ -51,6 +45,25 @@
 			loading = false;
 		}
 	}
+
+	async function createConversationWithMsg(message: string) {
+		// Ugly hack to use a store as temp storage, feel free to improve ^^
+		pendingMessage.set({
+			content: message,
+			files,
+		});
+
+		await createConversation();
+	}
+
+	async function createConversationWithPdf(pdfFile: File) {
+		pendingMessage.set({
+			files,
+			pdfFile,
+		});
+
+		await createConversation();
+	}
 </script>
 
 <svelte:head>
@@ -58,7 +71,8 @@
 </svelte:head>
 
 <ChatWindow
-	on:message={(ev) => createConversation(ev.detail)}
+	on:message={(ev) => createConversationWithMsg(ev.detail)}
+	on:uploadpdf={(ev) => createConversationWithPdf(ev.detail)}
 	{loading}
 	currentModel={findCurrentModel([...data.models, ...data.oldModels], $settings.activeModel)}
 	models={data.models}
diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte
index ba00e9757a9..8cb0bd407fb 100644
--- a/src/routes/conversation/[id]/+page.svelte
+++ b/src/routes/conversation/[id]/+page.svelte
@@ -13,15 +13,16 @@
 	import { findCurrentModel } from "$lib/utils/models";
 	import { webSearchParameters } from "$lib/stores/webSearchParameters";
 	import type { Message } from "$lib/types/Message";
-	import type { MessageUpdate, WebSearchUpdate } from "$lib/types/MessageUpdate";
+	import type { MessageUpdate, RAGUpdate } from "$lib/types/MessageUpdate";
 	import titleUpdate from "$lib/stores/titleUpdate";
 	import file2base64 from "$lib/utils/file2base64";
+	import { PdfUploadStatus } from "$lib/types/PdfChat.js";
 	export let data;
 
 	let messages = data.messages;
 	let lastLoadedMessages = data.messages;
 
-	let webSearchMessages: WebSearchUpdate[] = [];
+	let RAGMessages: RAGUpdate[] = [];
 
 	// Since we modify the messages array locally, we don't want to reset it if an old version is passed
 	$: if (data.messages !== lastLoadedMessages) {
@@ -31,6 +32,7 @@
 
 	let loading = false;
 	let pending = false;
+	let uploadPdfStatus: PdfUploadStatus;
 
 	let files: File[] = [];
 
@@ -194,8 +196,8 @@
 									lastMessage.content += update.token;
 									messages = [...messages];
 								}
-							} else if (update.type === "webSearch") {
-								webSearchMessages = [...webSearchMessages, update];
+							} else if (update.type === "webSearch" || update.type === "pdfSearch") {
+								RAGMessages = [...RAGMessages, update];
 							} else if (update.type === "status") {
 								if (update.status === "title" && update.message) {
 									const conv = data.conversations.find(({ id }) => id === $page.params.id);
@@ -226,8 +228,8 @@
 				});
 			}
 
-			// reset the websearchMessages
-			webSearchMessages = [];
+			// reset the RAGMessages
+			RAGMessages = [];
 
 			await invalidate(UrlDependency.ConversationList);
 		} catch (err) {
@@ -273,11 +275,35 @@
 		}
 	}
 
+	async function uploadPdf(file: File) {
+		uploadPdfStatus = PdfUploadStatus.Uploading;
+
+		const formData = new FormData();
+        formData.append('pdf', file);
+
+		const res = await fetch(`${base}/conversation/${$page.params.id}/upload-pdf`, {
+			method: "POST",
+			body: formData,
+		});
+
+		if (!res.ok) {
+			error.set("Error while uploading PDF, try again.");
+			console.error("Error while uploading PDF: " + (await res.text()));
+		}
+
+		uploadPdfStatus = PdfUploadStatus.Uploaded;
+	}
+
 	onMount(async () => {
 		// only used in case of creating new conversations (from the parent POST endpoint)
 		if ($pendingMessage) {
 			files = $pendingMessage.files;
-			await writeMessage($pendingMessage.content);
+			if($pendingMessage.content){
+				await writeMessage($pendingMessage.content);
+			}
+			if($pendingMessage.pdfFile){
+				await uploadPdf($pendingMessage.pdfFile);
+			}
 			$pendingMessage = undefined;
 		}
 	});
@@ -328,11 +354,13 @@
 	{messages}
 	shared={data.shared}
 	preprompt={data.preprompt}
-	bind:webSearchMessages
+	bind:RAGMessages={RAGMessages}
 	bind:files
 	on:message={onMessage}
 	on:retry={onRetry}
 	on:vote={(event) => voteMessage(event.detail.score, event.detail.id)}
+	on:uploadpdf={(event) => uploadPdf(event.detail)}
+	{uploadPdfStatus}
 	on:share={() => shareConversation($page.params.id, data.title)}
 	on:stop={() => (($isAborted = true), (loading = false))}
 	models={data.models}
diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts
index daad2dd8283..fd623ae8668 100644
--- a/src/routes/conversation/[id]/+server.ts
+++ b/src/routes/conversation/[id]/+server.ts
@@ -9,10 +9,12 @@ import { ObjectId } from "mongodb";
 import { z } from "zod";
 import type { MessageUpdate } from "$lib/types/MessageUpdate";
 import { runWebSearch } from "$lib/server/websearch/runWebSearch";
+import { runPdfSearch } from "$lib/server/pdfSearch";
 import type { WebSearch } from "$lib/types/WebSearch";
+import type { PdfSearch } from "$lib/types/PdfChat";
 import { abortedGenerations } from "$lib/server/abortedGenerations";
 import { summarize } from "$lib/server/summarize";
-import { uploadFile } from "$lib/server/files/uploadFile";
+import { uploadImgFile } from "$lib/server/files/uploadFile";
 import sizeof from "image-size";
 
 export async function POST({ request, locals, params, getClientAddress }) {
@@ -135,7 +137,7 @@ export async function POST({ request, locals, params, getClientAddress }) {
 	let hashes: undefined | string[];
 
 	if (files) {
-		hashes = await Promise.all(files.map(async (file) => await uploadFile(file, conv)));
+		hashes = await Promise.all(files.map(async (file) => await uploadImgFile(file, conv)));
 	}
 
 	// get the list of messages
@@ -240,6 +242,14 @@ export async function POST({ request, locals, params, getClientAddress }) {
 
 			messages[messages.length - 1].webSearch = webSearchResults;
 
+			let pdfSearchResults: PdfSearch | undefined;
+			const pdfSearch = await collections.files.findOne({ filename: `${convId.toString()}-pdf` });
+			if(pdfSearch){
+				pdfSearchResults = await runPdfSearch(conv, newPrompt, update);
+			}
+
+			messages[messages.length - 1].pdfSearch = pdfSearchResults;
+
 			conv.messages = messages;
 
 			try {
diff --git a/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts b/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts
index 5ecac0bbcc1..01cbf56e88f 100644
--- a/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts
+++ b/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts
@@ -39,6 +39,7 @@ export async function GET({ params, locals }) {
 	const prompt = await buildPrompt({
 		preprompt: conv.preprompt,
 		webSearch: messagesUpTo[messagesUpTo.length - 1].webSearch,
+		pdfSearch: messagesUpTo[messagesUpTo.length - 1].pdfSearch,
 		messages: messagesUpTo,
 		model: model,
 	});
diff --git a/src/routes/conversation/[id]/output/[sha256]/+server.ts b/src/routes/conversation/[id]/output/[sha256]/+server.ts
index 79ae37b7585..0ab95db7ebd 100644
--- a/src/routes/conversation/[id]/output/[sha256]/+server.ts
+++ b/src/routes/conversation/[id]/output/[sha256]/+server.ts
@@ -4,7 +4,7 @@ import { error } from "@sveltejs/kit";
 import { ObjectId } from "mongodb";
 import { z } from "zod";
 import type { RequestHandler } from "./$types";
-import { downloadFile } from "$lib/server/files/downloadFile";
+import { downloadImgFile } from "$lib/server/files/downloadFile";
 
 export const GET: RequestHandler = async ({ locals, params }) => {
 	const sha256 = z.string().parse(params.sha256);
@@ -39,7 +39,7 @@ export const GET: RequestHandler = async ({ locals, params }) => {
 		}
 	}
 
-	const { content, mime } = await downloadFile(sha256, params.id);
+	const { content, mime } = await downloadImgFile(sha256, params.id);
 
 	return new Response(content, {
 		headers: {
diff --git a/src/routes/conversation/[id]/upload-pdf/+server.ts b/src/routes/conversation/[id]/upload-pdf/+server.ts
new file mode 100644
index 00000000000..e4532f1567c
--- /dev/null
+++ b/src/routes/conversation/[id]/upload-pdf/+server.ts
@@ -0,0 +1,41 @@
+import { authCondition } from "$lib/server/auth";
+import { collections } from "$lib/server/database";
+import { MAX_SEQ_LEN as CHUNK_CAR_LEN, createEmbeddings } from "$lib/server/embeddings";
+import { uploadPdfEmbeddings } from "$lib/server/files/uploadFile";
+import { chunk } from "$lib/utils/chunk";
+import { error } from "@sveltejs/kit";
+import { ObjectId } from "mongodb";
+import * as pdfjsLib from "pdfjs-dist/legacy/build/pdf";
+
+export async function POST({ request, params, locals }) {
+	const conversationId = new ObjectId(params.id);
+	const conversation = await collections.conversations.findOne({
+		_id: conversationId,
+		...authCondition(locals),
+	});
+
+	if (!conversation) {
+		throw error(404, "Conversation not found");
+	}
+
+	const formData = await request.formData();
+	const file = formData.get('pdf'); // 'pdf' is the name used in FormData on the frontend
+	const data = new Uint8Array(await file.arrayBuffer())
+	const loadingTask = pdfjsLib.getDocument({ data });
+	const pdf = await loadingTask.promise;
+
+	const N_MAX_PAGES = 20;
+	let text = '';
+	for (let i = 1; i <= Math.min(pdf.numPages, N_MAX_PAGES); i++) {
+		const page = await pdf.getPage(i);
+		const content = await page.getTextContent();
+		text += content.items.map(item => item.str).join(' ');
+	}
+
+	const textChunks = chunk(text, CHUNK_CAR_LEN);
+	const embeddings = await createEmbeddings(textChunks);
+
+	await uploadPdfEmbeddings(embeddings, textChunks, conversation);
+
+	return new Response();
+}

From cf4eddf7be03d0c2fe8b9e59f8694158c3b2eac8 Mon Sep 17 00:00:00 2001
From: Mishig Davaadorj <dmishig@gmail.com>
Date: Mon, 18 Dec 2023 15:44:26 +0100
Subject: [PATCH 02/11] prettier

---
 src/lib/buildPrompt.ts                         |  4 ++--
 src/lib/components/UploadBtn.svelte            | 18 +++++++++---------
 src/lib/components/chat/ChatMessage.svelte     | 10 +++++-----
 src/lib/components/chat/ChatWindow.svelte      |  8 +++++++-
 src/lib/server/embeddings.ts                   |  2 +-
 src/lib/server/files/downloadFile.ts           |  2 +-
 src/lib/server/files/uploadFile.ts             |  6 +++++-
 src/lib/server/pdfSearch.ts                    | 15 ++++++++-------
 src/routes/conversation/[id]/+page.svelte      |  8 ++++----
 src/routes/conversation/[id]/+server.ts        |  2 +-
 .../conversation/[id]/upload-pdf/+server.ts    |  8 ++++----
 11 files changed, 47 insertions(+), 36 deletions(-)

diff --git a/src/lib/buildPrompt.ts b/src/lib/buildPrompt.ts
index a1262f91045..27e20cbcf7c 100644
--- a/src/lib/buildPrompt.ts
+++ b/src/lib/buildPrompt.ts
@@ -50,7 +50,7 @@ export async function buildPrompt({
 				`,
 			},
 		];
-	}else if (pdfSearch && pdfSearch.context) {
+	} else if (pdfSearch && pdfSearch.context) {
 		const lastMsg = messages.slice(-1)[0];
 		const messagesWithoutLastUsrMsg = messages.slice(0, -1);
 		const previousUserMessages = messages.filter((el) => el.from === "user").slice(0, -1);
@@ -61,7 +61,7 @@ export async function buildPrompt({
 						.map(({ content }) => `- ${content}`)
 						.join("\n")}`
 				: "";
-		
+
 		messages = [
 			...messagesWithoutLastUsrMsg,
 			{
diff --git a/src/lib/components/UploadBtn.svelte b/src/lib/components/UploadBtn.svelte
index e49d895849e..58ab1f6c67d 100644
--- a/src/lib/components/UploadBtn.svelte
+++ b/src/lib/components/UploadBtn.svelte
@@ -1,21 +1,21 @@
 <script lang="ts">
 	import { PdfUploadStatus } from "$lib/types/PdfChat";
-	import {createEventDispatcher, onDestroy} from "svelte";
+	import { createEventDispatcher, onDestroy } from "svelte";
 	import CarbonUpload from "~icons/carbon/upload";
 
 	export let classNames = "";
 	export let multimodal = false;
 	export let files: File[];
 	export let uploadPdfStatus: PdfUploadStatus;
-	
+
 	const accept = multimodal ? "image/*,.pdf" : ".pdf";
 	const label = multimodal ? "Upload image or PDF" : "Upload PDF";
 	let fileInput: HTMLInputElement;
 	let interval: ReturnType<typeof setInterval>;
 
 	$: uploading = uploadPdfStatus === PdfUploadStatus.Uploading;
-	$:{
-		if(uploadPdfStatus === PdfUploadStatus.Uploaded){
+	$: {
+		if (uploadPdfStatus === PdfUploadStatus.Uploaded) {
 			interval = setInterval(() => {
 				uploadPdfStatus = PdfUploadStatus.Ready;
 			}, 1500);
@@ -27,25 +27,25 @@
 	}>();
 
 	function onChange() {
-		if(!fileInput.files){
+		if (!fileInput.files) {
 			return;
 		}
-		
+
 		const file = fileInput.files?.[0];
 		if (file?.type === "application/pdf") {
 			// pdf upload
 			dispatch("uploadpdf", file);
-		}else{
+		} else {
 			// image files for multimodal models
 			files = Array.from(fileInput.files);
 		}
 	}
 
 	onDestroy(() => {
-		if(interval){
+		if (interval) {
 			clearInterval(interval);
 		}
-	})
+	});
 </script>
 
 <button
diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte
index b0f94c57d69..7d52b067a11 100644
--- a/src/lib/components/chat/ChatMessage.svelte
+++ b/src/lib/components/chat/ChatMessage.svelte
@@ -109,14 +109,14 @@
 
 	let searchUpdates: WebSearchUpdate[] = [];
 
-	$: searchUpdates = ((RAGMessages.filter(({type}) => type === "webSearch").length > 0
-		? RAGMessages.filter(({type}) => type === "webSearch")
+	$: searchUpdates = ((RAGMessages.filter(({ type }) => type === "webSearch").length > 0
+		? RAGMessages.filter(({ type }) => type === "webSearch")
 		: message.updates?.filter(({ type }) => type === "webSearch")) ?? []) as WebSearchUpdate[];
-	
+
 	let pdfUpdates: PdfSearchUpdate[] = [];
 
-	$: pdfUpdates = ((RAGMessages.filter(({type}) => type === "pdfSearch").length > 0
-		? RAGMessages.filter(({type}) => type === "pdfSearch")
+	$: pdfUpdates = ((RAGMessages.filter(({ type }) => type === "pdfSearch").length > 0
+		? RAGMessages.filter(({ type }) => type === "pdfSearch")
 		: message.updates?.filter(({ type }) => type === "pdfSearch")) ?? []) as PdfSearchUpdate[];
 
 	$: downloadLink =
diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte
index 6cbbfad7f13..9f04448cc74 100644
--- a/src/lib/components/chat/ChatWindow.svelte
+++ b/src/lib/components/chat/ChatWindow.svelte
@@ -176,7 +176,13 @@
 							})}
 					/>
 				{/if}
-				<UploadBtn bind:files on:uploadpdf classNames="ml-auto" multimodal={currentModel.multimodal} {uploadPdfStatus} />
+				<UploadBtn
+					bind:files
+					on:uploadpdf
+					classNames="ml-auto"
+					multimodal={currentModel.multimodal}
+					{uploadPdfStatus}
+				/>
 			</div>
 			<form
 				on:dragover={onDragOver}
diff --git a/src/lib/server/embeddings.ts b/src/lib/server/embeddings.ts
index 921c00d7569..bd6651683c1 100644
--- a/src/lib/server/embeddings.ts
+++ b/src/lib/server/embeddings.ts
@@ -48,4 +48,4 @@ export function findSimilarSentences(
 
 	// Return the indexes of the closest topK sentences
 	return distancesFromQuery.slice(0, topK).map((item) => item.index);
-}
\ No newline at end of file
+}
diff --git a/src/lib/server/files/downloadFile.ts b/src/lib/server/files/downloadFile.ts
index 5ec8d704d5f..c33e827e53e 100644
--- a/src/lib/server/files/downloadFile.ts
+++ b/src/lib/server/files/downloadFile.ts
@@ -40,7 +40,7 @@ export async function downloadPdfEmbeddings(
 ) {
 	const fileId = collections.bucket.find({ filename: `${convId.toString()}-pdf` });
 	let textChunks: string[] = [];
-	let dims: number[] = []
+	let dims: number[] = [];
 
 	const content = await fileId.next().then(async (file) => {
 		if (!file) {
diff --git a/src/lib/server/files/uploadFile.ts b/src/lib/server/files/uploadFile.ts
index 96859cd6424..87022b19cfc 100644
--- a/src/lib/server/files/uploadFile.ts
+++ b/src/lib/server/files/uploadFile.ts
@@ -21,7 +21,11 @@ export async function uploadImgFile(file: Blob, conv: Conversation): Promise<str
 	});
 }
 
-export async function uploadPdfEmbeddings(embeddings: Tensor, textChunks: string[], conv: Conversation): Promise<void> {
+export async function uploadPdfEmbeddings(
+	embeddings: Tensor,
+	textChunks: string[],
+	conv: Conversation
+): Promise<void> {
 	const filename = `${conv._id}-pdf`;
 
 	// Step 1: Check if the file exists
diff --git a/src/lib/server/pdfSearch.ts b/src/lib/server/pdfSearch.ts
index 3a573c5f42d..6fdbaf8668a 100644
--- a/src/lib/server/pdfSearch.ts
+++ b/src/lib/server/pdfSearch.ts
@@ -1,8 +1,5 @@
 import type { PdfSearch } from "$lib/types/PdfChat";
-import {
-	createEmbeddings,
-	findSimilarSentences,
-} from "$lib/server/embeddings";
+import { createEmbeddings, findSimilarSentences } from "$lib/server/embeddings";
 import type { Conversation } from "$lib/types/Conversation";
 import type { MessageUpdate } from "$lib/types/MessageUpdate";
 import { downloadPdfEmbeddings } from "./files/downloadFile";
@@ -29,11 +26,15 @@ export async function runPdfSearch(
 		const { content, textChunks, dims } = await downloadPdfEmbeddings(conv._id);
 		// reconstruct pdfEmbeddings
 		const buffer = Buffer.from(content);
-		const data = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.length / Float32Array.BYTES_PER_ELEMENT);
-		const pdfEmbeddings = new Tensor('float32', data, dims);
+		const data = new Float32Array(
+			buffer.buffer,
+			buffer.byteOffset,
+			buffer.length / Float32Array.BYTES_PER_ELEMENT
+		);
+		const pdfEmbeddings = new Tensor("float32", data, dims);
 		const promptEmbeddings = await createEmbeddings([prompt]);
 
-		const indices = findSimilarSentences(pdfEmbeddings, promptEmbeddings, {topK: 5});
+		const indices = findSimilarSentences(pdfEmbeddings, promptEmbeddings, { topK: 5 });
 		pdfSearch.context = indices.map((idx) => textChunks[idx]).join(" ");
 
 		appendUpdate("Done", [], "done");
diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte
index 8cb0bd407fb..123f00800da 100644
--- a/src/routes/conversation/[id]/+page.svelte
+++ b/src/routes/conversation/[id]/+page.svelte
@@ -279,7 +279,7 @@
 		uploadPdfStatus = PdfUploadStatus.Uploading;
 
 		const formData = new FormData();
-        formData.append('pdf', file);
+		formData.append("pdf", file);
 
 		const res = await fetch(`${base}/conversation/${$page.params.id}/upload-pdf`, {
 			method: "POST",
@@ -298,10 +298,10 @@
 		// only used in case of creating new conversations (from the parent POST endpoint)
 		if ($pendingMessage) {
 			files = $pendingMessage.files;
-			if($pendingMessage.content){
+			if ($pendingMessage.content) {
 				await writeMessage($pendingMessage.content);
 			}
-			if($pendingMessage.pdfFile){
+			if ($pendingMessage.pdfFile) {
 				await uploadPdf($pendingMessage.pdfFile);
 			}
 			$pendingMessage = undefined;
@@ -354,7 +354,7 @@
 	{messages}
 	shared={data.shared}
 	preprompt={data.preprompt}
-	bind:RAGMessages={RAGMessages}
+	bind:RAGMessages
 	bind:files
 	on:message={onMessage}
 	on:retry={onRetry}
diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts
index fd623ae8668..e921377221d 100644
--- a/src/routes/conversation/[id]/+server.ts
+++ b/src/routes/conversation/[id]/+server.ts
@@ -244,7 +244,7 @@ export async function POST({ request, locals, params, getClientAddress }) {
 
 			let pdfSearchResults: PdfSearch | undefined;
 			const pdfSearch = await collections.files.findOne({ filename: `${convId.toString()}-pdf` });
-			if(pdfSearch){
+			if (pdfSearch) {
 				pdfSearchResults = await runPdfSearch(conv, newPrompt, update);
 			}
 
diff --git a/src/routes/conversation/[id]/upload-pdf/+server.ts b/src/routes/conversation/[id]/upload-pdf/+server.ts
index e4532f1567c..f3cd34f1609 100644
--- a/src/routes/conversation/[id]/upload-pdf/+server.ts
+++ b/src/routes/conversation/[id]/upload-pdf/+server.ts
@@ -19,17 +19,17 @@ export async function POST({ request, params, locals }) {
 	}
 
 	const formData = await request.formData();
-	const file = formData.get('pdf'); // 'pdf' is the name used in FormData on the frontend
-	const data = new Uint8Array(await file.arrayBuffer())
+	const file = formData.get("pdf"); // 'pdf' is the name used in FormData on the frontend
+	const data = new Uint8Array(await file.arrayBuffer());
 	const loadingTask = pdfjsLib.getDocument({ data });
 	const pdf = await loadingTask.promise;
 
 	const N_MAX_PAGES = 20;
-	let text = '';
+	let text = "";
 	for (let i = 1; i <= Math.min(pdf.numPages, N_MAX_PAGES); i++) {
 		const page = await pdf.getPage(i);
 		const content = await page.getTextContent();
-		text += content.items.map(item => item.str).join(' ');
+		text += content.items.map((item) => item.str).join(" ");
 	}
 
 	const textChunks = chunk(text, CHUNK_CAR_LEN);

From b96bc8ba6551fba8edf3f8a11331935685f7c5ca Mon Sep 17 00:00:00 2001
From: Mishig Davaadorj <dmishig@gmail.com>
Date: Mon, 8 Jan 2024 16:00:17 +0100
Subject: [PATCH 03/11] correct usage of `pdfjs-dist`

---
 src/routes/conversation/[id]/upload-pdf/+server.ts | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/routes/conversation/[id]/upload-pdf/+server.ts b/src/routes/conversation/[id]/upload-pdf/+server.ts
index f3cd34f1609..a34c36d69cb 100644
--- a/src/routes/conversation/[id]/upload-pdf/+server.ts
+++ b/src/routes/conversation/[id]/upload-pdf/+server.ts
@@ -5,7 +5,7 @@ import { uploadPdfEmbeddings } from "$lib/server/files/uploadFile";
 import { chunk } from "$lib/utils/chunk";
 import { error } from "@sveltejs/kit";
 import { ObjectId } from "mongodb";
-import * as pdfjsLib from "pdfjs-dist/legacy/build/pdf";
+import { getDocument } from "pdfjs-dist";
 
 export async function POST({ request, params, locals }) {
 	const conversationId = new ObjectId(params.id);
@@ -21,8 +21,7 @@ export async function POST({ request, params, locals }) {
 	const formData = await request.formData();
 	const file = formData.get("pdf"); // 'pdf' is the name used in FormData on the frontend
 	const data = new Uint8Array(await file.arrayBuffer());
-	const loadingTask = pdfjsLib.getDocument({ data });
-	const pdf = await loadingTask.promise;
+	const pdf = await getDocument({ data }).promise;
 
 	const N_MAX_PAGES = 20;
 	let text = "";

From 9a0a1e6a77bbdf78ae7f5de47f51871bb9dfc228 Mon Sep 17 00:00:00 2001
From: Mishig <mishig.davaadorj@coloradocollege.edu>
Date: Mon, 8 Jan 2024 07:11:59 -0800
Subject: [PATCH 04/11] Updates from code reviews

Co-authored-by: Nathan Sarrazin <sarrazin.nathan@gmail.com>
---
 src/lib/components/UploadBtn.svelte                | 3 +--
 src/lib/components/chat/ChatWindow.svelte          | 2 +-
 src/routes/conversation/[id]/upload-pdf/+server.ts | 5 ++++-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/lib/components/UploadBtn.svelte b/src/lib/components/UploadBtn.svelte
index 58ab1f6c67d..8235bbf2d0d 100644
--- a/src/lib/components/UploadBtn.svelte
+++ b/src/lib/components/UploadBtn.svelte
@@ -6,8 +6,7 @@
 	export let classNames = "";
 	export let multimodal = false;
 	export let files: File[];
-	export let uploadPdfStatus: PdfUploadStatus;
-
+	export let uploadPdfStatus: PdfUploadStatus | undefined = undefined;
 	const accept = multimodal ? "image/*,.pdf" : ".pdf";
 	const label = multimodal ? "Upload image or PDF" : "Upload PDF";
 	let fileInput: HTMLInputElement;
diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte
index 9f04448cc74..f0390c495cc 100644
--- a/src/lib/components/chat/ChatWindow.svelte
+++ b/src/lib/components/chat/ChatWindow.svelte
@@ -35,7 +35,7 @@
 	export let RAGMessages: RAGUpdate[] = [];
 	export let preprompt: string | undefined = undefined;
 	export let files: File[] = [];
-	export let uploadPdfStatus: PdfUploadStatus;
+	export let uploadPdfStatus: PdfUploadStatus | undefined = undefined;
 
 	$: isReadOnly = !models.some((model) => model.id === currentModel.id);
 
diff --git a/src/routes/conversation/[id]/upload-pdf/+server.ts b/src/routes/conversation/[id]/upload-pdf/+server.ts
index a34c36d69cb..f88d0d1c149 100644
--- a/src/routes/conversation/[id]/upload-pdf/+server.ts
+++ b/src/routes/conversation/[id]/upload-pdf/+server.ts
@@ -20,6 +20,9 @@ export async function POST({ request, params, locals }) {
 
 	const formData = await request.formData();
 	const file = formData.get("pdf"); // 'pdf' is the name used in FormData on the frontend
+	if (!file || typeof file === "string") {
+		throw error(400, "No file provided");
+	}
 	const data = new Uint8Array(await file.arrayBuffer());
 	const pdf = await getDocument({ data }).promise;
 
@@ -28,7 +31,7 @@ export async function POST({ request, params, locals }) {
 	for (let i = 1; i <= Math.min(pdf.numPages, N_MAX_PAGES); i++) {
 		const page = await pdf.getPage(i);
 		const content = await page.getTextContent();
-		text += content.items.map((item) => item.str).join(" ");
+		text += content.items.map((item) => (item as { str?: string }).str ?? "").join(" ");
 	}
 
 	const textChunks = chunk(text, CHUNK_CAR_LEN);

From fd512ddf1d8c9e72e7de32ebd1dfc732f015ce34 Mon Sep 17 00:00:00 2001
From: Mishig Davaadorj <dmishig@gmail.com>
Date: Mon, 8 Jan 2024 18:05:52 +0100
Subject: [PATCH 05/11] fix file drag-n-drop

---
 src/lib/components/UploadBtn.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/components/UploadBtn.svelte b/src/lib/components/UploadBtn.svelte
index 8235bbf2d0d..616136c10b3 100644
--- a/src/lib/components/UploadBtn.svelte
+++ b/src/lib/components/UploadBtn.svelte
@@ -34,7 +34,7 @@
 		if (file?.type === "application/pdf") {
 			// pdf upload
 			dispatch("uploadpdf", file);
-		} else {
+		} else if(multimodal && file?.type.startsWith("image")){
 			// image files for multimodal models
 			files = Array.from(fileInput.files);
 		}

From 9f29fa7b557a380004fbdf0a3729a9bdd2fb1dc4 Mon Sep 17 00:00:00 2001
From: Mishig Davaadorj <dmishig@gmail.com>
Date: Tue, 9 Jan 2024 09:51:52 +0100
Subject: [PATCH 06/11] Better UI/UX feedback for uploaded pdf

---
 src/lib/components/UploadBtn.svelte           | 36 +++++++++++--------
 src/lib/components/UploadedPdfStatus.svelte   | 28 +++++++++++++++
 src/lib/components/chat/ChatWindow.svelte     | 18 +++++-----
 src/lib/server/files/uploadFile.ts            | 21 ++++++-----
 src/lib/types/PdfChat.ts                      |  5 +++
 src/routes/conversation/[id]/+page.svelte     | 24 ++++++++++---
 .../conversation/[id]/upload-pdf/+server.ts   | 20 ++++++++++-
 7 files changed, 114 insertions(+), 38 deletions(-)
 create mode 100644 src/lib/components/UploadedPdfStatus.svelte

diff --git a/src/lib/components/UploadBtn.svelte b/src/lib/components/UploadBtn.svelte
index 616136c10b3..62b87316cee 100644
--- a/src/lib/components/UploadBtn.svelte
+++ b/src/lib/components/UploadBtn.svelte
@@ -1,22 +1,26 @@
 <script lang="ts">
-	import { PdfUploadStatus } from "$lib/types/PdfChat";
+	import { PdfUploadStatus, type PdfUpload } from "$lib/types/PdfChat";
 	import { createEventDispatcher, onDestroy } from "svelte";
 	import CarbonUpload from "~icons/carbon/upload";
+	import CarbonCheckmark from "~icons/carbon/checkmark";
 
 	export let classNames = "";
 	export let multimodal = false;
 	export let files: File[];
-	export let uploadPdfStatus: PdfUploadStatus | undefined = undefined;
+	export let pdfUpload: PdfUpload | undefined = undefined;
 	const accept = multimodal ? "image/*,.pdf" : ".pdf";
 	const label = multimodal ? "Upload image or PDF" : "Upload PDF";
 	let fileInput: HTMLInputElement;
 	let interval: ReturnType<typeof setInterval>;
 
-	$: uploading = uploadPdfStatus === PdfUploadStatus.Uploading;
+	$: pdfUploading = pdfUpload?.status === PdfUploadStatus.Uploading;
 	$: {
-		if (uploadPdfStatus === PdfUploadStatus.Uploaded) {
+		if (pdfUpload?.status === PdfUploadStatus.Uploaded) {
 			interval = setInterval(() => {
-				uploadPdfStatus = PdfUploadStatus.Ready;
+				if (!pdfUpload) {
+					return;
+				}
+				pdfUpload.status = PdfUploadStatus.Ready;
 			}, 1500);
 		}
 	}
@@ -34,7 +38,7 @@
 		if (file?.type === "application/pdf") {
 			// pdf upload
 			dispatch("uploadpdf", file);
-		} else if(multimodal && file?.type.startsWith("image")){
+		} else if (multimodal && file?.type.startsWith("image")) {
 			// image files for multimodal models
 			files = Array.from(fileInput.files);
 		}
@@ -49,8 +53,8 @@
 
 <button
 	class="btn relative h-8 rounded-lg border bg-white px-3 py-1 text-sm text-gray-500 shadow-sm transition-all hover:bg-gray-100 dark:border-gray-600 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600 {classNames}"
-	class:animate-pulse={uploading}
-	class:pointer-events-none={uploading}
+	class:animate-pulse={pdfUploading}
+	class:pointer-events-none={pdfUploading || pdfUpload?.status === PdfUploadStatus.Uploaded}
 >
 	<input
 		bind:this={fileInput}
@@ -58,14 +62,16 @@
 		class="absolute w-full cursor-pointer opacity-0"
 		type="file"
 		{accept}
-		disabled={uploading}
+		disabled={pdfUploading}
 	/>
-	<CarbonUpload class="mr-2 text-xs " />
-	{#if uploadPdfStatus === PdfUploadStatus.Uploaded}
-		PDF Uploaded ✅
-	{:else if uploading}
-		Processing PDF file
+	{#if pdfUpload?.status !== PdfUploadStatus.Uploaded}
+		<CarbonUpload class="text-xs" />
 	{:else}
-		{label}
+		<CarbonCheckmark class="text-xs text-green-500" />
+	{/if}
+	{#if multimodal || !pdfUpload?.name}
+		<div class="ml-2">
+			{label}
+		</div>
 	{/if}
 </button>
diff --git a/src/lib/components/UploadedPdfStatus.svelte b/src/lib/components/UploadedPdfStatus.svelte
new file mode 100644
index 00000000000..fec70e51d7b
--- /dev/null
+++ b/src/lib/components/UploadedPdfStatus.svelte
@@ -0,0 +1,28 @@
+<script lang="ts">
+	import { PdfUploadStatus, type PdfUpload } from "$lib/types/PdfChat";
+	import CarbonDocumentBlank from "~icons/carbon/document-blank";
+	import CarbonClose from "~icons/carbon/close";
+	import { createEventDispatcher } from "svelte";
+
+	export let pdfUpload: PdfUpload;
+
+	const dispatch = createEventDispatcher<{
+		deletepdf: void;
+	}>();
+
+	$: uploading = pdfUpload.status === PdfUploadStatus.Uploading;
+</script>
+
+<div
+	class="max-w-48 group flex items-center gap-x-1"
+	class:animate-pulse={uploading}
+	class:pointer-events-none={uploading}
+	title={pdfUpload.name}
+>
+	<button
+		class="-mr-1 block md:hidden shrink-0 opacity-70 group-hover:block"
+		on:click={() => dispatch("deletepdf")}><CarbonClose /></button
+	>
+	<CarbonDocumentBlank class="shrink-0" />
+	<p class="truncate hidden md:block">{pdfUpload.name}</p>
+</div>
diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte
index f0390c495cc..673d67465a1 100644
--- a/src/lib/components/chat/ChatWindow.svelte
+++ b/src/lib/components/chat/ChatWindow.svelte
@@ -24,7 +24,8 @@
 	import UploadBtn from "../UploadBtn.svelte";
 	import file2base64 from "$lib/utils/file2base64";
 	import { useSettingsStore } from "$lib/stores/settings";
-	import type { PdfUploadStatus } from "$lib/types/PdfChat";
+	import type { PdfUpload } from "$lib/types/PdfChat";
+	import UploadedPdfStatus from "../UploadedPdfStatus.svelte";
 
 	export let messages: Message[] = [];
 	export let loading = false;
@@ -35,7 +36,7 @@
 	export let RAGMessages: RAGUpdate[] = [];
 	export let preprompt: string | undefined = undefined;
 	export let files: File[] = [];
-	export let uploadPdfStatus: PdfUploadStatus | undefined = undefined;
+	export let pdfUpload: PdfUpload | undefined = undefined;
 
 	$: isReadOnly = !models.some((model) => model.id === currentModel.id);
 
@@ -176,13 +177,12 @@
 							})}
 					/>
 				{/if}
-				<UploadBtn
-					bind:files
-					on:uploadpdf
-					classNames="ml-auto"
-					multimodal={currentModel.multimodal}
-					{uploadPdfStatus}
-				/>
+				<div class="ml-auto flex items-center gap-x-3">
+					{#if pdfUpload?.name}
+						<UploadedPdfStatus on:deletepdf {pdfUpload} />
+					{/if}
+					<UploadBtn bind:files on:uploadpdf multimodal={currentModel.multimodal} {pdfUpload} />
+				</div>
 			</div>
 			<form
 				on:dragover={onDragOver}
diff --git a/src/lib/server/files/uploadFile.ts b/src/lib/server/files/uploadFile.ts
index 87022b19cfc..7b2b0dbe1f4 100644
--- a/src/lib/server/files/uploadFile.ts
+++ b/src/lib/server/files/uploadFile.ts
@@ -21,13 +21,7 @@ export async function uploadImgFile(file: Blob, conv: Conversation): Promise<str
 	});
 }
 
-export async function uploadPdfEmbeddings(
-	embeddings: Tensor,
-	textChunks: string[],
-	conv: Conversation
-): Promise<void> {
-	const filename = `${conv._id}-pdf`;
-
+export async function deleteFile(filename: string){
 	// Step 1: Check if the file exists
 	const existingFile = await collections.files.findOne({ filename });
 
@@ -35,8 +29,19 @@ export async function uploadPdfEmbeddings(
 	if (existingFile) {
 		await collections.bucket.delete(existingFile._id);
 	}
+}
+
+export async function uploadPdfEmbeddings(
+	embeddings: Tensor,
+	textChunks: string[],
+	conv: Conversation
+): Promise<void> {
+	const filename = `${conv._id}-pdf`;
+
+	// Step 1: Delete the existing file if it exists
+	await deleteFile(filename);
 
-	// Step 3: Upload the new file
+	// Step 2: Upload the new file
 	const upload = collections.bucket.openUploadStream(filename, {
 		metadata: { conversation: conv._id.toString(), textChunks, dims: embeddings.dims },
 	});
diff --git a/src/lib/types/PdfChat.ts b/src/lib/types/PdfChat.ts
index 9294ad8ba05..3595920a07c 100644
--- a/src/lib/types/PdfChat.ts
+++ b/src/lib/types/PdfChat.ts
@@ -12,3 +12,8 @@ export enum PdfUploadStatus {
 	Uploading = "Uploading",
 	Uploaded = "Uploaded",
 }
+
+export interface PdfUpload {
+	status: PdfUploadStatus;
+	name: string;
+}
diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte
index 123f00800da..675db4b349c 100644
--- a/src/routes/conversation/[id]/+page.svelte
+++ b/src/routes/conversation/[id]/+page.svelte
@@ -16,7 +16,7 @@
 	import type { MessageUpdate, RAGUpdate } from "$lib/types/MessageUpdate";
 	import titleUpdate from "$lib/stores/titleUpdate";
 	import file2base64 from "$lib/utils/file2base64";
-	import { PdfUploadStatus } from "$lib/types/PdfChat.js";
+	import { PdfUploadStatus, type PdfUpload } from "$lib/types/PdfChat.js";
 	export let data;
 
 	let messages = data.messages;
@@ -32,7 +32,7 @@
 
 	let loading = false;
 	let pending = false;
-	let uploadPdfStatus: PdfUploadStatus;
+	let pdfUpload: PdfUpload | undefined = undefined;
 
 	let files: File[] = [];
 
@@ -276,7 +276,7 @@
 	}
 
 	async function uploadPdf(file: File) {
-		uploadPdfStatus = PdfUploadStatus.Uploading;
+		pdfUpload = { status: PdfUploadStatus.Uploading, name: file.name };
 
 		const formData = new FormData();
 		formData.append("pdf", file);
@@ -291,7 +291,20 @@
 			console.error("Error while uploading PDF: " + (await res.text()));
 		}
 
-		uploadPdfStatus = PdfUploadStatus.Uploaded;
+		pdfUpload.status = PdfUploadStatus.Uploaded;
+	}
+
+	async function deletePdf() {
+		const res = await fetch(`${base}/conversation/${$page.params.id}/upload-pdf`, {
+			method: "DELETE",
+		});
+
+		if (!res.ok) {
+			error.set("Error while deleting PDF, try again.");
+			console.error("Error while deleting PDF: " + (await res.text()));
+		}
+
+		pdfUpload = undefined;
 	}
 
 	onMount(async () => {
@@ -360,7 +373,8 @@
 	on:retry={onRetry}
 	on:vote={(event) => voteMessage(event.detail.score, event.detail.id)}
 	on:uploadpdf={(event) => uploadPdf(event.detail)}
-	{uploadPdfStatus}
+	on:deletepdf={deletePdf}
+	{pdfUpload}
 	on:share={() => shareConversation($page.params.id, data.title)}
 	on:stop={() => (($isAborted = true), (loading = false))}
 	models={data.models}
diff --git a/src/routes/conversation/[id]/upload-pdf/+server.ts b/src/routes/conversation/[id]/upload-pdf/+server.ts
index f88d0d1c149..99f1e7b0bd9 100644
--- a/src/routes/conversation/[id]/upload-pdf/+server.ts
+++ b/src/routes/conversation/[id]/upload-pdf/+server.ts
@@ -1,7 +1,7 @@
 import { authCondition } from "$lib/server/auth";
 import { collections } from "$lib/server/database";
 import { MAX_SEQ_LEN as CHUNK_CAR_LEN, createEmbeddings } from "$lib/server/embeddings";
-import { uploadPdfEmbeddings } from "$lib/server/files/uploadFile";
+import { deleteFile, uploadPdfEmbeddings } from "$lib/server/files/uploadFile";
 import { chunk } from "$lib/utils/chunk";
 import { error } from "@sveltejs/kit";
 import { ObjectId } from "mongodb";
@@ -41,3 +41,21 @@ export async function POST({ request, params, locals }) {
 
 	return new Response();
 }
+
+export async function DELETE({ params, locals }) {
+	const conversationId = new ObjectId(params.id);
+	const conversation = await collections.conversations.findOne({
+		_id: conversationId,
+		...authCondition(locals),
+	});
+
+	if (!conversation) {
+		throw error(404, "Conversation not found");
+	}
+
+	const filename = `${conversation._id}-pdf`;
+
+	await deleteFile(filename);
+
+	return new Response();
+}

From 2ab33a2f8ca0455978c32dcbb9f3c9cef22b306b Mon Sep 17 00:00:00 2001
From: Mishig Davaadorj <dmishig@gmail.com>
Date: Tue, 9 Jan 2024 09:53:05 +0100
Subject: [PATCH 07/11] format

---
 src/lib/components/UploadedPdfStatus.svelte | 4 ++--
 src/lib/server/files/uploadFile.ts          | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/lib/components/UploadedPdfStatus.svelte b/src/lib/components/UploadedPdfStatus.svelte
index fec70e51d7b..ac79103bedf 100644
--- a/src/lib/components/UploadedPdfStatus.svelte
+++ b/src/lib/components/UploadedPdfStatus.svelte
@@ -20,9 +20,9 @@
 	title={pdfUpload.name}
 >
 	<button
-		class="-mr-1 block md:hidden shrink-0 opacity-70 group-hover:block"
+		class="-mr-1 block shrink-0 opacity-70 group-hover:block md:hidden"
 		on:click={() => dispatch("deletepdf")}><CarbonClose /></button
 	>
 	<CarbonDocumentBlank class="shrink-0" />
-	<p class="truncate hidden md:block">{pdfUpload.name}</p>
+	<p class="hidden truncate md:block">{pdfUpload.name}</p>
 </div>
diff --git a/src/lib/server/files/uploadFile.ts b/src/lib/server/files/uploadFile.ts
index 7b2b0dbe1f4..fdb43bc23db 100644
--- a/src/lib/server/files/uploadFile.ts
+++ b/src/lib/server/files/uploadFile.ts
@@ -21,7 +21,7 @@ export async function uploadImgFile(file: Blob, conv: Conversation): Promise<str
 	});
 }
 
-export async function deleteFile(filename: string){
+export async function deleteFile(filename: string) {
 	// Step 1: Check if the file exists
 	const existingFile = await collections.files.findOne({ filename });
 

From ccbdf275aee603dcefd6e715c84ae3c59ab1f280 Mon Sep 17 00:00:00 2001
From: Mishig Davaadorj <dmishig@gmail.com>
Date: Tue, 9 Jan 2024 10:39:03 +0100
Subject: [PATCH 08/11] `ENABLE_PDF_CHAT` env var

---
 .env                                      | 2 ++
 src/lib/components/UploadBtn.svelte       | 9 ++++++---
 src/lib/components/chat/ChatWindow.svelte | 6 ++++--
 src/routes/+layout.server.ts              | 4 ++++
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/.env b/.env
index 01ee88b4cf2..a152fa844d2 100644
--- a/.env
+++ b/.env
@@ -120,6 +120,8 @@ PUBLIC_APP_DATA_SHARING=#set to 1 to enable options & text regarding data sharin
 PUBLIC_APP_DISCLAIMER=#set to 1 to show a disclaimer on login page
 LLM_SUMMERIZATION=true
 
+ENABLE_PDF_CHAT=false #set to true to enable PDF-chat feature
+
 # PUBLIC_APP_NAME=HuggingChat
 # PUBLIC_APP_ASSETS=huggingchat
 # PUBLIC_APP_COLOR=yellow
diff --git a/src/lib/components/UploadBtn.svelte b/src/lib/components/UploadBtn.svelte
index 62b87316cee..2c8005b9a0c 100644
--- a/src/lib/components/UploadBtn.svelte
+++ b/src/lib/components/UploadBtn.svelte
@@ -6,10 +6,13 @@
 
 	export let classNames = "";
 	export let multimodal = false;
+	export let pdfChat = false;
 	export let files: File[];
 	export let pdfUpload: PdfUpload | undefined = undefined;
-	const accept = multimodal ? "image/*,.pdf" : ".pdf";
-	const label = multimodal ? "Upload image or PDF" : "Upload PDF";
+
+	const accept = (multimodal && pdfChat) ? "image/*,.pdf" : multimodal ? "image/*" : ".pdf";
+	const label = (multimodal && pdfChat) ? "Upload image or PDF" :  multimodal ? "Upload image" : "Upload PDF";
+	
 	let fileInput: HTMLInputElement;
 	let interval: ReturnType<typeof setInterval>;
 
@@ -35,7 +38,7 @@
 		}
 
 		const file = fileInput.files?.[0];
-		if (file?.type === "application/pdf") {
+		if (pdfChat && file?.type === "application/pdf") {
 			// pdf upload
 			dispatch("uploadpdf", file);
 		} else if (multimodal && file?.type.startsWith("image")) {
diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte
index 673d67465a1..31001e4db49 100644
--- a/src/lib/components/chat/ChatWindow.svelte
+++ b/src/lib/components/chat/ChatWindow.svelte
@@ -178,10 +178,12 @@
 					/>
 				{/if}
 				<div class="ml-auto flex items-center gap-x-3">
-					{#if pdfUpload?.name}
+					{#if $page.data.enablePdfChat && pdfUpload?.name}
 						<UploadedPdfStatus on:deletepdf {pdfUpload} />
 					{/if}
-					<UploadBtn bind:files on:uploadpdf multimodal={currentModel.multimodal} {pdfUpload} />
+					{#if currentModel.multimodal || $page.data.enablePdfChat}
+						 <UploadBtn bind:files on:uploadpdf multimodal={currentModel.multimodal} pdfChat={$page.data.enablePdfChat} {pdfUpload} />
+					{/if}
 				</div>
 			</div>
 			<form
diff --git a/src/routes/+layout.server.ts b/src/routes/+layout.server.ts
index 9ce2c126a44..d482f031e3e 100644
--- a/src/routes/+layout.server.ts
+++ b/src/routes/+layout.server.ts
@@ -12,6 +12,7 @@ import {
 	MESSAGES_BEFORE_LOGIN,
 	YDC_API_KEY,
 	USE_LOCAL_WEBSEARCH,
+	ENABLE_PDF_CHAT,
 } from "$env/static/private";
 
 export const load: LayoutServerLoad = async ({ locals, depends }) => {
@@ -59,6 +60,8 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => {
 
 	const loginRequired = requiresUser && !locals.user && userHasExceededMessages;
 
+	const enablePdfChat = ENABLE_PDF_CHAT === "true";
+
 	return {
 		conversations: await conversations
 			.find(authCondition(locals))
@@ -127,5 +130,6 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => {
 		loginRequired,
 		loginEnabled: requiresUser,
 		guestMode: requiresUser && messagesBeforeLogin > 0,
+		enablePdfChat,
 	};
 };

From 61d3bc2ee0995f341bdd2afbf0279d95d1175425 Mon Sep 17 00:00:00 2001
From: Mishig Davaadorj <dmishig@gmail.com>
Date: Tue, 9 Jan 2024 12:13:15 +0100
Subject: [PATCH 09/11] format

---
 src/lib/components/UploadBtn.svelte         | 7 ++++---
 src/lib/components/UploadedPdfStatus.svelte | 2 +-
 src/lib/components/chat/ChatWindow.svelte   | 8 +++++++-
 src/routes/conversation/[id]/+page.svelte   | 4 ++--
 4 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/lib/components/UploadBtn.svelte b/src/lib/components/UploadBtn.svelte
index 2c8005b9a0c..12ee4ca2072 100644
--- a/src/lib/components/UploadBtn.svelte
+++ b/src/lib/components/UploadBtn.svelte
@@ -10,9 +10,10 @@
 	export let files: File[];
 	export let pdfUpload: PdfUpload | undefined = undefined;
 
-	const accept = (multimodal && pdfChat) ? "image/*,.pdf" : multimodal ? "image/*" : ".pdf";
-	const label = (multimodal && pdfChat) ? "Upload image or PDF" :  multimodal ? "Upload image" : "Upload PDF";
-	
+	const accept = multimodal && pdfChat ? "image/*,.pdf" : multimodal ? "image/*" : ".pdf";
+	const label =
+		multimodal && pdfChat ? "Upload image or PDF" : multimodal ? "Upload image" : "Upload PDF";
+
 	let fileInput: HTMLInputElement;
 	let interval: ReturnType<typeof setInterval>;
 
diff --git a/src/lib/components/UploadedPdfStatus.svelte b/src/lib/components/UploadedPdfStatus.svelte
index ac79103bedf..9b3343db4b0 100644
--- a/src/lib/components/UploadedPdfStatus.svelte
+++ b/src/lib/components/UploadedPdfStatus.svelte
@@ -14,7 +14,7 @@
 </script>
 
 <div
-	class="max-w-48 group flex items-center gap-x-1"
+	class="group flex max-w-48 items-center gap-x-1"
 	class:animate-pulse={uploading}
 	class:pointer-events-none={uploading}
 	title={pdfUpload.name}
diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte
index 31001e4db49..7e5b20a1d1c 100644
--- a/src/lib/components/chat/ChatWindow.svelte
+++ b/src/lib/components/chat/ChatWindow.svelte
@@ -182,7 +182,13 @@
 						<UploadedPdfStatus on:deletepdf {pdfUpload} />
 					{/if}
 					{#if currentModel.multimodal || $page.data.enablePdfChat}
-						 <UploadBtn bind:files on:uploadpdf multimodal={currentModel.multimodal} pdfChat={$page.data.enablePdfChat} {pdfUpload} />
+						<UploadBtn
+							bind:files
+							on:uploadpdf
+							multimodal={currentModel.multimodal}
+							pdfChat={$page.data.enablePdfChat}
+							{pdfUpload}
+						/>
 					{/if}
 				</div>
 			</div>
diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte
index 675db4b349c..13651a7654c 100644
--- a/src/routes/conversation/[id]/+page.svelte
+++ b/src/routes/conversation/[id]/+page.svelte
@@ -295,6 +295,8 @@
 	}
 
 	async function deletePdf() {
+		pdfUpload = undefined;
+
 		const res = await fetch(`${base}/conversation/${$page.params.id}/upload-pdf`, {
 			method: "DELETE",
 		});
@@ -303,8 +305,6 @@
 			error.set("Error while deleting PDF, try again.");
 			console.error("Error while deleting PDF: " + (await res.text()));
 		}
-
-		pdfUpload = undefined;
 	}
 
 	onMount(async () => {

From 8ffee0e11afc6ae997cff23bb9d5b40f02fc1b72 Mon Sep 17 00:00:00 2001
From: Mishig <mishig.davaadorj@coloradocollege.edu>
Date: Fri, 12 Jan 2024 09:19:17 +0100
Subject: [PATCH 10/11] Generlize RAG (#689)

* Generlize RAG

* wip

* fix casting
---
 src/lib/buildPrompt.ts                        |  68 ++---------
 .../components/OpenPdfSearchResults.svelte    | 114 ------------------
 ...chResults.svelte => OpenRAGResults.svelte} |  24 +++-
 src/lib/components/chat/ChatMessage.svelte    |  46 +++----
 src/lib/server/endpoints/aws/endpointAws.ts   |   3 +-
 .../endpoints/llamacpp/endpointLlamacpp.ts    |   3 +-
 .../server/endpoints/ollama/endpointOllama.ts |   3 +-
 .../server/endpoints/openai/endpointOai.ts    |   3 +-
 src/lib/server/endpoints/tgi/endpointTgi.ts   |   3 +-
 .../{pdfSearch.ts => rag/pdfchat/rag.ts}      |  47 +++++++-
 src/lib/server/rag/rag.ts                     |  36 ++++++
 .../{ => rag}/websearch/generateQuery.ts      |   2 +-
 .../server/{ => rag}/websearch/parseWeb.ts    |   0
 .../runWebSearch.ts => rag/websearch/rag.ts}  |  50 +++++++-
 .../server/{ => rag}/websearch/searchWeb.ts   |   4 +-
 .../{ => rag}/websearch/searchWebLocal.ts     |   0
 src/lib/types/Message.ts                      |   9 +-
 src/lib/types/MessageUpdate.ts                |  28 ++---
 src/lib/types/PdfChat.ts                      |   8 --
 src/lib/types/WebSearch.ts                    |  11 +-
 src/lib/types/rag.ts                          |  14 +++
 src/routes/conversation/[id]/+page.svelte     |   3 +-
 src/routes/conversation/[id]/+server.ts       |  16 ++-
 .../message/[messageId]/prompt/+server.ts     |   3 +-
 24 files changed, 216 insertions(+), 282 deletions(-)
 delete mode 100644 src/lib/components/OpenPdfSearchResults.svelte
 rename src/lib/components/{OpenWebSearchResults.svelte => OpenRAGResults.svelte} (83%)
 rename src/lib/server/{pdfSearch.ts => rag/pdfchat/rag.ts} (54%)
 create mode 100644 src/lib/server/rag/rag.ts
 rename src/lib/server/{ => rag}/websearch/generateQuery.ts (96%)
 rename src/lib/server/{ => rag}/websearch/parseWeb.ts (100%)
 rename src/lib/server/{websearch/runWebSearch.ts => rag/websearch/rag.ts} (73%)
 rename src/lib/server/{ => rag}/websearch/searchWeb.ts (96%)
 rename src/lib/server/{ => rag}/websearch/searchWebLocal.ts (100%)
 create mode 100644 src/lib/types/rag.ts

diff --git a/src/lib/buildPrompt.ts b/src/lib/buildPrompt.ts
index 27e20cbcf7c..3c5c9149a46 100644
--- a/src/lib/buildPrompt.ts
+++ b/src/lib/buildPrompt.ts
@@ -1,18 +1,18 @@
 import type { BackendModel } from "./server/models";
 import type { Message } from "./types/Message";
-import { format } from "date-fns";
-import type { WebSearch } from "./types/WebSearch";
-import type { PdfSearch } from "./types/PdfChat";
 import { downloadImgFile } from "./server/files/downloadFile";
 import type { Conversation } from "./types/Conversation";
+import RAGs from "./server/rag/rag";
+import type { RagContext } from "./types/rag";
+
+export type BuildPromptMessage = Pick<Message, "from" | "content" | "files">;
 
 interface buildPromptOptions {
-	messages: Pick<Message, "from" | "content" | "files">[];
+	messages: BuildPromptMessage[];
 	id?: Conversation["_id"];
 	model: BackendModel;
 	locals?: App.Locals;
-	webSearch?: WebSearch;
-	pdfSearch?: PdfSearch;
+	ragContext?: RagContext;
 	preprompt?: string;
 	files?: File[];
 }
@@ -20,61 +20,13 @@ interface buildPromptOptions {
 export async function buildPrompt({
 	messages,
 	model,
-	webSearch,
-	pdfSearch,
+	ragContext,
 	preprompt,
 	id,
 }: buildPromptOptions): Promise<string> {
-	if (webSearch && webSearch.context) {
-		const lastMsg = messages.slice(-1)[0];
-		const messagesWithoutLastUsrMsg = messages.slice(0, -1);
-		const previousUserMessages = messages.filter((el) => el.from === "user").slice(0, -1);
-
-		const previousQuestions =
-			previousUserMessages.length > 0
-				? `Previous questions: \n${previousUserMessages
-						.map(({ content }) => `- ${content}`)
-						.join("\n")}`
-				: "";
-		const currentDate = format(new Date(), "MMMM d, yyyy");
-		messages = [
-			...messagesWithoutLastUsrMsg,
-			{
-				from: "user",
-				content: `I searched the web using the query: ${webSearch.searchQuery}. Today is ${currentDate} and here are the results:
-				=====================
-				${webSearch.context}
-				=====================
-				${previousQuestions}
-				Answer the question: ${lastMsg.content} 
-				`,
-			},
-		];
-	} else if (pdfSearch && pdfSearch.context) {
-		const lastMsg = messages.slice(-1)[0];
-		const messagesWithoutLastUsrMsg = messages.slice(0, -1);
-		const previousUserMessages = messages.filter((el) => el.from === "user").slice(0, -1);
-
-		const previousQuestions =
-			previousUserMessages.length > 0
-				? `Previous questions: \n${previousUserMessages
-						.map(({ content }) => `- ${content}`)
-						.join("\n")}`
-				: "";
-
-		messages = [
-			...messagesWithoutLastUsrMsg,
-			{
-				from: "user",
-				content: `Below are the information I extracted from a PDF file that might be useful:
-				=====================
-				${pdfSearch.context}
-				=====================
-				${previousQuestions}
-				Answer the question: ${lastMsg.content} 
-				`,
-			},
-		];
+	if (ragContext) {
+		const { type: ragType } = ragContext;
+		messages = RAGs[ragType].buildPrompt(messages, ragContext);
 	}
 
 	// section to handle potential files input
diff --git a/src/lib/components/OpenPdfSearchResults.svelte b/src/lib/components/OpenPdfSearchResults.svelte
deleted file mode 100644
index 71a372b0813..00000000000
--- a/src/lib/components/OpenPdfSearchResults.svelte
+++ /dev/null
@@ -1,114 +0,0 @@
-<script lang="ts">
-	import type { PdfSearchUpdate } from "$lib/types/MessageUpdate";
-	import CarbonCaretRight from "~icons/carbon/caret-right";
-
-	import CarbonCheckmark from "~icons/carbon/checkmark-filled";
-	import CarbonError from "~icons/carbon/error-filled";
-
-	import EosIconsLoading from "~icons/eos-icons/loading";
-
-	export let loading = false;
-	export let classNames = "";
-	export let pdfSearchMessages: PdfSearchUpdate[] = [];
-
-	let detailsOpen: boolean;
-	let error: boolean;
-	$: error = pdfSearchMessages[pdfSearchMessages.length - 1]?.messageType === "error";
-</script>
-
-<details
-	class="flex w-fit rounded-xl border border-gray-200 bg-white shadow-sm dark:border-gray-800 dark:bg-gray-900 {classNames} max-w-full"
-	bind:open={detailsOpen}
->
-	<summary
-		class="align-center flex cursor-pointer select-none list-none py-1 pl-2.5 pr-2 align-text-top transition-all"
-	>
-		{#if error}
-			<CarbonError class="my-auto text-red-700 dark:text-red-500" />
-		{:else if loading}
-			<EosIconsLoading class="my-auto text-gray-500" />
-		{:else}
-			<CarbonCheckmark class="my-auto text-gray-500" />
-		{/if}
-		<span class="px-2 font-medium" class:text-red-700={error} class:dark:text-red-500={error}
-			>PDF search
-		</span>
-		<div class="my-auto transition-all" class:rotate-90={detailsOpen}>
-			<CarbonCaretRight />
-		</div>
-	</summary>
-
-	<div class="content px-5 pb-5 pt-4">
-		{#if pdfSearchMessages.length === 0}
-			<div class="mx-auto w-fit">
-				<EosIconsLoading class="mb-3 h-4 w-4" />
-			</div>
-		{:else}
-			<ol>
-				{#each pdfSearchMessages as message}
-					{#if message.messageType === "update"}
-						<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
-							<div class="flex items-start">
-								<div
-									class="-ml-1.5 h-3 w-3 flex-none rounded-full bg-gray-200 dark:bg-gray-600 {loading
-										? 'group-last:animate-pulse group-last:bg-gray-300 group-last:dark:bg-gray-500'
-										: ''}"
-								/>
-								<h3 class="text-md -mt-1.5 pl-2.5 text-gray-800 dark:text-gray-100">
-									{message.message}
-								</h3>
-							</div>
-							{#if message.args}
-								<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
-									{message.args}
-								</p>
-							{/if}
-						</li>
-					{:else if message.messageType === "error"}
-						<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
-							<div class="flex items-start">
-								<CarbonError
-									class="-ml-1.5 h-3 w-3 flex-none scale-110 text-red-700 dark:text-red-500"
-								/>
-								<h3 class="text-md -mt-1.5 pl-2.5 text-red-700 dark:text-red-500">
-									{message.message}
-								</h3>
-							</div>
-							{#if message.args}
-								<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
-									{message.args}
-								</p>
-							{/if}
-						</li>
-					{/if}
-				{/each}
-			</ol>
-		{/if}
-	</div>
-</details>
-
-<style>
-	@keyframes grow {
-		0% {
-			font-size: 0;
-			opacity: 0;
-		}
-		30% {
-			font-size: 1em;
-			opacity: 0;
-		}
-		100% {
-			opacity: 1;
-		}
-	}
-
-	details[open] .content {
-		animation-name: grow;
-		animation-duration: 300ms;
-		animation-delay: 0ms;
-	}
-
-	details summary::-webkit-details-marker {
-		display: none;
-	}
-</style>
diff --git a/src/lib/components/OpenWebSearchResults.svelte b/src/lib/components/OpenRAGResults.svelte
similarity index 83%
rename from src/lib/components/OpenWebSearchResults.svelte
rename to src/lib/components/OpenRAGResults.svelte
index 3e8c8190410..0111213bd11 100644
--- a/src/lib/components/OpenWebSearchResults.svelte
+++ b/src/lib/components/OpenRAGResults.svelte
@@ -1,5 +1,6 @@
 <script lang="ts">
-	import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+	import type { RAGUpdate } from "$lib/types/MessageUpdate";
+	import type { RAGType } from "$lib/types/rag";
 	import CarbonCaretRight from "~icons/carbon/caret-right";
 
 	import CarbonCheckmark from "~icons/carbon/checkmark-filled";
@@ -9,11 +10,22 @@
 
 	export let loading = false;
 	export let classNames = "";
-	export let webSearchMessages: WebSearchUpdate[] = [];
+	export let ragUpdates: RAGUpdate[] = [];
+
+	const TITLE_MAPPING: Record<RAGType, string> = {
+		webSearch: "Web search",
+		pdfChat: "PDF Chat",
+	};
 
 	let detailsOpen: boolean;
 	let error: boolean;
-	$: error = webSearchMessages[webSearchMessages.length - 1]?.messageType === "error";
+	$: error = ragUpdates[ragUpdates.length - 1]?.messageType === "error";
+
+	$: ragType = ragUpdates[0].type;
+
+	$: loading =
+		ragUpdates.length > 0 &&
+		!["sources", "done"].includes(ragUpdates[ragUpdates.length - 1].messageType);
 </script>
 
 <details
@@ -31,7 +43,7 @@
 			<CarbonCheckmark class="my-auto text-gray-500" />
 		{/if}
 		<span class="px-2 font-medium" class:text-red-700={error} class:dark:text-red-500={error}>
-			Web search
+			{TITLE_MAPPING[ragType]}
 		</span>
 		<div class="my-auto transition-all" class:rotate-90={detailsOpen}>
 			<CarbonCaretRight />
@@ -39,13 +51,13 @@
 	</summary>
 
 	<div class="content px-5 pb-5 pt-4">
-		{#if webSearchMessages.length === 0}
+		{#if ragUpdates.length === 0}
 			<div class="mx-auto w-fit">
 				<EosIconsLoading class="mb-3 h-4 w-4" />
 			</div>
 		{:else}
 			<ol>
-				{#each webSearchMessages as message}
+				{#each ragUpdates as message}
 					{#if message.messageType === "update"}
 						<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
 							<div class="flex items-start">
diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte
index 7d52b067a11..63b9f4850e4 100644
--- a/src/lib/components/chat/ChatMessage.svelte
+++ b/src/lib/components/chat/ChatMessage.svelte
@@ -16,9 +16,9 @@
 	import { PUBLIC_SEP_TOKEN } from "$lib/constants/publicSepToken";
 	import type { Model } from "$lib/types/Model";
 
-	import OpenWebSearchResults from "../OpenWebSearchResults.svelte";
-	import OpenPdfSearchResults from "../OpenPdfSearchResults.svelte";
-	import type { RAGUpdate, WebSearchUpdate, PdfSearchUpdate } from "$lib/types/MessageUpdate";
+	import OpenRAGResults from "../OpenRAGResults.svelte";
+	import type { RAGUpdate } from "$lib/types/MessageUpdate";
+	import { ragTypes } from "$lib/types/rag";
 
 	function sanitizeMd(md: string) {
 		let ret = md
@@ -61,6 +61,7 @@
 	let loadingEl: IconLoading;
 	let pendingTimeout: ReturnType<typeof setTimeout>;
 	let isCopied = false;
+	let ragIsLoading = false;
 
 	const renderer = new marked.Renderer();
 	// For code blocks with simple backticks
@@ -107,29 +108,17 @@
 		}
 	});
 
-	let searchUpdates: WebSearchUpdate[] = [];
+	let ragUpdates: RAGUpdate[] = [];
 
-	$: searchUpdates = ((RAGMessages.filter(({ type }) => type === "webSearch").length > 0
-		? RAGMessages.filter(({ type }) => type === "webSearch")
-		: message.updates?.filter(({ type }) => type === "webSearch")) ?? []) as WebSearchUpdate[];
-
-	let pdfUpdates: PdfSearchUpdate[] = [];
-
-	$: pdfUpdates = ((RAGMessages.filter(({ type }) => type === "pdfSearch").length > 0
-		? RAGMessages.filter(({ type }) => type === "pdfSearch")
-		: message.updates?.filter(({ type }) => type === "pdfSearch")) ?? []) as PdfSearchUpdate[];
+	$: ragUpdates = ((RAGMessages.length > 0
+		? RAGMessages
+		: message.updates?.filter(({ type }) => ragTypes.includes(type))) ?? []) as RAGUpdate[];
 
 	$: downloadLink =
 		message.from === "user" ? `${$page.url.pathname}/message/${message.id}/prompt` : undefined;
 
-	let webSearchIsDone = true;
-
-	$: webSearchIsDone =
-		searchUpdates.length > 0 && searchUpdates[searchUpdates.length - 1].messageType === "sources";
-
 	$: webSearchSources =
-		searchUpdates &&
-		searchUpdates?.filter(({ messageType }) => messageType === "sources")?.[0]?.sources;
+		ragUpdates && ragUpdates?.filter(({ messageType }) => messageType === "sources")?.[0]?.sources;
 
 	$: if (isCopied) {
 		setTimeout(() => {
@@ -153,21 +142,14 @@
 		<div
 			class="relative min-h-[calc(2rem+theme(spacing[3.5])*2)] min-w-[60px] break-words rounded-2xl border border-gray-100 bg-gradient-to-br from-gray-50 px-5 py-3.5 text-gray-600 prose-pre:my-2 dark:border-gray-800 dark:from-gray-800/40 dark:text-gray-300"
 		>
-			{#if searchUpdates && searchUpdates.length > 0}
-				<OpenWebSearchResults
-					classNames={tokens.length ? "mb-3.5" : ""}
-					webSearchMessages={searchUpdates}
-					loading={!(searchUpdates[searchUpdates.length - 1]?.messageType === "sources")}
-				/>
-			{/if}
-			{#if pdfUpdates && pdfUpdates.length > 0}
-				<OpenPdfSearchResults
+			{#if ragUpdates && ragUpdates.length > 0}
+				<OpenRAGResults
 					classNames={tokens.length ? "mb-3.5" : ""}
-					pdfSearchMessages={pdfUpdates}
-					loading={!(pdfUpdates[pdfUpdates.length - 1]?.messageType === "done")}
+					{ragUpdates}
+					bind:loading={ragIsLoading}
 				/>
 			{/if}
-			{#if !message.content && (webSearchIsDone || (RAGMessages && RAGMessages.length === 0))}
+			{#if !message.content && (!ragIsLoading || (RAGMessages && RAGMessages.length === 0))}
 				<IconLoading />
 			{/if}
 
diff --git a/src/lib/server/endpoints/aws/endpointAws.ts b/src/lib/server/endpoints/aws/endpointAws.ts
index 0051ceea3ec..a5696e429e3 100644
--- a/src/lib/server/endpoints/aws/endpointAws.ts
+++ b/src/lib/server/endpoints/aws/endpointAws.ts
@@ -39,8 +39,7 @@ export async function endpointAws(
 	return async ({ conversation }) => {
 		const prompt = await buildPrompt({
 			messages: conversation.messages,
-			webSearch: conversation.messages[conversation.messages.length - 1].webSearch,
-			pdfSearch: conversation.messages[conversation.messages.length - 1].pdfSearch,
+			ragContext: conversation.messages[conversation.messages.length - 1].ragContext,
 			preprompt: conversation.preprompt,
 			model,
 		});
diff --git a/src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts b/src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts
index 9d6659eb0f4..6de6c6b0568 100644
--- a/src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts
+++ b/src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts
@@ -22,8 +22,7 @@ export function endpointLlamacpp(
 	return async ({ conversation }) => {
 		const prompt = await buildPrompt({
 			messages: conversation.messages,
-			webSearch: conversation.messages[conversation.messages.length - 1].webSearch,
-			pdfSearch: conversation.messages[conversation.messages.length - 1].pdfSearch,
+			ragContext: conversation.messages[conversation.messages.length - 1].ragContext,
 			preprompt: conversation.preprompt,
 			model,
 		});
diff --git a/src/lib/server/endpoints/ollama/endpointOllama.ts b/src/lib/server/endpoints/ollama/endpointOllama.ts
index f33600d1b7a..f8e825e9e48 100644
--- a/src/lib/server/endpoints/ollama/endpointOllama.ts
+++ b/src/lib/server/endpoints/ollama/endpointOllama.ts
@@ -17,8 +17,7 @@ export function endpointOllama(input: z.input<typeof endpointOllamaParametersSch
 	return async ({ conversation }) => {
 		const prompt = await buildPrompt({
 			messages: conversation.messages,
-			webSearch: conversation.messages[conversation.messages.length - 1].webSearch,
-			pdfSearch: conversation.messages[conversation.messages.length - 1].pdfSearch,
+			ragContext: conversation.messages[conversation.messages.length - 1].ragContext,
 			preprompt: conversation.preprompt,
 			model,
 		});
diff --git a/src/lib/server/endpoints/openai/endpointOai.ts b/src/lib/server/endpoints/openai/endpointOai.ts
index ae67ee05008..f7fe3db8afc 100644
--- a/src/lib/server/endpoints/openai/endpointOai.ts
+++ b/src/lib/server/endpoints/openai/endpointOai.ts
@@ -40,8 +40,7 @@ export async function endpointOai(
 					model: model.id ?? model.name,
 					prompt: await buildPrompt({
 						messages: conversation.messages,
-						webSearch: conversation.messages[conversation.messages.length - 1].webSearch,
-						pdfSearch: conversation.messages[conversation.messages.length - 1].pdfSearch,
+						ragContext: conversation.messages[conversation.messages.length - 1].ragContext,
 						preprompt: conversation.preprompt,
 						model,
 					}),
diff --git a/src/lib/server/endpoints/tgi/endpointTgi.ts b/src/lib/server/endpoints/tgi/endpointTgi.ts
index d181c144946..64b2e903145 100644
--- a/src/lib/server/endpoints/tgi/endpointTgi.ts
+++ b/src/lib/server/endpoints/tgi/endpointTgi.ts
@@ -18,8 +18,7 @@ export function endpointTgi(input: z.input<typeof endpointTgiParametersSchema>):
 	return async ({ conversation }) => {
 		const prompt = await buildPrompt({
 			messages: conversation.messages,
-			webSearch: conversation.messages[conversation.messages.length - 1].webSearch,
-			pdfSearch: conversation.messages[conversation.messages.length - 1].pdfSearch,
+			ragContext: conversation.messages[conversation.messages.length - 1].ragContext,
 			preprompt: conversation.preprompt,
 			model,
 			id: conversation._id,
diff --git a/src/lib/server/pdfSearch.ts b/src/lib/server/rag/pdfchat/rag.ts
similarity index 54%
rename from src/lib/server/pdfSearch.ts
rename to src/lib/server/rag/pdfchat/rag.ts
index 6fdbaf8668a..5a05aad69b0 100644
--- a/src/lib/server/pdfSearch.ts
+++ b/src/lib/server/rag/pdfchat/rag.ts
@@ -1,24 +1,27 @@
-import type { PdfSearch } from "$lib/types/PdfChat";
 import { createEmbeddings, findSimilarSentences } from "$lib/server/embeddings";
 import type { Conversation } from "$lib/types/Conversation";
 import type { MessageUpdate } from "$lib/types/MessageUpdate";
-import { downloadPdfEmbeddings } from "./files/downloadFile";
+import { downloadPdfEmbeddings } from "../../files/downloadFile";
 import { Tensor } from "@xenova/transformers";
+import type { RAG } from "../rag";
+import type { RagContext } from "$lib/types/rag";
+import type { BuildPromptMessage } from "$lib/buildPrompt";
 
 // todo: embed the prompt, download the embeddings, serialize them, and find the closest sentences, and get their texts, lets go
-export async function runPdfSearch(
+async function runPdfSearch(
 	conv: Conversation,
 	prompt: string,
 	updatePad: (upd: MessageUpdate) => void
 ) {
-	const pdfSearch: PdfSearch = {
+	const pdfSearch: RagContext = {
 		context: "",
+		type: "pdfChat",
 		createdAt: new Date(),
 		updatedAt: new Date(),
 	};
 
 	function appendUpdate(message: string, args?: string[], type?: "error" | "update" | "done") {
-		updatePad({ type: "pdfSearch", messageType: type ?? "update", message: message, args: args });
+		updatePad({ type: "pdfChat", messageType: type ?? "update", message: message, args: args });
 	}
 
 	try {
@@ -50,3 +53,37 @@ export async function runPdfSearch(
 
 	return pdfSearch;
 }
+
+function buildPrompt(messages: BuildPromptMessage[], context: RagContext) {
+	const lastMsg = messages.slice(-1)[0];
+	const messagesWithoutLastUsrMsg = messages.slice(0, -1);
+	const previousUserMessages = messages.filter((el) => el.from === "user").slice(0, -1);
+
+	const previousQuestions =
+		previousUserMessages.length > 0
+			? `Previous questions: \n${previousUserMessages
+					.map(({ content }) => `- ${content}`)
+					.join("\n")}`
+			: "";
+
+	messages = [
+		...messagesWithoutLastUsrMsg,
+		{
+			from: "user",
+			content: `Below are the information I extracted from a PDF file that might be useful:
+			=====================
+			${context.context}
+			=====================
+			${previousQuestions}
+			Answer the question: ${lastMsg.content} 
+			`,
+		},
+	];
+	return messages;
+}
+
+export const ragPdfchat: RAG = {
+	type: "pdfChat",
+	retrieveRagContext: runPdfSearch,
+	buildPrompt,
+};
diff --git a/src/lib/server/rag/rag.ts b/src/lib/server/rag/rag.ts
new file mode 100644
index 00000000000..6d89693aaca
--- /dev/null
+++ b/src/lib/server/rag/rag.ts
@@ -0,0 +1,36 @@
+import type { BuildPromptMessage } from "$lib/buildPrompt";
+import type { Conversation } from "$lib/types/Conversation";
+import type { MessageUpdate } from "$lib/types/MessageUpdate";
+import type { RagContextWebSearch } from "$lib/types/WebSearch";
+import type { RAGType, RagContext } from "$lib/types/rag";
+import { ragPdfchat } from "./pdfchat/rag";
+import { ragWebsearch } from "./websearch/rag";
+
+type RetrieveRagContextFn<T extends RagContext = RagContext> = (
+	conv: Conversation,
+	prompt: string,
+	updatePad: (upd: MessageUpdate) => void
+) => Promise<T>;
+
+type BuildPromptFn<T extends RagContext = RagContext> = (
+	messages: BuildPromptMessage[],
+	context: T
+) => BuildPromptMessage[];
+
+export interface RAG<T extends RagContext = RagContext> {
+	type: RAGType;
+	retrieveRagContext: RetrieveRagContextFn<T>;
+	buildPrompt: BuildPromptFn<T>;
+}
+
+type RAGUnion = RAG<RagContext> | RAG<RagContextWebSearch>;
+
+// list of all rags
+export const RAGs: {
+	[Key in RAGType]: RAGUnion;
+} = {
+	webSearch: ragWebsearch,
+	pdfChat: ragPdfchat,
+};
+
+export default RAGs;
diff --git a/src/lib/server/websearch/generateQuery.ts b/src/lib/server/rag/websearch/generateQuery.ts
similarity index 96%
rename from src/lib/server/websearch/generateQuery.ts
rename to src/lib/server/rag/websearch/generateQuery.ts
index cdff0c7b937..d9cad9bef73 100644
--- a/src/lib/server/websearch/generateQuery.ts
+++ b/src/lib/server/rag/websearch/generateQuery.ts
@@ -1,6 +1,6 @@
 import type { Message } from "$lib/types/Message";
 import { format } from "date-fns";
-import { generateFromDefaultEndpoint } from "../generateFromDefaultEndpoint";
+import { generateFromDefaultEndpoint } from "../../generateFromDefaultEndpoint";
 import { WEBSEARCH_ALLOWLIST, WEBSEARCH_BLOCKLIST } from "$env/static/private";
 import { z } from "zod";
 import JSON5 from "json5";
diff --git a/src/lib/server/websearch/parseWeb.ts b/src/lib/server/rag/websearch/parseWeb.ts
similarity index 100%
rename from src/lib/server/websearch/parseWeb.ts
rename to src/lib/server/rag/websearch/parseWeb.ts
diff --git a/src/lib/server/websearch/runWebSearch.ts b/src/lib/server/rag/websearch/rag.ts
similarity index 73%
rename from src/lib/server/websearch/runWebSearch.ts
rename to src/lib/server/rag/websearch/rag.ts
index 76b5106fdf4..c4ceb0098fb 100644
--- a/src/lib/server/websearch/runWebSearch.ts
+++ b/src/lib/server/rag/websearch/rag.ts
@@ -1,21 +1,24 @@
-import { searchWeb } from "$lib/server/websearch/searchWeb";
+import { searchWeb } from "$lib/server/rag/websearch/searchWeb";
 import type { Message } from "$lib/types/Message";
-import type { WebSearch, WebSearchSource } from "$lib/types/WebSearch";
-import { generateQuery } from "$lib/server/websearch/generateQuery";
-import { parseWeb } from "$lib/server/websearch/parseWeb";
+import type { RagContextWebSearch, WebSearchSource } from "$lib/types/WebSearch";
+import { generateQuery } from "$lib/server/rag/websearch/generateQuery";
+import { parseWeb } from "$lib/server/rag/websearch/parseWeb";
 import { chunk } from "$lib/utils/chunk";
 import { findSimilarSentences } from "$lib/server/sentenceSimilarity";
 import type { Conversation } from "$lib/types/Conversation";
 import type { MessageUpdate } from "$lib/types/MessageUpdate";
 import { getWebSearchProvider } from "./searchWeb";
 import { defaultEmbeddingModel, embeddingModels } from "$lib/server/embeddingModels";
+import type { RAG } from "../rag";
+import type { BuildPromptMessage } from "$lib/buildPrompt";
+import { format } from "date-fns";
 
 const MAX_N_PAGES_SCRAPE = 10 as const;
 const MAX_N_PAGES_EMBED = 5 as const;
 
 const DOMAIN_BLOCKLIST = ["youtube.com", "twitter.com"];
 
-export async function runWebSearch(
+async function runWebSearch(
 	conv: Conversation,
 	prompt: string,
 	updatePad: (upd: MessageUpdate) => void
@@ -24,7 +27,8 @@ export async function runWebSearch(
 		return [...conv.messages, { content: prompt, from: "user", id: crypto.randomUUID() }];
 	})() satisfies Message[];
 
-	const webSearch: WebSearch = {
+	const webSearch: RagContextWebSearch = {
+		type: "webSearch",
 		prompt: prompt,
 		searchQuery: "",
 		results: [],
@@ -130,3 +134,37 @@ export async function runWebSearch(
 
 	return webSearch;
 }
+
+function buildPrompt(messages: BuildPromptMessage[], context: RagContextWebSearch) {
+	const lastMsg = messages.slice(-1)[0];
+	const messagesWithoutLastUsrMsg = messages.slice(0, -1);
+	const previousUserMessages = messages.filter((el) => el.from === "user").slice(0, -1);
+
+	const previousQuestions =
+		previousUserMessages.length > 0
+			? `Previous questions: \n${previousUserMessages
+					.map(({ content }) => `- ${content}`)
+					.join("\n")}`
+			: "";
+	const currentDate = format(new Date(), "MMMM d, yyyy");
+	messages = [
+		...messagesWithoutLastUsrMsg,
+		{
+			from: "user",
+			content: `I searched the web using the query: ${context.searchQuery}. Today is ${currentDate} and here are the results:
+			=====================
+			${context.context}
+			=====================
+			${previousQuestions}
+			Answer the question: ${lastMsg.content} 
+			`,
+		},
+	];
+	return messages;
+}
+
+export const ragWebsearch: RAG<RagContextWebSearch> = {
+	type: "webSearch",
+	retrieveRagContext: runWebSearch,
+	buildPrompt,
+};
diff --git a/src/lib/server/websearch/searchWeb.ts b/src/lib/server/rag/websearch/searchWeb.ts
similarity index 96%
rename from src/lib/server/websearch/searchWeb.ts
rename to src/lib/server/rag/websearch/searchWeb.ts
index 2dd7991dcef..09fa8826000 100644
--- a/src/lib/server/websearch/searchWeb.ts
+++ b/src/lib/server/rag/websearch/searchWeb.ts
@@ -1,5 +1,5 @@
-import type { YouWebSearch } from "../../types/WebSearch";
-import { WebSearchProvider } from "../../types/WebSearch";
+import type { YouWebSearch } from "../../../types/WebSearch";
+import { WebSearchProvider } from "../../../types/WebSearch";
 import {
 	SERPAPI_KEY,
 	SERPER_API_KEY,
diff --git a/src/lib/server/websearch/searchWebLocal.ts b/src/lib/server/rag/websearch/searchWebLocal.ts
similarity index 100%
rename from src/lib/server/websearch/searchWebLocal.ts
rename to src/lib/server/rag/websearch/searchWebLocal.ts
diff --git a/src/lib/types/Message.ts b/src/lib/types/Message.ts
index 164fe6085f6..d0cca04e242 100644
--- a/src/lib/types/Message.ts
+++ b/src/lib/types/Message.ts
@@ -1,16 +1,15 @@
 import type { MessageUpdate } from "./MessageUpdate";
-import type { PdfSearch } from "./PdfChat";
 import type { Timestamps } from "./Timestamps";
-import type { WebSearch } from "./WebSearch";
+import type { RagContextWebSearch } from "./WebSearch";
+import type { RagContext } from "./rag";
 
 export type Message = Partial<Timestamps> & {
 	from: "user" | "assistant";
 	id: ReturnType<typeof crypto.randomUUID>;
 	content: string;
 	updates?: MessageUpdate[];
-	webSearchId?: WebSearch["_id"]; // legacy version
-	webSearch?: WebSearch;
-	pdfSearch?: PdfSearch;
+	webSearchId?: RagContextWebSearch["_id"]; // legacy version
+	ragContext?: RagContext;
 	score?: -1 | 0 | 1;
 	files?: string[]; // can contain either the hash of the file or the b64 encoded image data on the client side when uploading
 };
diff --git a/src/lib/types/MessageUpdate.ts b/src/lib/types/MessageUpdate.ts
index 684bb4cd24c..130c664f169 100644
--- a/src/lib/types/MessageUpdate.ts
+++ b/src/lib/types/MessageUpdate.ts
@@ -1,4 +1,5 @@
 import type { WebSearchSource } from "./WebSearch";
+import type { RAGType } from "./rag";
 
 export type FinalAnswer = {
 	type: "finalAnswer";
@@ -17,22 +18,22 @@ export type AgentUpdate = {
 	binary?: Blob;
 };
 
-export type WebSearchUpdate = {
-	type: "webSearch";
-	messageType: "update" | "error" | "sources";
+export interface RAGUpdate {
+	type: RAGType;
+	messageType: "update" | "error" | "done" | string;
 	message: string;
 	args?: string[];
-	sources?: WebSearchSource[];
-};
+}
 
-export type PdfSearchUpdate = {
-	type: "pdfSearch";
-	messageType: "update" | "error" | "done";
-	message: string;
-	args?: string[];
-};
+export interface WebSearchUpdate extends RAGUpdate {
+	type: "websearch";
+	messageType: RAGUpdate["messageType"] | "sources";
+	sources?: WebSearchSource[];
+}
 
-export type RAGUpdate = WebSearchUpdate | PdfSearchUpdate;
+export interface PdfSearchUpdate extends RAGUpdate {
+	type: "pdfChat";
+}
 
 export type StatusUpdate = {
 	type: "status";
@@ -50,7 +51,6 @@ export type MessageUpdate =
 	| FinalAnswer
 	| TextStreamUpdate
 	| AgentUpdate
-	| WebSearchUpdate
-	| PdfSearchUpdate
+	| RAGUpdate
 	| StatusUpdate
 	| ErrorUpdate;
diff --git a/src/lib/types/PdfChat.ts b/src/lib/types/PdfChat.ts
index 3595920a07c..4e36a643253 100644
--- a/src/lib/types/PdfChat.ts
+++ b/src/lib/types/PdfChat.ts
@@ -1,11 +1,3 @@
-import type { ObjectId } from "mongodb";
-import type { Timestamps } from "./Timestamps";
-
-export interface PdfSearch extends Timestamps {
-	_id?: ObjectId;
-	context: string;
-}
-
 /* eslint-disable no-shadow */
 export enum PdfUploadStatus {
 	Ready = "Ready",
diff --git a/src/lib/types/WebSearch.ts b/src/lib/types/WebSearch.ts
index ad4ac744144..055114bb96a 100644
--- a/src/lib/types/WebSearch.ts
+++ b/src/lib/types/WebSearch.ts
@@ -1,16 +1,9 @@
-import type { ObjectId } from "mongodb";
-import type { Conversation } from "./Conversation";
-import type { Timestamps } from "./Timestamps";
-
-export interface WebSearch extends Timestamps {
-	_id?: ObjectId;
-	convId?: Conversation["_id"];
+import type { RagContext } from "./rag";
 
+export interface RagContextWebSearch extends RagContext {
 	prompt: string;
-
 	searchQuery: string;
 	results: WebSearchSource[];
-	context: string;
 	contextSources: WebSearchSource[];
 }
 
diff --git a/src/lib/types/rag.ts b/src/lib/types/rag.ts
new file mode 100644
index 00000000000..94eb7e7eeae
--- /dev/null
+++ b/src/lib/types/rag.ts
@@ -0,0 +1,14 @@
+import type { ObjectId } from "mongodb";
+import type { Conversation } from "./Conversation";
+import type { Timestamps } from "./Timestamps";
+
+export interface RagContext extends Timestamps {
+	_id?: ObjectId;
+	convId?: Conversation["_id"];
+	type: RAGType;
+	context: string;
+}
+
+export const ragTypes = ["webSearch", "pdfChat"];
+
+export type RAGType = (typeof ragTypes)[number];
diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte
index 13651a7654c..202517982f0 100644
--- a/src/routes/conversation/[id]/+page.svelte
+++ b/src/routes/conversation/[id]/+page.svelte
@@ -17,6 +17,7 @@
 	import titleUpdate from "$lib/stores/titleUpdate";
 	import file2base64 from "$lib/utils/file2base64";
 	import { PdfUploadStatus, type PdfUpload } from "$lib/types/PdfChat.js";
+	import { ragTypes } from "$lib/types/rag.js";
 	export let data;
 
 	let messages = data.messages;
@@ -196,7 +197,7 @@
 									lastMessage.content += update.token;
 									messages = [...messages];
 								}
-							} else if (update.type === "webSearch" || update.type === "pdfSearch") {
+							} else if (ragTypes.includes(update.type)) {
 								RAGMessages = [...RAGMessages, update];
 							} else if (update.type === "status") {
 								if (update.status === "title" && update.message) {
diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts
index e921377221d..f21e4a4c958 100644
--- a/src/routes/conversation/[id]/+server.ts
+++ b/src/routes/conversation/[id]/+server.ts
@@ -8,14 +8,12 @@ import { error } from "@sveltejs/kit";
 import { ObjectId } from "mongodb";
 import { z } from "zod";
 import type { MessageUpdate } from "$lib/types/MessageUpdate";
-import { runWebSearch } from "$lib/server/websearch/runWebSearch";
-import { runPdfSearch } from "$lib/server/pdfSearch";
-import type { WebSearch } from "$lib/types/WebSearch";
-import type { PdfSearch } from "$lib/types/PdfChat";
+import type { RagContextWebSearch } from "$lib/types/WebSearch";
 import { abortedGenerations } from "$lib/server/abortedGenerations";
 import { summarize } from "$lib/server/summarize";
 import { uploadImgFile } from "$lib/server/files/uploadFile";
 import sizeof from "image-size";
+import RAGs from "$lib/server/rag/rag";
 
 export async function POST({ request, locals, params, getClientAddress }) {
 	const id = z.string().parse(params.id);
@@ -234,21 +232,21 @@ export async function POST({ request, locals, params, getClientAddress }) {
 				}
 			);
 
-			let webSearchResults: WebSearch | undefined;
+			let webSearchResults: RagContextWebSearch | undefined;
 
 			if (webSearch) {
-				webSearchResults = await runWebSearch(conv, newPrompt, update);
+				webSearchResults = await RAGs["webSearch"].retrieveRagContext(conv, newPrompt, update);
 			}
 
-			messages[messages.length - 1].webSearch = webSearchResults;
+			messages[messages.length - 1].ragContext = webSearchResults;
 
 			let pdfSearchResults: PdfSearch | undefined;
 			const pdfSearch = await collections.files.findOne({ filename: `${convId.toString()}-pdf` });
 			if (pdfSearch) {
-				pdfSearchResults = await runPdfSearch(conv, newPrompt, update);
+				pdfSearchResults = await RAGs["pdfChat"].retrieveRagContext(conv, newPrompt, update);
 			}
 
-			messages[messages.length - 1].pdfSearch = pdfSearchResults;
+			messages[messages.length - 1].ragContext = pdfSearchResults;
 
 			conv.messages = messages;
 
diff --git a/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts b/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts
index 01cbf56e88f..fad178131da 100644
--- a/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts
+++ b/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts
@@ -38,8 +38,7 @@ export async function GET({ params, locals }) {
 
 	const prompt = await buildPrompt({
 		preprompt: conv.preprompt,
-		webSearch: messagesUpTo[messagesUpTo.length - 1].webSearch,
-		pdfSearch: messagesUpTo[messagesUpTo.length - 1].pdfSearch,
+		ragContext: messagesUpTo[messagesUpTo.length - 1].ragContext,
 		messages: messagesUpTo,
 		model: model,
 	});

From f8b2ec5ee6c78a2baa6add6449c610b92c3206ab Mon Sep 17 00:00:00 2001
From: Mishig Davaadorj <dmishig@gmail.com>
Date: Fri, 12 Jan 2024 10:33:56 +0100
Subject: [PATCH 11/11] fix typings

---
 src/lib/buildPrompt.ts                         |  3 ++-
 src/lib/components/chat/ChatMessage.svelte     |  8 +++++---
 src/lib/server/endpoints/openai/endpointOai.ts | 10 ++++++----
 src/lib/types/MessageUpdate.ts                 |  5 +++--
 src/routes/conversation/[id]/+page.svelte      |  5 ++---
 src/routes/conversation/[id]/+server.ts        | 11 ++++++++---
 6 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/src/lib/buildPrompt.ts b/src/lib/buildPrompt.ts
index 3c5c9149a46..c26eefbc96e 100644
--- a/src/lib/buildPrompt.ts
+++ b/src/lib/buildPrompt.ts
@@ -4,6 +4,7 @@ import { downloadImgFile } from "./server/files/downloadFile";
 import type { Conversation } from "./types/Conversation";
 import RAGs from "./server/rag/rag";
 import type { RagContext } from "./types/rag";
+import type { RagContextWebSearch } from "./types/WebSearch";
 
 export type BuildPromptMessage = Pick<Message, "from" | "content" | "files">;
 
@@ -26,7 +27,7 @@ export async function buildPrompt({
 }: buildPromptOptions): Promise<string> {
 	if (ragContext) {
 		const { type: ragType } = ragContext;
-		messages = RAGs[ragType].buildPrompt(messages, ragContext);
+		messages = RAGs[ragType].buildPrompt(messages, ragContext as RagContextWebSearch);
 	}
 
 	// section to handle potential files input
diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte
index 63b9f4850e4..28874eaa91c 100644
--- a/src/lib/components/chat/ChatMessage.svelte
+++ b/src/lib/components/chat/ChatMessage.svelte
@@ -17,7 +17,7 @@
 	import type { Model } from "$lib/types/Model";
 
 	import OpenRAGResults from "../OpenRAGResults.svelte";
-	import type { RAGUpdate } from "$lib/types/MessageUpdate";
+	import type { RAGUpdate, WebSearchUpdate } from "$lib/types/MessageUpdate";
 	import { ragTypes } from "$lib/types/rag";
 
 	function sanitizeMd(md: string) {
@@ -117,8 +117,10 @@
 	$: downloadLink =
 		message.from === "user" ? `${$page.url.pathname}/message/${message.id}/prompt` : undefined;
 
-	$: webSearchSources =
-		ragUpdates && ragUpdates?.filter(({ messageType }) => messageType === "sources")?.[0]?.sources;
+	$: webSearchSources = (
+		ragUpdates &&
+		(ragUpdates?.filter(({ messageType }) => messageType === "sources")?.[0] as WebSearchUpdate)
+	)?.sources;
 
 	$: if (isCopied) {
 		setTimeout(() => {
diff --git a/src/lib/server/endpoints/openai/endpointOai.ts b/src/lib/server/endpoints/openai/endpointOai.ts
index f7fe3db8afc..e0b3315ed43 100644
--- a/src/lib/server/endpoints/openai/endpointOai.ts
+++ b/src/lib/server/endpoints/openai/endpointOai.ts
@@ -5,6 +5,7 @@ import { buildPrompt } from "$lib/buildPrompt";
 import { OPENAI_API_KEY } from "$env/static/private";
 import type { Endpoint } from "../endpoints";
 import { format } from "date-fns";
+import type { RagContextWebSearch } from "$lib/types/WebSearch";
 
 export const endpointOAIParametersSchema = z.object({
 	weight: z.number().int().positive().default(1),
@@ -56,9 +57,10 @@ export async function endpointOai(
 	} else if (completion === "chat_completions") {
 		return async ({ conversation }) => {
 			let messages = conversation.messages;
-			const webSearch = conversation.messages[conversation.messages.length - 1].webSearch;
+			const ragContext = conversation.messages[conversation.messages.length - 1].ragContext;
 
-			if (webSearch && webSearch.context) {
+			if (ragContext && ragContext.type === "webSearch") {
+				const webSearchContext = ragContext as RagContextWebSearch;
 				const lastMsg = messages.slice(-1)[0];
 				const messagesWithoutLastUsrMsg = messages.slice(0, -1);
 				const previousUserMessages = messages.filter((el) => el.from === "user").slice(0, -1);
@@ -74,9 +76,9 @@ export async function endpointOai(
 					...messagesWithoutLastUsrMsg,
 					{
 						from: "user",
-						content: `I searched the web using the query: ${webSearch.searchQuery}. Today is ${currentDate} and here are the results:
+						content: `I searched the web using the query: ${webSearchContext.searchQuery}. Today is ${currentDate} and here are the results:
 						=====================
-						${webSearch.context}
+						${webSearchContext.context}
 						=====================
 						${previousQuestions}
 						Answer the question: ${lastMsg.content} 
diff --git a/src/lib/types/MessageUpdate.ts b/src/lib/types/MessageUpdate.ts
index 130c664f169..83d9e3840f7 100644
--- a/src/lib/types/MessageUpdate.ts
+++ b/src/lib/types/MessageUpdate.ts
@@ -26,7 +26,7 @@ export interface RAGUpdate {
 }
 
 export interface WebSearchUpdate extends RAGUpdate {
-	type: "websearch";
+	type: "webSearch";
 	messageType: RAGUpdate["messageType"] | "sources";
 	sources?: WebSearchSource[];
 }
@@ -51,6 +51,7 @@ export type MessageUpdate =
 	| FinalAnswer
 	| TextStreamUpdate
 	| AgentUpdate
-	| RAGUpdate
+	| WebSearchUpdate
+	| PdfSearchUpdate
 	| StatusUpdate
 	| ErrorUpdate;
diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte
index 202517982f0..c0ce7f084ec 100644
--- a/src/routes/conversation/[id]/+page.svelte
+++ b/src/routes/conversation/[id]/+page.svelte
@@ -16,8 +16,7 @@
 	import type { MessageUpdate, RAGUpdate } from "$lib/types/MessageUpdate";
 	import titleUpdate from "$lib/stores/titleUpdate";
 	import file2base64 from "$lib/utils/file2base64";
-	import { PdfUploadStatus, type PdfUpload } from "$lib/types/PdfChat.js";
-	import { ragTypes } from "$lib/types/rag.js";
+	import { PdfUploadStatus, type PdfUpload } from "$lib/types/PdfChat";
 	export let data;
 
 	let messages = data.messages;
@@ -197,7 +196,7 @@
 									lastMessage.content += update.token;
 									messages = [...messages];
 								}
-							} else if (ragTypes.includes(update.type)) {
+							} else if (update.type === "webSearch" || update.type === "pdfChat") {
 								RAGMessages = [...RAGMessages, update];
 							} else if (update.type === "status") {
 								if (update.status === "title" && update.message) {
diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts
index f21e4a4c958..49085c20e6b 100644
--- a/src/routes/conversation/[id]/+server.ts
+++ b/src/routes/conversation/[id]/+server.ts
@@ -14,6 +14,7 @@ import { summarize } from "$lib/server/summarize";
 import { uploadImgFile } from "$lib/server/files/uploadFile";
 import sizeof from "image-size";
 import RAGs from "$lib/server/rag/rag";
+import type { RagContext } from "$lib/types/rag";
 
 export async function POST({ request, locals, params, getClientAddress }) {
 	const id = z.string().parse(params.id);
@@ -235,12 +236,16 @@ export async function POST({ request, locals, params, getClientAddress }) {
 			let webSearchResults: RagContextWebSearch | undefined;
 
 			if (webSearch) {
-				webSearchResults = await RAGs["webSearch"].retrieveRagContext(conv, newPrompt, update);
+				webSearchResults = (await RAGs["webSearch"].retrieveRagContext(
+					conv,
+					newPrompt,
+					update
+				)) as RagContextWebSearch;
 			}
 
 			messages[messages.length - 1].ragContext = webSearchResults;
 
-			let pdfSearchResults: PdfSearch | undefined;
+			let pdfSearchResults: RagContext | undefined;
 			const pdfSearch = await collections.files.findOne({ filename: `${convId.toString()}-pdf` });
 			if (pdfSearch) {
 				pdfSearchResults = await RAGs["pdfChat"].retrieveRagContext(conv, newPrompt, update);
@@ -274,7 +279,7 @@ export async function POST({ request, locals, params, getClientAddress }) {
 									{
 										from: "assistant",
 										content: output.token.text.trimStart(),
-										webSearch: webSearchResults,
+										ragContext: webSearchResults,
 										updates: updates,
 										id: (responseId as Message["id"]) || crypto.randomUUID(),
 										createdAt: new Date(),