fix: increase gpu utilization

jitsi · Oct 10, 2024 · 5f4f694 · 5f4f694
1 parent 63513e8
commit 5f4f694
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/run.sh b/run.sh
@@ -9,5 +9,5 @@ else
   cd ..
 fi
 
-export LLAMA_N_CTX=16000
+export LLAMA_N_CTX=44000
 poetry run python -m uvicorn skynet.main:app --reload
diff --git a/skynet/modules/ttt/openai_api/app.py b/skynet/modules/ttt/openai_api/app.py
@@ -31,7 +31,7 @@ def initialize():
             f'python -m {openai_api_server_path} \
                 --disable-log-requests \
                 --model {llama_path} \
-                --gpu_memory_utilization 0.95 \
+                --gpu_memory_utilization 0.99 \
                 --max-model-len {llama_n_ctx} \
                 --port {openai_api_server_port}'.split(),
             shell=False,