Merge pull request #184 from SkanderHellal/master

Fix issue #183: OpenChat API Non-Deterministic Behavior

Author: imoneoi

ochat/serving/openai_api_protocol.py

@@ -60,6 +60,7 @@ class ChatCompletionRequest(BaseModel):
     top_p: Optional[float] = 1.0
     n: Optional[int] = 1
     max_tokens: Optional[int] = None
+    seed: Optional[int] = None
     stop: Optional[Union[str, List[str]]] = None
     stream: Optional[bool] = False
     presence_penalty: Optional[float] = 0.0

ochat/serving/openai_api_server.py

@@ -177,6 +177,7 @@ async def create_chat_completion(raw_request: Request, background_tasks: Backgro
             temperature=request.temperature,
             top_p=request.top_p,
             max_tokens=request.max_tokens,
+            seed=request.seed,
             # Override stop tokens
             stop_token_ids=model.eot_tokens,
             ignore_eos=True