From 3b3e1325752a160c020c90db2e4076d556f3062f Mon Sep 17 00:00:00 2001
From: Uche Ogbuji <uche@ogbuji.net>
Date: Fri, 15 Mar 2024 18:53:21 -0600
Subject: [PATCH] Skeleton

---
 pylib/llm_wrapper.py | 49 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/pylib/llm_wrapper.py b/pylib/llm_wrapper.py
index 2f49a78..8e18fc5 100644
--- a/pylib/llm_wrapper.py
+++ b/pylib/llm_wrapper.py
@@ -475,6 +475,8 @@ def first_choice_message(response):
 {repr(response)}''')
 
 
+# FIXME: Move the non-HTTP ones to a different file, since they have so little code in common
+
 class ctransformer:
     '''
     ctransformers wrapper for LLMs
@@ -520,6 +522,7 @@ def __init__(self, model=None, **kwargs):
         self.model = model
         self.parameters = kwargs
 
+    # FIXME: Should be async, no?
     def __call__(self, prompt, **kwargs):
         '''
         Invoke the LLM with a completion request
@@ -537,6 +540,52 @@ def __call__(self, prompt, **kwargs):
         return self.model(prompt, **kwargs)
 
 
+class llama_cpp_cmdline:
+    '''
+    Command line wrapper for llama.cpp (runs the main llama.cpp executable as a subprocess)
+    '''
+    def __init__(self, cmd=None, **kwargs):
+        '''
+        Args:
+            model (XYZ): Name of the model being wrapped
+
+            kwargs (dict, optional): Extra parameters for the API, the model, etc.
+
+        '''
+        import subprocess  # noqa: F401
+        cmd = cmd or ['main']
+        assert isinstance(cmd, list), 'cmd must be a list'
+
+        self.parameters = kwargs
+
+    async def __call__(self, prompt, **kwargs):
+        '''
+        Invoke the LLM with a completion request
+
+        Args:
+            prompt (str): Prompt to send to the LLM
+
+            kwargs (dict, optional): Extra parameters to pass to the model via API
+
+        Returns:
+            dict: JSON response from the LLM
+        '''
+        # https://docs.python.org/3/library/asyncio-subprocess.html
+        proc = await asyncio.create_subprocess_shell(self.cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE)
+
+        # Probably use proc.wait(), or wait_for() if timeout parameter is needed
+        stdout, stderr = await proc.communicate()
+        # Line by line read example: https://docs.python.org/3/library/asyncio-subprocess.html#asyncio-subprocess-threads
+
+        # print(f'[{self.cmd!r} exited with {proc.returncode}]')
+        # if stdout:
+        #     print(f'[stdout]\n{stdout.decode()}')
+        # if stderr:
+        #     print(f'[stderr]\n{stderr.decode()}')
+
+
 def prompt_to_chat(prompt, system=None):
     '''
     Convert a prompt string to a chat-style message list