Merge pull request #71 from fractalego/llm-webserving

Llm webserving

todo.txt

@@ -1,5 +1,23 @@
 ### TODO
 
+**** what to do with conversational collapse?
+  - the system just repeats the last utterance
+    - how much is 2+2, what is the real name of bon jovi, how tall is mt everest
+    - the collapse is due to <execute>NUMBER</execute> being returing unknown (execute becomes more likely after one prior <execute>)
+    - the system is also more likely to return unknown after one unknown. Select the answer that has no unknowns?
+
+* solve math expression execute (import do not work in eval and exec needs a print on stdout)
+
+* add a memory that the execute command was called/not called.
+
+* no more than one rule (two rules it already gets confused)
+* better ui
+* better documentation
+* better tests
+# the dimension of the sentence emn model 384 should be in config
+
+
+
 * multi-step interactions for function execution are still hard.
   - perhaps the rules need to stay in the prior discourse for longer
       - the same rule appears after the first execution, therefore the bot thinks it has already executed it

wafl/answerer/dialogue_answerer.py

@@ -158,6 +158,12 @@ async def _run_code(self, to_execute):
             if any(item + "(" in to_execute for item in self._functions):
                 result = eval(f"self._module.{to_execute}")
 
+            else:
+                ldict = {}
+                exec(to_execute, globals(), ldict)
+                if "result" in ldict:
+                    result = str(ldict["result"])
+
         except Exception as e:
             traceback.print_exc()
             result = f"Error in the code to execute: {str(e)}"

wafl/connectors/local/local_llm_connector.py

@@ -43,7 +43,7 @@ def __init__(self, config, last_strings=None):
 
 
         if not last_strings:
-            self._last_strings = ["\nuser:", "\nbot:", "<|EOS|>", "</remember>", "</execute>\n", "</execute>\n", "</s>"]
+            self._last_strings = ["\nuser", "\nbot", "<|EOS|>", "</remember>", "</execute>\n", "</execute>\n", "</s>"]
 
         else:
             self._last_strings = last_strings

wafl/connectors/remote/remote_llm_connector.py

@@ -1,6 +1,5 @@
 import aiohttp
 import asyncio
-import time
 import re
 
 from wafl.connectors.utils import select_best_answer
@@ -17,7 +16,7 @@ def __init__(self, config, last_strings=None):
         port = config["remote_model"]["model_port"]
         self._server_url = f"https://{host}:{port}/predictions/bot"
         if not last_strings:
-            self._last_strings = ["\nuser:", "\nbot:", "<|EOS|>", "</remember>", "</execute>\n", "</execute>\n", "</s>"]
+            self._last_strings = ["\nuser", "\nbot", "<|EOS|>", "</remember>", "</execute>\n", "</execute>\n", "</s>"]
 
         else:
             self._last_strings = last_strings
@@ -59,10 +58,7 @@ async def predict(self, prompt: str, temperature=None, num_tokens=None) -> str:
         return "UNKNOWN"
 
     async def generate(self, prompt: str) -> str:
-        print(__name__)
-        start_time = time.time()
         if prompt in self._cache:
-            print(time.time() - start_time)
             return self._cache[prompt]
 
         text = prompt
@@ -71,7 +67,7 @@ async def generate(self, prompt: str) -> str:
             all(item not in text[start:] for item in self._last_strings)
             and len(text) < start + self._max_reply_length
         ):
-            text += await self.predict(text)
+            text += await self.predict(text) + " "
 
         end_set = set()
         for item in self._last_strings:
@@ -91,8 +87,6 @@ async def generate(self, prompt: str) -> str:
         candidate_answer = re.sub(r"(.*)<\|.*\|>", r"\1", candidate_answer).strip()
 
         self._cache[prompt] = candidate_answer
-
-        print(time.time() - start_time)
         if not candidate_answer:
             candidate_answer = "unknown"
 

wafl/connectors/remote/remote_sentence_embedder_connector.py

@@ -1,5 +1,3 @@
-import time
-
 import aiohttp
 import asyncio
 import json

wafl/connectors/utils.py

@@ -1,3 +1,3 @@
 def select_best_answer(answers):
-    special_words = ["</remember>", "</execute>"]
+    special_words = ["</remember>", "</execute>", "result ="]
     return sorted(answers, key=lambda x: sum([x.count(word) for word in special_words]))[-1]

wafl/knowledge/single_file_knowledge.py

@@ -10,14 +10,11 @@
 from wafl.knowledge.base_knowledge import BaseKnowledge
 from wafl.knowledge.utils import (
     text_is_exact_string,
-    rules_are_too_different,
     get_first_cluster_of_rules,
     filter_out_rules_that_are_too_dissimilar_to_query,
-    filter_out_rules_through_entailment,
 )
 from wafl.parsing.line_rules_parser import parse_rule_from_single_line
 from wafl.parsing.rules_parser import get_facts_and_rules_from_text
-from wafl.extractors.entailer import Entailer
 from wafl.extractors.dataclasses import Query
 from wafl.retriever.string_retriever import StringRetriever
 from wafl.retriever.dense_retriever import DenseRetriever
@@ -45,10 +42,9 @@ def __init__(self, config, rules_text=None, knowledge_name=None, logger=None):
         self._rules_dict = {}
         self._facts_retriever = DenseRetriever("text_embedding_model", config)
         self._facts_retriever_for_questions = DenseRetriever(
-            "qa_embedding_model",
+            "text_embedding_model",
             config,
         )
-        self._entailer = Entailer(config, logger)
         self._rules_incomplete_retriever = DenseRetriever(
             "text_embedding_model", config
         )

wafl/retriever/dense_retriever.py

@@ -18,7 +18,7 @@ def __init__(self, model_name, config):
         self._connector = SentenceEmbedderConnectorFactory.get_connector(
             model_name, config
         )
-        self._embeddings_model = KeyedVectors(768)
+        self._embeddings_model = KeyedVectors(384)
 
     async def add_text_and_index(self, text: str, index: str):
         embeddings = await self._get_embeddings_from_text(text)