From ff588533e9d06297d956c48f1db58404168a6978 Mon Sep 17 00:00:00 2001 From: Alberto Cetoli Date: Sat, 23 Dec 2023 12:28:05 +0000 Subject: [PATCH 1/5] installing sphinx theme --- README.md | 10 +++------- .../build/doctrees/environment.pickle | Bin 98168 -> 98168 bytes documentation/source/requirements.txt | 2 ++ 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 099aeae2..04672d88 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# WAFL 0.0.70 [![Tests](https://github.com/fractalego/wafl/actions/workflows/development-tests1.yml/badge.svg)](https://github.com/fractalego/wafl/actions/workflows/development-tests1.yml)[![Docs](https://readthedocs.org/projects/wafl/badge/?version=latest)](https://wafl.readthedocs.io/en/latest/) +# WAFL 0.0.80 [![Tests](https://github.com/fractalego/wafl/actions/workflows/development-tests1.yml/badge.svg)](https://github.com/fractalego/wafl/actions/workflows/development-tests1.yml)[![Docs](https://readthedocs.org/projects/wafl/badge/?version=latest)](https://wafl.readthedocs.io/en/latest/) Introduction ============ @@ -46,12 +46,8 @@ Please see the examples in the following chapters. ## LLM side (needs a GPU) -The second part is a machine that runs on a machine accessible from the interface side. -The initial configuration is for a local deployment of language models. -No action is needed to run WAFL if you want to run it as a local instance. - -However, a multi-user setup will benefit for a dedicated server. -In this case, a docker image can be used +The second part (LLM side) is a model server for the speech-to-text model, the LLM, the embedding system, and the text-to-speech model. +A docker image can be used to run it as in the following: ```bash $ docker run -p8080:8080 --env NVIDIA_DISABLE_REQUIRE=1 --gpus all fractalego/wafl-llm:0.80 diff --git a/documentation/build/doctrees/environment.pickle b/documentation/build/doctrees/environment.pickle index b3797a5017d866a446f798fdf6a77a0d259335cc..ca42bd9e21c8e1b8670c9d486169a79719c25e61 100644 GIT binary patch delta 1726 zcmY*ZZA@Eb6z0H+4B4QH7O)M&v7s)yEzaq-KO+)aC?A8#I)%bYX}kNbT)x^;DNsxAG)|di!RF&ogZ0DOctYwld&agvdj%V=Y_7}56|88#C^T}%R7oX z6!C?_zKGJN&P=WI_o+c86djaWU$on7>lIarD88^VvU0|vs6BzW{;j=Sf6MF0pDIw) z$Y5Vj)UQw4i}M}{EB=5w6fjyGIXTIAc2<_N`nqra__=E~5zDB5o)eLW>a$JPL?Wa7 zKZucxx=d*(i0NliPUEZgAVAEh^4?#gf0S|=N4m?3^^M&Q<816s4kQeH zB(;X1COMEzaDFn#;CSaLi#~hPWlU)0RWNLvN&V}9aq_D-e!uF>*N?^=`rhs>M)dkt zt8TgHG8S%@7Q(1}u<_oU9mbgAjxgBGE*q4thbS$$%!`}AUJOazG&k*rJp~X)UoDK` zkQ+AYKeZR3)d{uuw+lAntyZYO51p_VJx-EKP8h~uJ+adrV8?kEZ4A`GX6)<)8*Z;7 zd8n35maZ1#xF2$Hj+cI>tNQWlg}9-feE*>9`no^M@MBiL&~YqqLkJh^h=pzNG=A=; zjYI><2fa{+dm6}NiE6{s4IpGP#Jh!ASuz#kJ!YH;ah4gUL43-LlOVoe#wifrFyjP> zADQv=#jnhG^5PD&QDzUA@xVoX5&hv|3mY>Yw5Tk?V~wyLw{^hFc%>0inC*p1Jktc1 z@X`_3iKm(YaW+hAo_&yln3>lCF~9{c^l>v4{IU}&&5`|32u!W!2d_cv3VqN9LBR1= zaNvXozQct!uwuUok7Mj0c_qxM*WuA@UVaQ~+F=2Y1VQCV^k8*}Ufk@ZD(-hLSFsD* zJE0q|`02SDolpWyt!Dlq2(Qp%K8OO0IaY zxUlsgo%kd`C$=1>xBgS%6*!`G2cZ?Zc>n?{^wc2q zEK{Td1*Rhce!vsGD^=oR6nyVTsHulyaJ!9vV%#88M8FYp=g9oHvt@W_k&{D<92-)c z83GOr0jGsda#TohPTUre#MxnZbq)Vs>F2y9?5A%{yns96up!B-)1ugtm34CF8TszC zcyg60S*7Z7mrru0r-gf!YFwq7SE-g&YJXbUrA=E}O65r_^L8A51KAVrjI51-dqTkP z6YxJ20r!M}dqTiHA>f`6a8C%hCj{IRf+RkhfQCA5w3mw54UG89@(3UL4oIK!OKy)G@Wf|w-Tksx@`VQ1 zPste!$+9H-Ivurj_V|oZl6yQ6t=XKJIT6h;7%a6*E-mohPoJ={4K<&c%SKbGdIyW8 z)OZt%q}1JU7Ti!*Yxc6ClxjQA##3tQ0vk!GYag-U>J7O7-?DH@?Q3BZDb@J_3#HWb z33W@^5eTZs%cdZzeN(nrqZ*%BwJ+Md0ApH}^WQ9WDs0w{b)P7Nn0h5)(&odt^-cgiZVHj%6#1i)Lu)Kmm*h z`rv*^p1N?~sx>@WG(nhZ{IDNPdS?-oY=a3bSoNEGz*Gn!YU+EAz|kB&P!Hw2q!D)U zxjHz;|2PTd>W^*tyubo={AUX`|7w9Ue#wH(hqQLp0zB6ya6Y{SN= z9py?NS{}8d$0~a8IXf_s4mO=x1iMCz(!f3?MoC~_6QdNcABa%` z*ssLs_1QA9VPY%9=+W8MeEdTX&Wec9W3!5U{ze0AgD5}P4%Pg210?uuH&pN|jc|+a zKMVb0c=wv1k#CW(?l=Vr7}Ga5Lm0T*0cL(tf)akY32JDt5^p&J75d0&$OHaj8*JAv zy#meabhQ;cz;E>6rkDf1=aU{Z44*;6m|pc7WMfeHp*C272;Js8KGul+N+(_<+5-Lb z9^L#wD-`f^GVIWQ>40J&yhw-4g9UX Date: Sat, 23 Dec 2023 17:40:05 +0000 Subject: [PATCH 2/5] The rules persist until a new one comes along or until all the steps are completed --- .../build/doctrees/environment.pickle | Bin 98168 -> 98168 bytes wafl/answerer/dialogue_answerer.py | 31 ++++++++++++++++++ .../bridges/llm_chitchat_answer_bridge.py | 13 -------- wafl/knowledge/single_file_knowledge.py | 6 ++-- wafl/variables.py | 2 +- 5 files changed, 35 insertions(+), 17 deletions(-) diff --git a/documentation/build/doctrees/environment.pickle b/documentation/build/doctrees/environment.pickle index ca42bd9e21c8e1b8670c9d486169a79719c25e61..7b5e4febd17118c4c42bdac68c0069c3541a94f5 100644 GIT binary patch delta 1623 zcmY*ZT})eL817r>oRjgB;b5#)L6A+2jyR{=Zbr7ymKGR8WXcLVEtH&(atduv=~`%6 zbeYGHjpS6n3zv-`*I3uZvn;wyFBUZmFYFwW6UYq$-N64D@z2Ho4Q2C4R{l3HM4$UvEgon)GC`tjG~r3dr#_TV3+lY~&q| zNP)omR!@axvJ^dO{jBGitix7Rkcya1CVS)0(zOTm-&|#*Ikiy7;+tyE(={xbQyrIC zBB$p1*!ZT}TX2u5IaRaFCUVLVVzHbWo?&ALa`Hw6OXbwBSJ|0O^&Kfq|V-t(vs;`(&Po*nd7l|vM%H9HesE`VrOk7o)I zj-^J-2w$A_6WHG0ZPCv~ZTb=KOW6tS*?JhyPG|nJfr_oo*+1uR?uM8!T)bcMGNg#| z{QtsIhc%cwg9hl?P>r6w5X|2p7#{cd+DFSA1vS5fh$6Hv`R zvBMGmTNBDvJB;#sc4U`Nft6o#;Ek{gWp@u$@dhW#RTtFikLP#rqz|_6IVW7_r?FMv z{#OATj2{+aUDxij6ci}evW|S*K*nZfJ?ITVycLNi7vkyv4CXqG!h#2L}W{6S7 z>@#ANFZ+@hWy`)JM!B+|h*75O4`P%jTP8L}Y=s!b$hMW@g(74X#3(*iTguP3Krv3H z;S?O;cUmCB7dxSh4=b>PPq)HtzUYCVnASoYwDPZdpq6(XhYXAwTiYQDd>=JlYJ-C` zM46jCP;10HpadXpeE1f4370yl(%%L)KG6j~^4E@|OY|ge_C!0B7_YwrD?p27{Af2U zabFh}UX-Do#^1+JoW$xEJb0GdgByEc5MDkTz`cI&fl44+W^6kRfpvOb0u}g$A*_F_ zz?=H&d^si6`f+6a;f!S;B+L+_2OkotJ@Di6jkTg>{{TpZd_o0fEx@+}c&9Cjr}#F2 zrx^5aoC5Ro^2Pydc~5~VfHNE-_zvC(LN^Ly=LjAGaWaI``RphhEu=q) z{<)`;{Pb57ukigbcrHb&Q>@%#GF?pX5&KTDoq4Jj zHjd=0oBmB62JXWqJI`$mc=+PF&dM_=nrKMqb5rn)pK47P5*G7=Y7xnoWpre zzx%1Y`>DJ{*b*)BxZHl9)1?kMCj;JLUp$sS8BkS49dX)QT07#ghlazhu~X|eit6*b zyk3`I8CT=60{6J;QAPrjY5oWcI75nm)R}oxsHloxae0-gbXK0C4h@Inu_{?kA9a-J zbB@O?v3y1KPmT`--0R}od1v$~M*<`Y|eD7L%^Xlz#5zMfEo)P{{_F?^P z5y>$3H)1NomJf>1CQBB7EoL%o=e(HBF#C!)ondo(#Pr?_z44ehlVNvbBCyGpcH9xs z3~O!DAFkOCA^l*@SqLX@)GY4P&jzdY=D}x^pZ0kGg2_h5gIw_I3khrTw3GxY3G^4D9-+u^hn zTluZ%IsH3uO<5s?c1pvG{b0eRHoEwU^ZEv$3U{>=jdYMm>Q*VvxM2$} zw!<|{+32X?_bPn<5PXS$vyitOp2TZ5Dl^dzb(k81N<7d(&wZ4O7dt>mzr`CxCX@79 zyv>Mx7MB^Z$KnGx4p@%&vaei#IzVfh{A{=|UG=N4FiS@w`GkU+4zJo?~=jKTIhN$Dzn5I|8Es zetx-!6ZY{G6`byZI%Dc66am8$&^km)$TzC};0C;Y z5>)&n04}`ZC3#;Ee(dFsfDPnEP~dg2<;Y#IndKtTA#VX4@(K`R|0%Hf6xeynWZQ|c z=e9f%#Va9rDVM(}`c+2->*+5_`?ny1^_`rgwu%X5V1Oy)gU&%{&uFT=qtRXF`GNQ{W#51)d27o(Tn> z2?d@Bg=zeA3g~-)Pe-7moy9b%rAaAGDrr(klRBD|(WHtdMKn2AO^#KQQ`JP&SdPG} MW(XOTGvI{(0aC&o7XSbN diff --git a/wafl/answerer/dialogue_answerer.py b/wafl/answerer/dialogue_answerer.py index 53dce11b..998ab0ab 100644 --- a/wafl/answerer/dialogue_answerer.py +++ b/wafl/answerer/dialogue_answerer.py @@ -18,6 +18,7 @@ class DialogueAnswerer(BaseAnswerer): def __init__(self, config, knowledge, interface, code_path, logger): + self._delete_current_rule = "" self._bridge = LLMChitChatAnswerBridge(config) self._knowledge = knowledge self._logger = logger @@ -27,6 +28,7 @@ def __init__(self, config, knowledge, interface, code_path, logger): self._max_num_past_utterances_for_rules = 0 self._prior_facts_with_timestamp = [] self._init_python_module(code_path.replace(".py", "")) + self._prior_rule_with_timestamp = None self._max_predictions = 3 async def answer(self, query_text): @@ -48,6 +50,16 @@ async def answer(self, query_text): dialogue_items = dialogue dialogue_items = sorted(dialogue_items, key=lambda x: x[0]) + if rules_texts: + last_timestamp = dialogue_items[-1][0] + self._prior_rule_with_timestamp = (last_timestamp, rules_texts) + dialogue_items = self._insert_rule_into_dialogue_items(rules_texts, last_timestamp, dialogue_items) + + elif self._prior_rule_with_timestamp: + last_timestamp = self._prior_rule_with_timestamp[0] + rules_texts = self._prior_rule_with_timestamp[1] + dialogue_items = self._insert_rule_into_dialogue_items(rules_texts, last_timestamp, dialogue_items) + last_bot_utterances = get_last_bot_utterances(dialogue_items, num_utterances=3) last_user_utterance = get_last_user_utterance(dialogue_items) dialogue_items = [item[1] for item in dialogue_items if item[0] >= start_time] @@ -76,6 +88,11 @@ async def answer(self, query_text): dialogue_items = last_user_utterance continue + if self._delete_current_rule in answer_text: + self._prior_rule_with_timestamp = None + dialogue_items += f"\n{original_answer_text}" + continue + if not memories: break @@ -131,6 +148,8 @@ async def _get_relevant_rules(self, query, max_num_rules=1): for cause_index, causes in enumerate(rule.causes): rules_text += f" {cause_index + 1}) {causes.text}\n" + rules_text += f' {len(rule.causes) + 1}) After you completed all the steps output "{self._delete_current_rule}"\n' + rules_texts.append(rules_text) await self._interface.add_fact(f"The bot remembers the rule:\n{rules_text}") @@ -196,3 +215,15 @@ async def _run_code(self, to_execute): result = f'\n"""python\n{to_execute}\n"""' return result + + def _insert_rule_into_dialogue_items(self, rules_texts, rule_timestamp, dialogue_items): + new_dialogue_items = [] + already_inserted = False + for timestamp, utterance in dialogue_items: + if not already_inserted and utterance.startswith("user:") and rule_timestamp == timestamp: + new_dialogue_items.append((rule_timestamp, f"user: I want you to follow these rules:\n{rules_texts}")) + already_inserted = True + + new_dialogue_items.append((timestamp, utterance)) + + return new_dialogue_items diff --git a/wafl/connectors/bridges/llm_chitchat_answer_bridge.py b/wafl/connectors/bridges/llm_chitchat_answer_bridge.py index 3acb086b..88449c68 100644 --- a/wafl/connectors/bridges/llm_chitchat_answer_bridge.py +++ b/wafl/connectors/bridges/llm_chitchat_answer_bridge.py @@ -15,19 +15,6 @@ async def get_answer(self, text: str, dialogue: str, query: str) -> str: return await self._connector.generate(prompt) async def _get_answer_prompt(self, text, rules_text, dialogue=None): - if rules_text: - rules_to_use = f"I want you to follow these rules:\n{rules_text.strip()}\n" - pattern = "\nuser: " - if pattern in dialogue: - last_user_position = dialogue.rfind(pattern) - before_user_dialogue, after_user_dialogue = ( - dialogue[:last_user_position], - dialogue[last_user_position + len(pattern) :], - ) - dialogue = f"{before_user_dialogue}\nuser: {rules_to_use}\nuser: {after_user_dialogue}" - else: - dialogue = f"user: {rules_to_use}\n{dialogue}" - prompt = f""" The following is a summary of a conversation. All the elements of the conversation are described briefly: diff --git a/wafl/knowledge/single_file_knowledge.py b/wafl/knowledge/single_file_knowledge.py index 8d69f1d1..66589d5b 100644 --- a/wafl/knowledge/single_file_knowledge.py +++ b/wafl/knowledge/single_file_knowledge.py @@ -31,9 +31,9 @@ class SingleFileKnowledge(BaseKnowledge): _threshold_for_questions_from_bot = 0.6 _threshold_for_questions_in_rules = 0.49 _threshold_for_facts = 0.4 - _threshold_for_fact_rules = 0.22 - _threshold_for_fact_rules_for_creation = 0.1 - _threshold_for_partial_facts = 0.48 + _threshold_for_fact_rules = 0.6 + _threshold_for_fact_rules_for_creation = 0.6 + _threshold_for_partial_facts = 0.6 _max_rules_per_type = 3 def __init__(self, config, rules_text=None, knowledge_name=None, logger=None): diff --git a/wafl/variables.py b/wafl/variables.py index 0a5126b3..47a4ebed 100644 --- a/wafl/variables.py +++ b/wafl/variables.py @@ -1,4 +1,4 @@ def get_variables(): return { - "version": "0.0.80", + "version": "0.0.81", } From 907243bd9fe28695c5d33b247bf5f84d3910081c Mon Sep 17 00:00:00 2001 From: Alberto Cetoli Date: Sat, 23 Dec 2023 17:40:48 +0000 Subject: [PATCH 3/5] updated tests config.json --- tests/config.json | 38 ++++++++------------------------------ 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/tests/config.json b/tests/config.json index 2dfa3b8c..4bb4fa90 100644 --- a/tests/config.json +++ b/tests/config.json @@ -1,48 +1,26 @@ { - "allow_interruptions": true, "waking_up_word": "computer", "waking_up_sound": true, "deactivate_sound": true, - "improvise_tasks": false, "rules": "rules.yaml", "functions": "functions.py", "llm_model": { - "model_is_local": false, - "local_model": "mistralai/Mistral-7B-Instruct-v0.1", - "remote_model": { - "model_host": "localhost", - "model_port": 8080 - } + "model_host": "localhost", + "model_port": 8080 }, "listener_model": { - "model_is_local": false, - "local_model": "fractalego/personal-whisper-distilled-model", - "remote_model": { - "model_host": "localhost", - "model_port": 8080 - }, + "model_host": "localhost", + "model_port": 8080, "listener_hotword_logp": -8, "listener_volume_threshold": 0.6, "listener_silence_timeout": 0.7 }, "speaker_model": { - "model_is_local": false, - "local_model": "facebook/fastspeech2-en-ljspeech", - "remote_model": { - "model_host": "localhost", - "model_port": 8080 - } - }, - "entailment_model": { - "model_is_local": false, - "local_model": "MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli" + "model_host": "localhost", + "model_port": 8080 }, "text_embedding_model": { - "model_is_local": false, - "local_model": "TaylorAI/gte-tiny", - "remote_model": { - "model_host": "localhost", - "model_port": 8080 - } + "model_host": "localhost", + "model_port": 8080 } } From 6f287825a0eb1a7cdc37d63e7e3d1e8bb4ac0fa5 Mon Sep 17 00:00:00 2001 From: Alberto Cetoli Date: Sat, 23 Dec 2023 17:42:35 +0000 Subject: [PATCH 4/5] removed listener model name from tests --- tests/test_connection.py | 1 - tests/test_voice.py | 8 -------- wafl/interface/voice_interface.py | 1 - 3 files changed, 10 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index dd2df36e..532d1265 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -8,7 +8,6 @@ from wafl.config import Configuration from wafl.connectors.bridges.llm_chitchat_answer_bridge import LLMChitChatAnswerBridge -from wafl.connectors.local.local_llm_connector import LocalLLMConnector from wafl.connectors.remote.remote_llm_connector import RemoteLLMConnector from wafl.listener.whisper_listener import WhisperListener from wafl.speaker.fairseq_speaker import FairSeqSpeaker diff --git a/tests/test_voice.py b/tests/test_voice.py index a4abbd12..6a95cba8 100644 --- a/tests/test_voice.py +++ b/tests/test_voice.py @@ -68,14 +68,6 @@ def test__random_sounds_are_excluded(self): expected = "[unclear]" assert result == expected - def test__voice_interface_receives_config(self): - config = Configuration.load_local_config() - interface = VoiceInterface(config) - assert ( - interface.listener_model_name - == config.get_value("listener_model")["local_model"] - ) - def test__hotword_listener_activated_using_recording_of_hotword(self): f = wave.open(os.path.join(_path, "data/computer.wav"), "rb") waveform = np.frombuffer(f.readframes(f.getnframes()), dtype=np.int16) / 32768 diff --git a/wafl/interface/voice_interface.py b/wafl/interface/voice_interface.py index 44caea9a..ce9d149c 100644 --- a/wafl/interface/voice_interface.py +++ b/wafl/interface/voice_interface.py @@ -28,7 +28,6 @@ def __init__(self, config, output_filter=None): self._deactivation_sound_filename = self.__get_deactivation_sound_from_config( config ) - self.listener_model_name = config.get_value("listener_model")["local_model"] self._speaker = FairSeqSpeaker(config) self._listener = WhisperListener(config) self._listener.set_timeout( From 42a3cd2199d814cd4a395a9f82c8d7476400518d Mon Sep 17 00:00:00 2001 From: Alberto Cetoli Date: Sat, 23 Dec 2023 17:43:39 +0000 Subject: [PATCH 5/5] removed local connectors tests --- tests/test_connection.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index 532d1265..d81daf20 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1,15 +1,10 @@ import asyncio import os -import wave from unittest import TestCase - -import numpy as np - from wafl.config import Configuration from wafl.connectors.bridges.llm_chitchat_answer_bridge import LLMChitChatAnswerBridge from wafl.connectors.remote.remote_llm_connector import RemoteLLMConnector -from wafl.listener.whisper_listener import WhisperListener from wafl.speaker.fairseq_speaker import FairSeqSpeaker _path = os.path.dirname(__file__) @@ -51,33 +46,3 @@ def test__connection_to_generative_model_can_generate_a_python_list(self): prediction = asyncio.run(connector.predict(prompt)) print(prediction) assert len(prediction) > 0 - - def test__local_llm_connector_can_generate_a_python_list(self): - config = Configuration.load_from_filename("local_config.json") - connector = LocalLLMConnector(config.get_value("llm_model")) - connector._num_prediction_tokens = 200 - prompt = "Generate a list of 4 chapters names for a space opera book. The output needs to be a python list of strings: " - prediction = asyncio.run(connector.predict(prompt)) - print(prediction) - assert len(prediction) > 0 - - def test__chit_chat_bridge_can_run_locally(self): - config = Configuration.load_from_filename("local_config.json") - dialogue_bridge = LLMChitChatAnswerBridge(config) - answer = asyncio.run(dialogue_bridge.get_answer("", "", "bot: hello")) - assert len(answer) > 0 - - def test__listener_local_connector(self): - config = Configuration.load_from_filename("local_config.json") - listener = WhisperListener(config) - f = wave.open(os.path.join(_path, "data/1002.wav"), "rb") - waveform = np.frombuffer(f.readframes(f.getnframes()), dtype=np.int16) / 32768 - result = asyncio.run(listener.input_waveform(waveform)) - expected = "DELETE BATTERIES FROM THE GROCERY LIST" - assert expected.lower() in result - - def test__speaker_local_connector(self): - config = Configuration.load_from_filename("local_config.json") - speaker = FairSeqSpeaker(config) - text = "Hello world" - asyncio.run(speaker.speak(text))