From 36b64665400c5baf89e38a05102a20cdf20d97e8 Mon Sep 17 00:00:00 2001 From: Alberto Cetoli Date: Mon, 6 May 2024 17:02:45 +0100 Subject: [PATCH 1/4] Introduced list interface --- wafl/interface/base_interface.py | 3 - wafl/interface/list_interface.py | 26 ++++++++ wafl/runners/routes.py | 56 +---------------- wafl/runners/run_web_and_audio_interface.py | 66 +++++++++++++++++++++ wafl/runners/run_web_interface.py | 55 ++++++++++++++++- wafl/scheduler/web_loop.py | 4 +- 6 files changed, 150 insertions(+), 60 deletions(-) create mode 100644 wafl/interface/list_interface.py create mode 100644 wafl/runners/run_web_and_audio_interface.py diff --git a/wafl/interface/base_interface.py b/wafl/interface/base_interface.py index 18089c28..8fba8a82 100644 --- a/wafl/interface/base_interface.py +++ b/wafl/interface/base_interface.py @@ -32,9 +32,6 @@ def deactivate(self): self._facts = [] self._utterances = [] - def add_hotwords(self, hotwords: List[str]): - raise NotImplementedError - async def add_choice(self, text): self._choices.append((time.time(), text)) await self.output(f"Making the choice: {text}", silent=True) diff --git a/wafl/interface/list_interface.py b/wafl/interface/list_interface.py new file mode 100644 index 00000000..451089a9 --- /dev/null +++ b/wafl/interface/list_interface.py @@ -0,0 +1,26 @@ +from typing import List + +from wafl.interface.base_interface import BaseInterface + + +class ListInterface(BaseInterface): + def __init__(self, interfaces_list: List[BaseInterface]): + super().__init__() + self._interfaces_list = interfaces_list + + async def output(self, text: str, silent: bool = False): + for interface in self._interfaces_list: + await interface.output(text, silent) + + async def input(self) -> str: + for interface in self._interfaces_list: + if interface.is_listening(): + text = await interface.input() + if text: + return text + + return "" + + def bot_has_spoken(self, to_set: bool = None): + for interface in self._interfaces_list: + interface.bot_has_spoken(to_set) diff --git a/wafl/runners/routes.py b/wafl/runners/routes.py index 0cd5961b..6ea8e691 100644 --- a/wafl/runners/routes.py +++ b/wafl/runners/routes.py @@ -1,21 +1,9 @@ -import asyncio import os -import random -import sys -import threading -from flask import Flask, render_template, redirect, url_for +from flask import Flask from flask_cors import CORS -from wafl.config import Configuration -from wafl.events.conversation_events import ConversationEvents -from wafl.interface.queue_interface import QueueInterface -from wafl.logger.local_file_logger import LocalFileLogger -from wafl.scheduler.conversation_loop import ConversationLoop -from wafl.scheduler.scheduler import Scheduler -from wafl.scheduler.web_loop import WebLoop _path = os.path.dirname(__file__) -_logger = LocalFileLogger() app = Flask( __name__, static_url_path="", @@ -25,51 +13,11 @@ CORS(app) -@app.route("/create_new_instance", methods=["POST"]) -def create_new_instance(): - conversation_id = random.randint(0, sys.maxsize) - result = create_scheduler_and_webserver_loop(conversation_id) - add_new_rules(app, conversation_id, result["web_server_loop"]) - thread = threading.Thread(target=result["scheduler"].run) - thread.start() - return redirect(url_for(f"index_{conversation_id}")) - - -@app.route("/") -async def index(): - return render_template("selector.html") - - def get_app(): return app -def create_scheduler_and_webserver_loop(conversation_id): - config = Configuration.load_local_config() - interface = QueueInterface() - interface.activate() - conversation_events = ConversationEvents( - config=config, - interface=interface, - logger=_logger, - ) - conversation_loop = ConversationLoop( - interface, - conversation_events, - _logger, - activation_word="", - max_misses=-1, - deactivate_on_closed_conversation=False, - ) - asyncio.run(interface.output("Hello. How may I help you?")) - web_loop = WebLoop(interface, conversation_id, conversation_events) - return { - "scheduler": Scheduler([conversation_loop, web_loop]), - "web_server_loop": web_loop, - } - - -def add_new_rules(app, conversation_id, web_server_loop): +def add_new_rules(app: Flask, conversation_id: int, web_server_loop: "WebLoop"): app.add_url_rule( f"/{conversation_id}/", f"index_{conversation_id}", diff --git a/wafl/runners/run_web_and_audio_interface.py b/wafl/runners/run_web_and_audio_interface.py new file mode 100644 index 00000000..d0041259 --- /dev/null +++ b/wafl/runners/run_web_and_audio_interface.py @@ -0,0 +1,66 @@ +import asyncio +import random +import sys +import threading + +from flask import render_template, redirect, url_for + +from wafl.interface.list_interface import ListInterface +from wafl.interface.voice_interface import VoiceInterface +from wafl.scheduler.scheduler import Scheduler +from wafl.scheduler.web_loop import WebLoop +from wafl.scheduler.conversation_loop import ConversationLoop +from wafl.logger.local_file_logger import LocalFileLogger +from wafl.events.conversation_events import ConversationEvents +from wafl.interface.queue_interface import QueueInterface +from wafl.config import Configuration +from wafl.runners.routes import get_app, add_new_rules + + +app = get_app() +_logger = LocalFileLogger() + + +def run_app(): + @app.route("/create_new_instance", methods=["POST"]) + def create_new_instance(): + conversation_id = random.randint(0, sys.maxsize) + result = create_scheduler_and_webserver_loop(conversation_id) + add_new_rules(app, conversation_id, result["web_server_loop"]) + thread = threading.Thread(target=result["scheduler"].run) + thread.start() + return redirect(url_for(f"index_{conversation_id}")) + + @app.route("/") + async def index(): + return render_template("selector.html") + + def create_scheduler_and_webserver_loop(conversation_id): + config = Configuration.load_local_config() + interface = ListInterface([VoiceInterface(config), QueueInterface()]) + interface.activate() + conversation_events = ConversationEvents( + config=config, + interface=interface, + logger=_logger, + ) + conversation_loop = ConversationLoop( + interface, + conversation_events, + _logger, + activation_word="", + max_misses=-1, + deactivate_on_closed_conversation=False, + ) + asyncio.run(interface.output("Hello. How may I help you?")) + web_loop = WebLoop(interface, conversation_id, conversation_events) + return { + "scheduler": Scheduler([conversation_loop, web_loop]), + "web_server_loop": web_loop, + } + + app.run(host="0.0.0.0", port=8889) + + +if __name__ == "__main__": + run_app() diff --git a/wafl/runners/run_web_interface.py b/wafl/runners/run_web_interface.py index 35814571..a9087101 100644 --- a/wafl/runners/run_web_interface.py +++ b/wafl/runners/run_web_interface.py @@ -1,9 +1,62 @@ -from wafl.runners.routes import get_app +import asyncio +import random +import sys +import threading + +from flask import render_template, redirect, url_for + +from wafl.scheduler.scheduler import Scheduler +from wafl.scheduler.web_loop import WebLoop +from wafl.scheduler.conversation_loop import ConversationLoop +from wafl.logger.local_file_logger import LocalFileLogger +from wafl.events.conversation_events import ConversationEvents +from wafl.interface.queue_interface import QueueInterface +from wafl.config import Configuration +from wafl.runners.routes import get_app, add_new_rules + app = get_app() +_logger = LocalFileLogger() def run_app(): + @app.route("/create_new_instance", methods=["POST"]) + def create_new_instance(): + conversation_id = random.randint(0, sys.maxsize) + result = create_scheduler_and_webserver_loop(conversation_id) + add_new_rules(app, conversation_id, result["web_server_loop"]) + thread = threading.Thread(target=result["scheduler"].run) + thread.start() + return redirect(url_for(f"index_{conversation_id}")) + + @app.route("/") + async def index(): + return render_template("selector.html") + + def create_scheduler_and_webserver_loop(conversation_id): + config = Configuration.load_local_config() + interface = QueueInterface() + interface.activate() + conversation_events = ConversationEvents( + config=config, + interface=interface, + logger=_logger, + ) + conversation_loop = ConversationLoop( + interface, + conversation_events, + _logger, + activation_word="", + max_misses=-1, + deactivate_on_closed_conversation=False, + ) + asyncio.run(interface.output("Hello. How may I help you?")) + web_loop = WebLoop(interface, conversation_id, conversation_events) + return { + "scheduler": Scheduler([conversation_loop, web_loop]), + "web_server_loop": web_loop, + } + app.run(host="0.0.0.0", port=8889) diff --git a/wafl/scheduler/web_loop.py b/wafl/scheduler/web_loop.py index 0d3510d0..8b19ee6b 100644 --- a/wafl/scheduler/web_loop.py +++ b/wafl/scheduler/web_loop.py @@ -2,7 +2,7 @@ import os from flask import render_template, request, jsonify -from wafl.interface.queue_interface import QueueInterface +from wafl.interface.base_interface import BaseInterface from wafl.logger.history_logger import HistoryLogger from wafl.scheduler.messages_creator import MessagesCreator @@ -12,7 +12,7 @@ class WebLoop: def __init__( self, - interface: QueueInterface, + interface: BaseInterface, conversation_id: int, conversation_events: "ConversationEvents", ): From 175f3ccc9a4c79839156fc6f9f61ecbe498de0b4 Mon Sep 17 00:00:00 2001 From: Alberto Cetoli Date: Tue, 7 May 2024 11:40:00 +0100 Subject: [PATCH 2/4] Created combined interface --- .../remote/remote_whisper_connector.py | 3 +++ wafl/events/conversation_events.py | 4 +++- wafl/interface/base_interface.py | 7 ++++++ wafl/interface/list_interface.py | 24 ++++++++++++------- wafl/interface/queue_interface.py | 11 +++++---- wafl/interface/voice_interface.py | 5 ++-- wafl/runners/run_web_and_audio_interface.py | 2 +- wafl/scheduler/web_loop.py | 2 +- wafl/speaker/utils.py | 6 ++++- wafl/templates/functions.py | 2 +- 10 files changed, 44 insertions(+), 22 deletions(-) diff --git a/wafl/connectors/remote/remote_whisper_connector.py b/wafl/connectors/remote/remote_whisper_connector.py index 3b7a8cba..8a1a8465 100644 --- a/wafl/connectors/remote/remote_whisper_connector.py +++ b/wafl/connectors/remote/remote_whisper_connector.py @@ -38,6 +38,9 @@ async def predict(self, waveform, hotword=None) -> Dict[str, float]: async with session.post(self._server_url, json=payload) as response: data = await response.text() prediction = json.loads(data) + if "transcription" not in prediction: + raise RuntimeError("No transcription found in prediction. Is your microphone working?") + transcription = prediction["transcription"] score = prediction["score"] logp = prediction["logp"] diff --git a/wafl/events/conversation_events.py b/wafl/events/conversation_events.py index cf38856b..e00572f1 100644 --- a/wafl/events/conversation_events.py +++ b/wafl/events/conversation_events.py @@ -1,5 +1,6 @@ import os import re +import traceback from wafl.events.answerer_creator import create_answerer from wafl.simple_text_processing.normalize import normalized @@ -59,6 +60,7 @@ async def _process_query(self, text: str): if ( not text_is_question + and self._interface.get_utterances_list() and self._interface.get_utterances_list()[-1].find("user:") == 0 ): await self._interface.output("I don't know what to reply") @@ -108,7 +110,7 @@ def reload_knowledge(self): def reset_discourse_memory(self): self._answerer = create_answerer( - self._config, self._knowledge, self._interface, logger + self._config, self._knowledge, self._interface, self._logger ) def _activation_word_in_text(self, activation_word, text): diff --git a/wafl/interface/base_interface.py b/wafl/interface/base_interface.py index 8fba8a82..941b34b6 100644 --- a/wafl/interface/base_interface.py +++ b/wafl/interface/base_interface.py @@ -20,6 +20,9 @@ async def input(self) -> str: def bot_has_spoken(self, to_set: bool = None): raise NotImplementedError + async def insert_input(self, text: str): + pass + def is_listening(self): return self._is_listening @@ -62,3 +65,7 @@ def _decorate_reply(self, text: str) -> str: return text return self._decorator.extract(text, self._utterances) + + def _insert_utterance(self, speaker, text: str): + if self._utterances == [] or text != self._utterances[-1][1].replace(f"{speaker}: ", ""): + self._utterances.append((time.time(), f"{speaker}: {text}")) diff --git a/wafl/interface/list_interface.py b/wafl/interface/list_interface.py index 451089a9..5ca1875f 100644 --- a/wafl/interface/list_interface.py +++ b/wafl/interface/list_interface.py @@ -1,3 +1,4 @@ +import asyncio from typing import List from wafl.interface.base_interface import BaseInterface @@ -7,20 +8,25 @@ class ListInterface(BaseInterface): def __init__(self, interfaces_list: List[BaseInterface]): super().__init__() self._interfaces_list = interfaces_list + self._synchronize_interfaces() async def output(self, text: str, silent: bool = False): - for interface in self._interfaces_list: - await interface.output(text, silent) + await asyncio.wait( + [interface.output(text, silent) for interface in self._interfaces_list], + return_when=asyncio.ALL_COMPLETED + ) async def input(self) -> str: - for interface in self._interfaces_list: - if interface.is_listening(): - text = await interface.input() - if text: - return text - - return "" + done, pending = await asyncio.wait( + [interface.input() for interface in self._interfaces_list], + return_when=asyncio.FIRST_COMPLETED + ) + return done.pop().result() def bot_has_spoken(self, to_set: bool = None): for interface in self._interfaces_list: interface.bot_has_spoken(to_set) + + def _synchronize_interfaces(self): + for interface in self._interfaces_list: + interface._utterances = self._utterances \ No newline at end of file diff --git a/wafl/interface/queue_interface.py b/wafl/interface/queue_interface.py index 08fdc247..cf14c9ca 100644 --- a/wafl/interface/queue_interface.py +++ b/wafl/interface/queue_interface.py @@ -1,5 +1,4 @@ import asyncio -import time from wafl.interface.base_interface import BaseInterface @@ -16,9 +15,8 @@ async def output(self, text: str, silent: bool = False): self.output_queue.append({"text": text, "silent": True}) return - utterance = text - self.output_queue.append({"text": utterance, "silent": False}) - self._utterances.append((time.time(), f"bot: {text}")) + self.output_queue.append({"text": text, "silent": False}) + self._insert_utterance("bot", text) self.bot_has_spoken(True) async def input(self) -> str: @@ -26,9 +24,12 @@ async def input(self) -> str: await asyncio.sleep(0.1) text = self.input_queue.pop(0) - self._utterances.append((time.time(), f"user: {text}")) + self._insert_utterance("user", text) return text + async def insert_input(self, text: str): + self.input_queue.append(text) + def bot_has_spoken(self, to_set: bool = None): if to_set != None: self._bot_has_spoken = to_set diff --git a/wafl/interface/voice_interface.py b/wafl/interface/voice_interface.py index 46e53999..e1a03e82 100644 --- a/wafl/interface/voice_interface.py +++ b/wafl/interface/voice_interface.py @@ -1,7 +1,6 @@ import os import random import re -import time from wafl.events.utils import remove_text_between_brackets from wafl.simple_text_processing.deixis import from_bot_to_user @@ -66,7 +65,7 @@ async def output(self, text: str, silent: bool = False): self._listener.activate() text = from_bot_to_user(text) - self._utterances.append((time.time(), f"bot: {text}")) + self._insert_utterance("bot", text) print(COLOR_START + "bot> " + text + COLOR_END) await self._speaker.speak(text) self.bot_has_spoken(True) @@ -89,7 +88,7 @@ async def input(self) -> str: print(COLOR_START + "user> " + text + COLOR_END) utterance = remove_text_between_brackets(text) if utterance.strip(): - self._utterances.append((time.time(), f"user: {text}")) + self._insert_utterance("user", text) return text diff --git a/wafl/runners/run_web_and_audio_interface.py b/wafl/runners/run_web_and_audio_interface.py index d0041259..e78ff96d 100644 --- a/wafl/runners/run_web_and_audio_interface.py +++ b/wafl/runners/run_web_and_audio_interface.py @@ -48,7 +48,7 @@ def create_scheduler_and_webserver_loop(conversation_id): interface, conversation_events, _logger, - activation_word="", + activation_word="", ### use activation word in config!! max_misses=-1, deactivate_on_closed_conversation=False, ) diff --git a/wafl/scheduler/web_loop.py b/wafl/scheduler/web_loop.py index 8b19ee6b..2c16a416 100644 --- a/wafl/scheduler/web_loop.py +++ b/wafl/scheduler/web_loop.py @@ -28,7 +28,7 @@ async def index(self): async def handle_input(self): query = request.form["query"] - self._interface.input_queue.append(query) + await self._interface.insert_input(query) return f"""