diff --git a/wafl/connectors/remote/remote_whisper_connector.py b/wafl/connectors/remote/remote_whisper_connector.py index 3b7a8cba..8a1a8465 100644 --- a/wafl/connectors/remote/remote_whisper_connector.py +++ b/wafl/connectors/remote/remote_whisper_connector.py @@ -38,6 +38,9 @@ async def predict(self, waveform, hotword=None) -> Dict[str, float]: async with session.post(self._server_url, json=payload) as response: data = await response.text() prediction = json.loads(data) + if "transcription" not in prediction: + raise RuntimeError("No transcription found in prediction. Is your microphone working?") + transcription = prediction["transcription"] score = prediction["score"] logp = prediction["logp"] diff --git a/wafl/events/conversation_events.py b/wafl/events/conversation_events.py index cf38856b..e00572f1 100644 --- a/wafl/events/conversation_events.py +++ b/wafl/events/conversation_events.py @@ -1,5 +1,6 @@ import os import re +import traceback from wafl.events.answerer_creator import create_answerer from wafl.simple_text_processing.normalize import normalized @@ -59,6 +60,7 @@ async def _process_query(self, text: str): if ( not text_is_question + and self._interface.get_utterances_list() and self._interface.get_utterances_list()[-1].find("user:") == 0 ): await self._interface.output("I don't know what to reply") @@ -108,7 +110,7 @@ def reload_knowledge(self): def reset_discourse_memory(self): self._answerer = create_answerer( - self._config, self._knowledge, self._interface, logger + self._config, self._knowledge, self._interface, self._logger ) def _activation_word_in_text(self, activation_word, text): diff --git a/wafl/interface/base_interface.py b/wafl/interface/base_interface.py index 18089c28..07570ec7 100644 --- a/wafl/interface/base_interface.py +++ b/wafl/interface/base_interface.py @@ -20,6 +20,9 @@ async def input(self) -> str: def bot_has_spoken(self, to_set: bool = None): raise NotImplementedError + async def insert_input(self, text: str): + pass + def is_listening(self): return self._is_listening @@ -32,9 +35,6 @@ def deactivate(self): self._facts = [] self._utterances = [] - def add_hotwords(self, hotwords: List[str]): - raise NotImplementedError - async def add_choice(self, text): self._choices.append((time.time(), text)) await self.output(f"Making the choice: {text}", silent=True) @@ -60,8 +60,15 @@ def reset_history(self): self._choices = [] self._facts = [] + def add_hotwords(self, hotwords): + pass + def _decorate_reply(self, text: str) -> str: if not self._decorator: return text return self._decorator.extract(text, self._utterances) + + def _insert_utterance(self, speaker, text: str): + if self._utterances == [] or text != self._utterances[-1][1].replace(f"{speaker}: ", ""): + self._utterances.append((time.time(), f"{speaker}: {text}")) diff --git a/wafl/interface/list_interface.py b/wafl/interface/list_interface.py new file mode 100644 index 00000000..1230da36 --- /dev/null +++ b/wafl/interface/list_interface.py @@ -0,0 +1,54 @@ +import asyncio +from typing import List + +from wafl.interface.base_interface import BaseInterface + + +class ListInterface(BaseInterface): + def __init__(self, interfaces_list: List[BaseInterface]): + super().__init__() + self._interfaces_list = interfaces_list + self._synchronize_interfaces() + + async def output(self, text: str, silent: bool = False): + await asyncio.wait( + [interface.output(text, silent) for interface in self._interfaces_list], + return_when=asyncio.ALL_COMPLETED + ) + + async def input(self) -> str: + done, pending = await asyncio.wait( + [interface.input() for interface in self._interfaces_list], + return_when=asyncio.FIRST_COMPLETED + ) + return done.pop().result() + + async def insert_input(self, text: str): + await asyncio.wait( + [interface.insert_input(text) for interface in self._interfaces_list], + return_when=asyncio.ALL_COMPLETED + ) + + def bot_has_spoken(self, to_set: bool = None): + for interface in self._interfaces_list: + interface.bot_has_spoken(to_set) + + def activate(self): + for interface in self._interfaces_list: + interface.activate() + super().activate() + + def deactivate(self): + for interface in self._interfaces_list: + interface.deactivate() + super().deactivate() + self._synchronize_interfaces() + + + def add_hotwords(self, hotwords): + for interface in self._interfaces_list: + interface.add_hotwords(hotwords) + + def _synchronize_interfaces(self): + for interface in self._interfaces_list: + interface._utterances = self._utterances \ No newline at end of file diff --git a/wafl/interface/queue_interface.py b/wafl/interface/queue_interface.py index 08fdc247..cf14c9ca 100644 --- a/wafl/interface/queue_interface.py +++ b/wafl/interface/queue_interface.py @@ -1,5 +1,4 @@ import asyncio -import time from wafl.interface.base_interface import BaseInterface @@ -16,9 +15,8 @@ async def output(self, text: str, silent: bool = False): self.output_queue.append({"text": text, "silent": True}) return - utterance = text - self.output_queue.append({"text": utterance, "silent": False}) - self._utterances.append((time.time(), f"bot: {text}")) + self.output_queue.append({"text": text, "silent": False}) + self._insert_utterance("bot", text) self.bot_has_spoken(True) async def input(self) -> str: @@ -26,9 +24,12 @@ async def input(self) -> str: await asyncio.sleep(0.1) text = self.input_queue.pop(0) - self._utterances.append((time.time(), f"user: {text}")) + self._insert_utterance("user", text) return text + async def insert_input(self, text: str): + self.input_queue.append(text) + def bot_has_spoken(self, to_set: bool = None): if to_set != None: self._bot_has_spoken = to_set diff --git a/wafl/interface/voice_interface.py b/wafl/interface/voice_interface.py index 46e53999..b3a1c2f1 100644 --- a/wafl/interface/voice_interface.py +++ b/wafl/interface/voice_interface.py @@ -1,12 +1,10 @@ import os import random import re -import time from wafl.events.utils import remove_text_between_brackets -from wafl.simple_text_processing.deixis import from_bot_to_user from wafl.interface.base_interface import BaseInterface -from wafl.interface.utils import get_most_common_words, not_good_enough +from wafl.interface.utils import not_good_enough from wafl.listener.whisper_listener import WhisperListener from wafl.speaker.fairseq_speaker import FairSeqSpeaker from wafl.speaker.soundfile_speaker import SoundFileSpeaker @@ -42,17 +40,6 @@ def __init__(self, config): self._bot_has_spoken = False self._utterances = [] - async def add_hotwords_from_knowledge( - self, knowledge: "Knowledge", max_num_words: int = 100, count_threshold: int = 5 - ): - hotwords = get_most_common_words( - knowledge.get_facts_and_rule_as_text(), - max_num_words=max_num_words, - count_threshold=count_threshold, - ) - hotwords = [word.lower() for word in hotwords] - self._listener.add_hotwords(hotwords) - def add_hotwords(self, hotwords): self._listener.add_hotwords(hotwords) @@ -65,8 +52,8 @@ async def output(self, text: str, silent: bool = False): return self._listener.activate() - text = from_bot_to_user(text) - self._utterances.append((time.time(), f"bot: {text}")) + text = text + self._insert_utterance("bot", text) print(COLOR_START + "bot> " + text + COLOR_END) await self._speaker.speak(text) self.bot_has_spoken(True) @@ -89,7 +76,7 @@ async def input(self) -> str: print(COLOR_START + "user> " + text + COLOR_END) utterance = remove_text_between_brackets(text) if utterance.strip(): - self._utterances.append((time.time(), f"user: {text}")) + self._insert_utterance("user", text) return text diff --git a/wafl/listener/whisper_listener.py b/wafl/listener/whisper_listener.py index 4fbe75ff..f2b7770f 100644 --- a/wafl/listener/whisper_listener.py +++ b/wafl/listener/whisper_listener.py @@ -91,7 +91,12 @@ async def input(self): while True: await asyncio.sleep(0) - inp = self.stream.read(self._chunk) + try: + inp = self.stream.read(self._chunk) + except IOError: + self.activate() + inp = self.stream.read(self._chunk) + rms_val = _rms(inp) if rms_val > self._volume_threshold: waveform = self.record(start_with=inp) diff --git a/wafl/runners/routes.py b/wafl/runners/routes.py index 0cd5961b..6ea8e691 100644 --- a/wafl/runners/routes.py +++ b/wafl/runners/routes.py @@ -1,21 +1,9 @@ -import asyncio import os -import random -import sys -import threading -from flask import Flask, render_template, redirect, url_for +from flask import Flask from flask_cors import CORS -from wafl.config import Configuration -from wafl.events.conversation_events import ConversationEvents -from wafl.interface.queue_interface import QueueInterface -from wafl.logger.local_file_logger import LocalFileLogger -from wafl.scheduler.conversation_loop import ConversationLoop -from wafl.scheduler.scheduler import Scheduler -from wafl.scheduler.web_loop import WebLoop _path = os.path.dirname(__file__) -_logger = LocalFileLogger() app = Flask( __name__, static_url_path="", @@ -25,51 +13,11 @@ CORS(app) -@app.route("/create_new_instance", methods=["POST"]) -def create_new_instance(): - conversation_id = random.randint(0, sys.maxsize) - result = create_scheduler_and_webserver_loop(conversation_id) - add_new_rules(app, conversation_id, result["web_server_loop"]) - thread = threading.Thread(target=result["scheduler"].run) - thread.start() - return redirect(url_for(f"index_{conversation_id}")) - - -@app.route("/") -async def index(): - return render_template("selector.html") - - def get_app(): return app -def create_scheduler_and_webserver_loop(conversation_id): - config = Configuration.load_local_config() - interface = QueueInterface() - interface.activate() - conversation_events = ConversationEvents( - config=config, - interface=interface, - logger=_logger, - ) - conversation_loop = ConversationLoop( - interface, - conversation_events, - _logger, - activation_word="", - max_misses=-1, - deactivate_on_closed_conversation=False, - ) - asyncio.run(interface.output("Hello. How may I help you?")) - web_loop = WebLoop(interface, conversation_id, conversation_events) - return { - "scheduler": Scheduler([conversation_loop, web_loop]), - "web_server_loop": web_loop, - } - - -def add_new_rules(app, conversation_id, web_server_loop): +def add_new_rules(app: Flask, conversation_id: int, web_server_loop: "WebLoop"): app.add_url_rule( f"/{conversation_id}/", f"index_{conversation_id}", diff --git a/wafl/runners/run_web_and_audio_interface.py b/wafl/runners/run_web_and_audio_interface.py new file mode 100644 index 00000000..391d151e --- /dev/null +++ b/wafl/runners/run_web_and_audio_interface.py @@ -0,0 +1,63 @@ +import asyncio +import random +import sys +import threading + +from flask import render_template, redirect, url_for + +from wafl.interface.list_interface import ListInterface +from wafl.interface.voice_interface import VoiceInterface +from wafl.scheduler.scheduler import Scheduler +from wafl.scheduler.web_loop import WebLoop +from wafl.scheduler.conversation_loop import ConversationLoop +from wafl.logger.local_file_logger import LocalFileLogger +from wafl.events.conversation_events import ConversationEvents +from wafl.interface.queue_interface import QueueInterface +from wafl.config import Configuration +from wafl.runners.routes import get_app, add_new_rules + + +app = get_app() +_logger = LocalFileLogger() + + +def run_app(): + @app.route("/create_new_instance", methods=["POST"]) + def create_new_instance(): + conversation_id = random.randint(0, sys.maxsize) + result = create_scheduler_and_webserver_loop(conversation_id) + add_new_rules(app, conversation_id, result["web_server_loop"]) + thread = threading.Thread(target=result["scheduler"].run) + thread.start() + return redirect(url_for(f"index_{conversation_id}")) + + @app.route("/") + async def index(): + return render_template("selector.html") + + def create_scheduler_and_webserver_loop(conversation_id): + config = Configuration.load_local_config() + interface = ListInterface([VoiceInterface(config), QueueInterface()]) + interface.activate() + conversation_events = ConversationEvents( + config=config, + interface=interface, + logger=_logger, + ) + conversation_loop = ConversationLoop( + interface, + conversation_events, + _logger, + activation_word=config.get_value("waking_up_word"), + ) + web_loop = WebLoop(interface, conversation_id, conversation_events) + return { + "scheduler": Scheduler([conversation_loop, web_loop]), + "web_server_loop": web_loop, + } + + app.run(host="0.0.0.0", port=8889) + + +if __name__ == "__main__": + run_app() diff --git a/wafl/runners/run_web_interface.py b/wafl/runners/run_web_interface.py index 35814571..a9087101 100644 --- a/wafl/runners/run_web_interface.py +++ b/wafl/runners/run_web_interface.py @@ -1,9 +1,62 @@ -from wafl.runners.routes import get_app +import asyncio +import random +import sys +import threading + +from flask import render_template, redirect, url_for + +from wafl.scheduler.scheduler import Scheduler +from wafl.scheduler.web_loop import WebLoop +from wafl.scheduler.conversation_loop import ConversationLoop +from wafl.logger.local_file_logger import LocalFileLogger +from wafl.events.conversation_events import ConversationEvents +from wafl.interface.queue_interface import QueueInterface +from wafl.config import Configuration +from wafl.runners.routes import get_app, add_new_rules + app = get_app() +_logger = LocalFileLogger() def run_app(): + @app.route("/create_new_instance", methods=["POST"]) + def create_new_instance(): + conversation_id = random.randint(0, sys.maxsize) + result = create_scheduler_and_webserver_loop(conversation_id) + add_new_rules(app, conversation_id, result["web_server_loop"]) + thread = threading.Thread(target=result["scheduler"].run) + thread.start() + return redirect(url_for(f"index_{conversation_id}")) + + @app.route("/") + async def index(): + return render_template("selector.html") + + def create_scheduler_and_webserver_loop(conversation_id): + config = Configuration.load_local_config() + interface = QueueInterface() + interface.activate() + conversation_events = ConversationEvents( + config=config, + interface=interface, + logger=_logger, + ) + conversation_loop = ConversationLoop( + interface, + conversation_events, + _logger, + activation_word="", + max_misses=-1, + deactivate_on_closed_conversation=False, + ) + asyncio.run(interface.output("Hello. How may I help you?")) + web_loop = WebLoop(interface, conversation_id, conversation_events) + return { + "scheduler": Scheduler([conversation_loop, web_loop]), + "web_server_loop": web_loop, + } + app.run(host="0.0.0.0", port=8889) diff --git a/wafl/scheduler/web_loop.py b/wafl/scheduler/web_loop.py index 0d3510d0..2c16a416 100644 --- a/wafl/scheduler/web_loop.py +++ b/wafl/scheduler/web_loop.py @@ -2,7 +2,7 @@ import os from flask import render_template, request, jsonify -from wafl.interface.queue_interface import QueueInterface +from wafl.interface.base_interface import BaseInterface from wafl.logger.history_logger import HistoryLogger from wafl.scheduler.messages_creator import MessagesCreator @@ -12,7 +12,7 @@ class WebLoop: def __init__( self, - interface: QueueInterface, + interface: BaseInterface, conversation_id: int, conversation_events: "ConversationEvents", ): @@ -28,7 +28,7 @@ async def index(self): async def handle_input(self): query = request.form["query"] - self._interface.input_queue.append(query) + await self._interface.insert_input(query) return f"""