From 57edc5b036533fa51f022e9eaf38386d3b8a89fd Mon Sep 17 00:00:00 2001 From: Janaka Abeywardhana Date: Mon, 29 Apr 2024 22:20:18 +0100 Subject: [PATCH 1/5] refactor: switch setting log level to an env var. --- source/docq/config.py | 1 + source/docq/setup.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/source/docq/config.py b/source/docq/config.py index 45d0843a..7f61ebc6 100644 --- a/source/docq/config.py +++ b/source/docq/config.py @@ -4,6 +4,7 @@ ENV_VAR_DOCQ_DATA = "DOCQ_DATA" ENV_VAR_DOCQ_DEMO = "DOCQ_DEMO" +ENV_VAR_DOCQ_LOGLEVEL = "DOCQ_LOGLEVEL" ENV_VAR_OPENAI_API_KEY = "DOCQ_OPENAI_API_KEY" ENV_VAR_DOCQ_COOKIE_HMAC_SECRET_KEY = "DOCQ_COOKIE_HMAC_SECRET_KEY" ENV_VAR_DOCQ_API_SECRET = "DOCQ_API_SECRET" diff --git a/source/docq/setup.py b/source/docq/setup.py index cdd8a8c4..4fdfb924 100644 --- a/source/docq/setup.py +++ b/source/docq/setup.py @@ -1,5 +1,6 @@ """Initialize Docq.""" import logging +import os from opentelemetry import trace @@ -16,13 +17,19 @@ manage_users, services, ) +from .config import ENV_VAR_DOCQ_LOGLEVEL from .support import auth_utils, llm, store tracer = trace.get_tracer(__name__, docq.__version_str__) def _config_logging() -> None: """Configure logging.""" - logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(process)d %(levelname)s %(message)s", force=True) # force over rides Otel (or other) logging config with this. + log_level = os.environ.get(ENV_VAR_DOCQ_LOGLEVEL, "ERROR") + + logging.basicConfig( + level=log_level.upper(), format="%(asctime)s %(process)d %(levelname)s %(message)s", force=True + ) # force overrides Otel (or other) logging config with this. + #FIXME: right now this will run everytime a user hits the home page. add a global lock using st.cache to make this only run once. def init() -> None: @@ -49,3 +56,5 @@ def init() -> None: #metadata_extractors._cache_metadata_extractor_models() logging.info("Docq initialized") span.add_event("Docq initialized") + + logging.error("Docq error test message") \ No newline at end of file From a718ac4803321eee3f30f2d5afd88e5ff0091e5f Mon Sep 17 00:00:00 2001 From: Janaka Abeywardhana Date: Mon, 29 Apr 2024 22:25:57 +0100 Subject: [PATCH 2/5] refactor: clean up slack event handler add proper ack() --- .../integration/slack/slack_event_handler.py | 38 +++++++++++-------- .../slack/slack_event_handler_middleware.py | 23 +++++------ web/api/integration/slack/slack_utils.py | 3 +- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/web/api/integration/slack/slack_event_handler.py b/web/api/integration/slack/slack_event_handler.py index 5c31c055..03ffb35a 100644 --- a/web/api/integration/slack/slack_event_handler.py +++ b/web/api/integration/slack/slack_event_handler.py @@ -1,7 +1,8 @@ -"""Slack chat action handler.""" +"""Slack event handlers aka listeners.""" from docq.integrations.slack.slack_application import slack_app from opentelemetry import trace +from slack_bolt import Ack from slack_bolt.context.say import Say from web.api.integration.utils import rag_completion @@ -9,7 +10,6 @@ from .slack_event_handler_middleware import ( filter_duplicate_event_middleware, persist_message_middleware, - slack_event_tracker, ) tracer = trace.get_tracer(__name__) @@ -22,8 +22,9 @@ @slack_app.event("app_mention", middleware=[filter_duplicate_event_middleware]) @tracer.start_as_current_span(name="handle_app_mention") -def handle_app_mention_event(body: dict, say: Say) -> None: +def handle_app_mention_event(body: dict, ack: Ack, say: Say) -> None: """Handle of type app_mention. i.e. [at]botname.""" + span = trace.get_current_span() is_thread_message = False if body["event"].get("thread_ts", False): # there's a documented bug in the Slack Events API. subtype=message_replied is not being sent. This is the official workaround. @@ -34,23 +35,28 @@ def handle_app_mention_event(body: dict, say: Say) -> None: # always reply in a thread. # if thread_ts is missing it's an unthreaded message so set it as the parent in our response thread_ts = body["event"].get("thread_ts", ts) + text = body["event"]["text"] event_id = body.get("event_id") channel_id = body["event"]["channel"] - - print("slack: unthreaded message ", is_thread_message) - - response = rag_completion(body["event"]["text"], body["event"]["channel"], thread_ts) - say( - text=CHANNEL_TEMPLATE.format(user=body["event"]["user"], response=response), - channel=body["event"]["channel"], - thread_ts=thread_ts, - mrkdwn=True, - ) - # if event_id: - # slack_event_tracker.remove_event(event_id=event_id, channel_id=channel_id) + user_id = body["event"]["user"] + + try: + ack() + response = rag_completion(text=text, channel_id=channel_id, thread_ts=thread_ts) + say( + text=CHANNEL_TEMPLATE.format(user=user_id, response=response), + channel=channel_id, + thread_ts=thread_ts, + mrkdwn=True, + ) + except Exception as e: + span.record_exception(e) + span.set_status(trace.StatusCode.ERROR, str(e)) + raise e @slack_app.event("message", middleware=[persist_message_middleware]) @tracer.start_as_current_span(name="handle_message") -def handle_message(body: dict, say: Say) -> None: +def handle_message(body: dict, ack: Ack, say: Say) -> None: """Events of type message. This fires for multiple types including app_mention.""" + Ack() diff --git a/web/api/integration/slack/slack_event_handler_middleware.py b/web/api/integration/slack/slack_event_handler_middleware.py index fc15b0a3..d8b3358c 100644 --- a/web/api/integration/slack/slack_event_handler_middleware.py +++ b/web/api/integration/slack/slack_event_handler_middleware.py @@ -4,27 +4,28 @@ import docq.integrations.slack.manage_slack_messages as manage_slack_messages from opentelemetry import trace +from slack_bolt import Ack from .slack_event_tracker import SlackEventTracker from .slack_utils import get_org_id tracer = trace.get_tracer(__name__) -slack_event_tracker = SlackEventTracker() # global scope +slack_event_tracker = SlackEventTracker() # global scope #TODO: move to a function decorator -# NOTE: middleware calls inject args so name needs to match. See for all available args https://slack.dev/bolt-python/api-docs/slack_bolt/kwargs_injection/args.html +# NOTE: middleware calls inject args so name needs to match. See for all available args +# @see https://slack.dev/bolt-python/api-docs/slack_bolt/kwargs_injection/args.html @tracer.start_as_current_span(name="filter_duplicate_event_middleware") -def filter_duplicate_event_middleware(ack: Callable, body: dict, next_: Callable) -> None: +def filter_duplicate_event_middleware(ack: Ack, body: dict, next_: Callable) -> None: """Middleware to check if an event has already been seen and handled. This prevents duplicate processing of messages. Duplicate events are legit. Slack can send the same event multiple times. """ span = trace.get_current_span() - ack() - print("\033[32mmessage_handled_middleware\033[0m body: ", json.dumps(body, indent=4)) + # print("\033[32mmessage_handled_middleware\033[0m body: ", json.dumps(body, indent=4)) client_msg_id = body["event"]["client_msg_id"] type_ = body["event"]["type"] @@ -54,22 +55,16 @@ def filter_duplicate_event_middleware(ack: Callable, body: dict, next_: Callable "org_id": org_id if org_id else "None", } ) - # if org_id is None: - # span.record_exception(ValueError(f"No Org ID found for Slack team ID '{team_id}'")) - # span.set_status(trace.StatusCode.ERROR, "No Org ID found") - # raise ValueError(f"No Org ID found for Slack team ID '{team_id}'") - # message_handled = manage_slack_messages.is_message_handled(client_msg_id, ts, org_id) - - print(f"\033[32mmessage_handled_middleware\033[0m: duplicate message '{is_duplicate}'. event_id: {event_id}") if not is_duplicate: - next_() + next_() # continue processing the event. The main handler will ack + else: + ack() # acknowledge the duplicate event to prevent Slack from resending it @tracer.start_as_current_span(name="persist_message_middleware") def persist_message_middleware(body: dict, next_: Callable) -> None: """Middleware to persist messages.""" - print("\033[32mpersist_message_middleware\033[0m: persisting slack message") span = trace.get_current_span() client_msg_id = body["event"]["client_msg_id"] type_ = body["event"]["type"] diff --git a/web/api/integration/slack/slack_utils.py b/web/api/integration/slack/slack_utils.py index c6b732c6..3599a31c 100644 --- a/web/api/integration/slack/slack_utils.py +++ b/web/api/integration/slack/slack_utils.py @@ -1,5 +1,4 @@ """Slack application utils.""" -import streamlit as st from docq.integrations import manage_slack from opentelemetry import trace from slack_sdk import WebClient @@ -7,7 +6,7 @@ tracer = trace.get_tracer(__name__) -@st.cache_data(ttl=6000) + def get_org_id(team_id: str) -> int | None: """Get the org id for a Slack team / workspace.""" result = manage_slack.list_docq_slack_installations(org_id=None, team_id=team_id) From 2d44c44805bf3a1e94b2fc20c5e7ee38076d72d5 Mon Sep 17 00:00:00 2001 From: Janaka Abeywardhana Date: Mon, 29 Apr 2024 22:35:54 +0100 Subject: [PATCH 3/5] chore: remove test logging line --- source/docq/setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/docq/setup.py b/source/docq/setup.py index 4fdfb924..7e2238b8 100644 --- a/source/docq/setup.py +++ b/source/docq/setup.py @@ -56,5 +56,3 @@ def init() -> None: #metadata_extractors._cache_metadata_extractor_models() logging.info("Docq initialized") span.add_event("Docq initialized") - - logging.error("Docq error test message") \ No newline at end of file From 05fcde42866d35c318d449d7a25fbc3b04fe1d3a Mon Sep 17 00:00:00 2001 From: Janaka Abeywardhana Date: Mon, 29 Apr 2024 22:36:29 +0100 Subject: [PATCH 4/5] feat(slack): add empty reaction add remove handlers --- web/api/integration/slack/slack_event_handler.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/web/api/integration/slack/slack_event_handler.py b/web/api/integration/slack/slack_event_handler.py index 03ffb35a..e01995bf 100644 --- a/web/api/integration/slack/slack_event_handler.py +++ b/web/api/integration/slack/slack_event_handler.py @@ -60,3 +60,18 @@ def handle_app_mention_event(body: dict, ack: Ack, say: Say) -> None: def handle_message(body: dict, ack: Ack, say: Say) -> None: """Events of type message. This fires for multiple types including app_mention.""" Ack() + +@slack_app.event("reaction_added", middleware=[filter_duplicate_event_middleware]) +@tracer.start_as_current_span(name="handle_reaction_added") +def handle_reaction_added(body: dict, ack: Ack) -> None: + """Handle reaction added events.""" + reaction_emoji_name = body["event"]["reaction"] + ack() + + +@slack_app.event("reaction_added", middleware=[filter_duplicate_event_middleware]) +@tracer.start_as_current_span(name="handle_reaction_added") +def handle_reaction_removed(body: dict, ack: Ack) -> None: + """Handle reaction added events.""" + reaction_emoji_name = body["event"]["reaction"] + ack() From d393e320d646f83b66d97d76fa42edcb341732fc Mon Sep 17 00:00:00 2001 From: Janaka Abeywardhana Date: Mon, 29 Apr 2024 22:37:20 +0100 Subject: [PATCH 5/5] bump version 0.10.2 -> 0.10.4 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ec889182..7ce9fdab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "docq" -version = "0.10.2" +version = "0.10.4" description = "Docq.AI - Your private ChatGPT alternative. Securely unlock knowledge from confidential documents." authors = ["Docq.AI Team "] maintainers = ["Docq.AI Team "]