From af4743a7fa7db78a3e53616fb4ee05633c56786e Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 21 Jun 2024 20:21:05 +0530 Subject: [PATCH 01/57] litellm base version --- augmentation/paraphrase/gpt3/gpt.py | 7 +- custom/__init__.py | 0 custom/fallback.py | 58 - custom/ner.py | 169 --- kairon/actions/definitions/database.py | 2 +- kairon/actions/definitions/prompt.py | 31 +- kairon/api/models.py | 24 +- kairon/chat/agent/message_processor.py | 9 - kairon/importer/validator/file_validator.py | 31 +- kairon/shared/actions/data_objects.py | 8 +- .../concurrency/actors/pyscript_runner.py | 10 +- kairon/shared/data/constant.py | 1 + kairon/shared/data/processor.py | 4 +- kairon/shared/llm/base.py | 4 +- kairon/shared/llm/clients/__init__.py | 0 kairon/shared/llm/clients/azure.py | 25 - kairon/shared/llm/clients/base.py | 8 - kairon/shared/llm/clients/factory.py | 18 - kairon/shared/llm/clients/gpt3.py | 92 -- kairon/shared/llm/data_objects.py | 13 + kairon/shared/llm/factory.py | 17 - kairon/shared/llm/logger.py | 43 + kairon/shared/llm/{gpt3.py => processor.py} | 106 +- kairon/shared/utils.py | 85 +- kairon/shared/vector_embeddings/db/base.py | 8 +- kairon/shared/vector_embeddings/db/qdrant.py | 18 +- kairon/train.py | 7 +- metadata/integrations.yml | 127 +- requirements/dev.txt | 13 +- requirements/prod.txt | 68 +- tests/integration_test/action_service_test.py | 1066 ++++++++++------- tests/integration_test/chat_service_test.py | 10 +- tests/integration_test/services_test.py | 24 +- tests/unit_test/action/action_test.py | 6 +- tests/unit_test/api/api_processor_test.py | 9 +- .../augmentation/gpt_augmentation_test.py | 10 +- .../data_processor/data_processor_test.py | 112 +- tests/unit_test/events/events_test.py | 11 +- tests/unit_test/llm_test.py | 791 ++++++------ tests/unit_test/utility_test.py | 745 +----------- .../validator/training_data_validator_test.py | 73 +- training_data/ReadMe.md | 1 - 42 files changed, 1481 insertions(+), 2383 deletions(-) delete mode 100644 custom/__init__.py delete mode 100644 custom/fallback.py delete mode 100644 custom/ner.py delete mode 100644 kairon/shared/llm/clients/__init__.py delete mode 100644 kairon/shared/llm/clients/azure.py delete mode 100644 kairon/shared/llm/clients/base.py delete mode 100644 kairon/shared/llm/clients/factory.py delete mode 100644 kairon/shared/llm/clients/gpt3.py create mode 100644 kairon/shared/llm/data_objects.py delete mode 100644 kairon/shared/llm/factory.py create mode 100644 kairon/shared/llm/logger.py rename kairon/shared/llm/{gpt3.py => processor.py} (73%) delete mode 100644 training_data/ReadMe.md diff --git a/augmentation/paraphrase/gpt3/gpt.py b/augmentation/paraphrase/gpt3/gpt.py index 340c220c0..b11a8eac5 100644 --- a/augmentation/paraphrase/gpt3/gpt.py +++ b/augmentation/paraphrase/gpt3/gpt.py @@ -1,7 +1,7 @@ """Creates the Example and GPT classes for a user to interface with the OpenAI API.""" -import openai +from openai import OpenAI import uuid @@ -95,8 +95,9 @@ def submit_request(self, prompt, num_responses, api_key): """Calls the OpenAI API with the specified parameters.""" if num_responses < 1: num_responses = 1 - response = openai.Completion.create(api_key=api_key, - engine=self.get_engine(), + client = OpenAI(api_key=api_key) + response = client.completions.create( + model=self.get_engine(), prompt=self.craft_query(prompt), max_tokens=self.get_max_tokens(), temperature=self.get_temperature(), diff --git a/custom/__init__.py b/custom/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/custom/fallback.py b/custom/fallback.py deleted file mode 100644 index 4c396c039..000000000 --- a/custom/fallback.py +++ /dev/null @@ -1,58 +0,0 @@ -''' -Custom component to get fallback action intent -Reference: https://forum.rasa.com/t/fallback-intents-for-context-sensitive-fallbacks/963 -''' - -from rasa.nlu.classifiers.classifier import IntentClassifier - -class FallbackIntentFilter(IntentClassifier): - - # Name of the component to be used when integrating it in a - # pipeline. E.g. ``[ComponentA, ComponentB]`` - # will be a proper pipeline definition where ``ComponentA`` - # is the name of the first component of the pipeline. - name = "FallbackIntentFilter" - - # Defines what attributes the pipeline component will - # provide when called. The listed attributes - # should be set by the component on the message object - # during test and train, e.g. - # ```message.set("entities", [...])``` - provides = [] - - # Which attributes on a message are required by this - # component. e.g. if requires contains "tokens", than a - # previous component in the pipeline needs to have "tokens" - # within the above described `provides` property. - requires = [] - - # Defines the default configuration parameters of a component - # these values can be overwritten in the pipeline configuration - # of the model. The component should choose sensible defaults - # and should be able to create reasonable results with the defaults. - defaults = {} - - # Defines what language(s) this component can handle. - # This attribute is designed for instance method: `can_handle_language`. - # Default value is None which means it can handle all languages. - # This is an important feature for backwards compatibility of components. - language_list = None - - def __init__(self, component_config=None, low_threshold=0.3, high_threshold=0.4, fallback_intent="fallback", - out_of_scope_intent="out_of_scope"): - super().__init__(component_config) - self.fb_low_threshold = low_threshold - self.fb_high_threshold = high_threshold - self.fallback_intent = fallback_intent - self.out_of_scope_intent = out_of_scope_intent - - def process(self, message, **kwargs): - message_confidence = message.data['intent']['confidence'] - new_intent = None - if message_confidence <= self.fb_low_threshold: - new_intent = {'name': self.out_of_scope_intent, 'confidence': message_confidence} - elif message_confidence <= self.fb_high_threshold: - new_intent = {'name': self.fallback_intent, 'confidence': message_confidence} - if new_intent is not None: - message.data['intent'] = new_intent - message.data['intent_ranking'].insert(0, new_intent) diff --git a/custom/ner.py b/custom/ner.py deleted file mode 100644 index 1a38897fe..000000000 --- a/custom/ner.py +++ /dev/null @@ -1,169 +0,0 @@ -from rasa.nlu.components import Component -from typing import Any, Optional, Text, Dict, TYPE_CHECKING -import os -import spacy -import pickle -from spacy.matcher import Matcher -from rasa.nlu.extractors.extractor import EntityExtractor - - -if TYPE_CHECKING: - from rasa.nlu.model import Metadata - -PATTERN_NER_FILE = 'pattern_ner.pkl' -class SpacyPatternNER(EntityExtractor): - """A new component""" - name = "pattern_ner_spacy" - # Defines what attributes the pipeline component will - # provide when called. The listed attributes - # should be set by the component on the message object - # during test and train, e.g. - # ```message.set("entities", [...])``` - provides = ["entities"] - - # Which attributes on a message are required by this - # component. e.g. if requires contains "tokens", than a - # previous component in the pipeline needs to have "tokens" - # within the above described `provides` property. - requires = ["tokens"] - - # Defines the default configuration parameters of a component - # these values can be overwritten in the pipeline configuration - # of the model. The component should choose sensible defaults - # and should be able to create reasonable results with the defaults. - defaults = {} - - # Defines what language(s) this component can handle. - # This attribute is designed for instance method: `can_handle_language`. - # Default value is None which means it can handle all languages. - # This is an important feature for backwards compatibility of components. - language_list = None - - def __init__(self, component_config=None, matcher=None): - super(SpacyPatternNER, self).__init__(component_config) - if matcher: - self.matcher = matcher - self.spacy_nlp = spacy.blank('en') - self.spacy_nlp.vocab = self.matcher.vocab - else: - self.spacy_nlp = spacy.blank('en') - self.matcher = Matcher(self.spacy_nlp.vocab) - - def train(self, training_data, cfg, **kwargs): - """Train this component. - - This is the components chance to train itself provided - with the training data. The component can rely on - any context attribute to be present, that gets created - by a call to :meth:`components.Component.pipeline_init` - of ANY component and - on any context attributes created by a call to - :meth:`components.Component.train` - of components previous to this one.""" - for lookup_table in training_data.lookup_tables: - key = lookup_table['name'] - pattern = [] - for element in lookup_table['elements']: - tokens = [{'LOWER': token.lower()} for token in str(element).split()] - pattern.append(tokens) - self.matcher.add(key, pattern) - - def process(self, message, **kwargs): - """Process an incoming message. - - This is the components chance to process an incoming - message. The component can rely on - any context attribute to be present, that gets created - by a call to :meth:`components.Component.pipeline_init` - of ANY component and - on any context attributes created by a call to - :meth:`components.Component.process` - of components previous to this one.""" - entities = [] - - # with plural forms - doc = self.spacy_nlp(message.data['text'].lower()) - matches = self.matcher(doc) - entities = self.getNewEntityObj(doc, matches, entities) - - # Without plural forms - doc = self.spacy_nlp(' '.join([token.lemma_ for token in doc])) - matches = self.matcher(doc) - entities = self.getNewEntityObj(doc, matches, entities) - - # Remove duplicates - seen = set() - new_entities = [] - - for entityObj in entities: - record = tuple(entityObj.items()) - if record not in seen: - seen.add(record) - new_entities.append(entityObj) - - message.set("entities", message.get("entities", []) + new_entities, add_to_output=True) - - - def getNewEntityObj(self, doc, matches, entities): - - for ent_id, start, end in matches: - new_entity_value = doc[start:end].text - new_entity_value_len = len(new_entity_value.split()) - is_add = True - - for old_entity in entities: - old_entity_value = old_entity["value"] - old_entity_value_len = len(old_entity_value.split()) - - if old_entity_value_len > new_entity_value_len and new_entity_value in old_entity_value: - is_add = False - elif old_entity_value_len < new_entity_value_len and old_entity_value in new_entity_value: - entities.remove(old_entity) - - if is_add: - entities.append({ - 'start': start, - 'end': end, - 'value': doc[start:end].text, - 'entity': self.matcher.vocab.strings[ent_id], - 'confidence': None, - 'extractor': self.name - }) - - return entities - - - def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: - """Persist this component to disk for future loading.""" - if self.matcher: - modelFile = os.path.join(model_dir, PATTERN_NER_FILE) - self.saveModel(modelFile) - return {"pattern_ner_file": PATTERN_NER_FILE} - - - @classmethod - def load( - cls, - meta: Dict[Text, Any], - model_dir: Optional[Text] = None, - model_metadata: Optional["Metadata"] = None, - cached_component: Optional["Component"] = None, - **kwargs: Any - ) -> "Component": - """Load this component from file.""" - - file_name = meta.get("pattern_ner_file", PATTERN_NER_FILE) - modelFile = os.path.join(model_dir, file_name) - if os.path.exists(modelFile): - modelLoad = open(modelFile, "rb") - matcher = pickle.load(modelLoad) - modelLoad.close() - return cls(meta, matcher) - else: - return cls(meta) - - - def saveModel(self, modelFile): - modelSave = open(modelFile, "wb") - pickle.dump(self.matcher, modelSave) - modelSave.close() \ No newline at end of file diff --git a/kairon/actions/definitions/database.py b/kairon/actions/definitions/database.py index 6f0e48271..ebdf83510 100644 --- a/kairon/actions/definitions/database.py +++ b/kairon/actions/definitions/database.py @@ -83,7 +83,7 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma request_body = ActionUtility.get_payload(payload, tracker) msg_logger.append(request_body) tracker_data = ActionUtility.build_context(tracker, True) - response = await vector_db.perform_operation(operation_type, request_body) + response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id, bot=self.bot) logger.info("response: " + str(response)) response_context = self.__add_user_context_to_http_response(response, tracker_data) bot_response, bot_resp_log, _ = ActionUtility.compose_response(vector_action_config['response'], response_context) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 3de92f413..381e6f543 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -4,7 +4,6 @@ from rasa_sdk import Tracker from rasa_sdk.executor import CollectingDispatcher -from kairon import Utility from kairon.actions.definitions.base import ActionsBase from kairon.shared.actions.data_objects import ActionServerLogs from kairon.shared.actions.exception import ActionFailure @@ -12,8 +11,8 @@ from kairon.shared.actions.utils import ActionUtility from kairon.shared.constants import FAQ_DISABLED_ERR, KaironSystemSlots, KAIRON_USER_MSG_ENTITY from kairon.shared.data.constant import DEFAULT_NLU_FALLBACK_RESPONSE -from kairon.shared.llm.factory import LLMFactory from kairon.shared.models import LlmPromptType, LlmPromptSource +from kairon.shared.llm.processor import LLMProcessor class ActionPrompt(ActionsBase): @@ -62,14 +61,18 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma time_taken_slots = 0 final_slots = {"type": "slots_to_fill"} llm_response_log = {"type": "llm_response"} - + llm_processor = None try: k_faq_action_config, bot_settings = self.retrieve_config() user_question = k_faq_action_config.get('user_question') user_msg = self.__get_user_msg(tracker, user_question) + llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) - llm = LLMFactory.get_instance("faq")(self.bot, bot_settings["llm_settings"]) - llm_response, time_taken_llm_response = await llm.predict(user_msg, **llm_params) + llm_processor = LLMProcessor(self.bot) + llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, + user=tracker.sender_id, + bot=self.bot, + **llm_params) status = "FAILURE" if llm_response.get("is_failure", False) is True else status exception = llm_response.get("exception") bot_response = llm_response['content'] @@ -93,8 +96,8 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma total_time_elapsed = time_taken_llm_response + time_taken_slots events_to_extend = [llm_response_log, final_slots] events.extend(events_to_extend) - if llm: - llm_logs = llm.logs + if llm_processor: + llm_logs = llm_processor.logs ActionServerLogs( type=ActionType.prompt_action.value, intent=tracker.get_intent_of_latest_message(skip_fallback_intent=False), @@ -119,16 +122,6 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma return slots_to_fill async def __get_llm_params(self, k_faq_action_config: dict, dispatcher: CollectingDispatcher, tracker: Tracker, domain: Dict[Text, Any]): - implementations = { - "GPT3_FAQ_EMBED": self.__get_gpt_params, - } - - llm_type = Utility.environment['llm']["faq"] - if not implementations.get(llm_type): - raise ActionFailure(f'{llm_type} type LLM is not supported') - return await implementations[Utility.environment['llm']["faq"]](k_faq_action_config, dispatcher, tracker, domain) - - async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: CollectingDispatcher, tracker: Tracker, domain: Dict[Text, Any]): from kairon.actions.definitions.factory import ActionFactory system_prompt = None @@ -147,7 +140,7 @@ async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: Collecti history_prompt = ActionUtility.prepare_bot_responses(tracker, num_bot_responses) elif prompt['source'] == LlmPromptSource.bot_content.value and prompt['is_enabled']: use_similarity_prompt = True - hyperparameters = prompt.get('hyperparameters', {}) + hyperparameters = prompt.get("hyperparameters", {}) similarity_prompt.append({'similarity_prompt_name': prompt['name'], 'similarity_prompt_instructions': prompt['instructions'], 'collection': prompt['data'], @@ -179,7 +172,7 @@ async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: Collecti is_query_prompt_enabled = True query_prompt_dict.update({'query_prompt': query_prompt, 'use_query_prompt': is_query_prompt_enabled}) - params["hyperparameters"] = k_faq_action_config.get('hyperparameters', Utility.get_llm_hyperparameters()) + params["hyperparameters"] = k_faq_action_config['hyperparameters'] params["system_prompt"] = system_prompt params["context_prompt"] = context_prompt params["query_prompt"] = query_prompt_dict diff --git a/kairon/api/models.py b/kairon/api/models.py index d2ad8b11b..435566f34 100644 --- a/kairon/api/models.py +++ b/kairon/api/models.py @@ -16,6 +16,7 @@ INTEGRATION_STATUS, FALLBACK_MESSAGE, DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_LLM ) from ..shared.actions.models import ( ActionParameterType, @@ -37,6 +38,7 @@ CognitionDataType, CognitionMetadataType, ) +from kairon.shared.utils import Utility class RecaptchaVerifiedRequest(BaseModel): @@ -1035,6 +1037,7 @@ class PromptActionConfigRequest(BaseModel): num_bot_responses: int = 5 failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE user_question: UserQuestionModel = UserQuestionModel() + llm_type: str = DEFAULT_LLM hyperparameters: dict = None llm_prompts: List[LlmPromptRequest] instructions: List[str] = [] @@ -1056,16 +1059,27 @@ def validate_num_bot_responses(cls, v, values, **kwargs): raise ValueError("num_bot_responses should not be greater than 5") return v + @validator("llm_type") + def validate_llm_type(cls, v, values, **kwargs): + if v not in Utility.get_llms(): + raise ValueError("Invalid llm type") + return v + + @validator("hyperparameters") + def validate_llm_hyperparameters(cls, v, values, **kwargs): + Utility.validate_llm_hyperparameters(v, kwargs['llm_type'], ValueError) + @root_validator def check(cls, values): from kairon.shared.utils import Utility - if not values.get("hyperparameters"): - values["hyperparameters"] = {} + if values.get("llm_type"): + if not values.get("hyperparameters"): + values["hyperparameters"] = {} - for key, value in Utility.get_llm_hyperparameters().items(): - if key not in values["hyperparameters"]: - values["hyperparameters"][key] = value + for key, value in Utility.get_llm_hyperparameters(values.get("llm_type")).items(): + if key not in values["hyperparameters"]: + values["hyperparameters"][key] = value return values diff --git a/kairon/chat/agent/message_processor.py b/kairon/chat/agent/message_processor.py index 827404063..e4ae506d2 100644 --- a/kairon/chat/agent/message_processor.py +++ b/kairon/chat/agent/message_processor.py @@ -294,15 +294,6 @@ def predict_next_with_tracker_if_should( Raises: ActionLimitReached if the limit of actions to predict has been reached. """ - should_predict_another_action = self.should_predict_another_action( - tracker.latest_action_name - ) - - if self.is_action_limit_reached(tracker, should_predict_another_action): - raise ActionLimitReached( - "The limit of actions to predict has been reached." - ) - prediction = self._predict_next_with_tracker(tracker) action = self.action_for_index( diff --git a/kairon/importer/validator/file_validator.py b/kairon/importer/validator/file_validator.py index b55b3f0e6..ad2062c01 100644 --- a/kairon/importer/validator/file_validator.py +++ b/kairon/importer/validator/file_validator.py @@ -695,9 +695,9 @@ def __validate_prompt_actions(prompt_actions: list): data_error.append( f'num_bot_responses should not be greater than 5 and of type int: {action.get("name")}') llm_prompts_errors = TrainingDataValidator.__validate_llm_prompts(action['llm_prompts']) - if action.get('hyperparameters') is not None: - llm_hyperparameters_errors = TrainingDataValidator.__validate_llm_prompts_hyperparamters( - action.get('hyperparameters')) + if action.get('hyperparameters'): + llm_hyperparameters_errors = TrainingDataValidator.__validate_llm_prompts_hyperparameters( + action.get('hyperparameters'), action.get("llm_type", "openai")) data_error.extend(llm_hyperparameters_errors) data_error.extend(llm_prompts_errors) if action['name'] in actions_present: @@ -785,27 +785,12 @@ def __validate_llm_prompts(llm_prompts: dict): return error_list @staticmethod - def __validate_llm_prompts_hyperparamters(hyperparameters: dict): + def __validate_llm_prompts_hyperparameters(hyperparameters: dict, llm_type: str): error_list = [] - for key, value in hyperparameters.items(): - if key == 'temperature' and not 0.0 <= value <= 2.0: - error_list.append("Temperature must be between 0.0 and 2.0!") - elif key == 'presence_penalty' and not -2.0 <= value <= 2.0: - error_list.append("presence_penality must be between -2.0 and 2.0!") - elif key == 'frequency_penalty' and not -2.0 <= value <= 2.0: - error_list.append("frequency_penalty must be between -2.0 and 2.0!") - elif key == 'top_p' and not 0.0 <= value <= 1.0: - error_list.append("top_p must be between 0.0 and 1.0!") - elif key == 'n' and not 1 <= value <= 5: - error_list.append("n must be between 1 and 5!") - elif key == 'max_tokens' and not 5 <= value <= 4096: - error_list.append("max_tokens must be between 5 and 4096!") - elif key == 'logit_bias' and not isinstance(value, dict): - error_list.append("logit_bias must be a dictionary!") - elif key == 'stop': - if value and (not isinstance(value, (str, int, list)) or (isinstance(value, list) and len(value) > 4)): - error_list.append( - "Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers.") + try: + Utility.validate_llm_hyperparameters(hyperparameters, llm_type, AppException) + except AppException as e: + error_list.append(e.__str__()) return error_list @staticmethod diff --git a/kairon/shared/actions/data_objects.py b/kairon/shared/actions/data_objects.py index dc86e79e3..d3f64347f 100644 --- a/kairon/shared/actions/data_objects.py +++ b/kairon/shared/actions/data_objects.py @@ -34,6 +34,7 @@ KAIRON_TWO_STAGE_FALLBACK, FALLBACK_MESSAGE, DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_LLM ) from kairon.shared.data.signals import push_notification, auditlogger from kairon.shared.models import LlmPromptType, LlmPromptSource @@ -772,7 +773,8 @@ class PromptAction(Auditlog): bot = StringField(required=True) user = StringField(required=True) timestamp = DateTimeField(default=datetime.utcnow) - hyperparameters = DictField(default=Utility.get_llm_hyperparameters) + llm_type = StringField(default=DEFAULT_LLM, choices=Utility.get_llms()) + hyperparameters = DictField(default=Utility.get_default_llm_hyperparameters) llm_prompts = EmbeddedDocumentListField(LlmPrompt, required=True) instructions = ListField(StringField()) set_slots = EmbeddedDocumentListField(SetSlotsFromResponse) @@ -782,7 +784,7 @@ class PromptAction(Auditlog): meta = {"indexes": [{"fields": ["bot", ("bot", "name", "status")]}]} def clean(self): - for key, value in Utility.get_llm_hyperparameters().items(): + for key, value in Utility.get_llm_hyperparameters(self.llm_type).items(): if key not in self.hyperparameters: self.hyperparameters.update({key: value}) @@ -800,7 +802,7 @@ def validate(self, clean=True): dict_data["llm_prompts"], ValidationError ) Utility.validate_llm_hyperparameters( - dict_data["hyperparameters"], ValidationError + dict_data["hyperparameters"], self.llm_type, ValidationError ) diff --git a/kairon/shared/concurrency/actors/pyscript_runner.py b/kairon/shared/concurrency/actors/pyscript_runner.py index a68e352c9..bd5286739 100644 --- a/kairon/shared/concurrency/actors/pyscript_runner.py +++ b/kairon/shared/concurrency/actors/pyscript_runner.py @@ -1,20 +1,26 @@ from types import ModuleType from typing import Text, Dict, Optional, Callable +import orjson as json from AccessControl.ZopeGuards import _safe_globals from RestrictedPython import compile_restricted from RestrictedPython.Guards import safer_getattr from loguru import logger from timeout_decorator import timeout_decorator -import orjson as json -from ..actors.base import BaseActor from kairon.exceptions import AppException +from ..actors.base import BaseActor +from AccessControl.SecurityInfo import allow_module + +allow_module("datetime") +allow_module("time") + global_safe = _safe_globals global_safe['_getattr_'] = safer_getattr global_safe['json'] = json + class PyScriptRunner(BaseActor): def execute(self, source_code: Text, predefined_objects: Optional[Dict] = None, **kwargs): diff --git a/kairon/shared/data/constant.py b/kairon/shared/data/constant.py index 00b573548..b80e82ee0 100644 --- a/kairon/shared/data/constant.py +++ b/kairon/shared/data/constant.py @@ -208,6 +208,7 @@ class ModelTestType(str, Enum): DEFAULT_SYSTEM_PROMPT = ( "You are a personal assistant. Answer question based on the context below" ) +DEFAULT_LLM = "openai" class AuditlogActions(str, Enum): diff --git a/kairon/shared/data/processor.py b/kairon/shared/data/processor.py index 2c6a4a004..d533e937e 100644 --- a/kairon/shared/data/processor.py +++ b/kairon/shared/data/processor.py @@ -7269,9 +7269,7 @@ def edit_prompt_action( action.failure_message = request_data.get("failure_message") action.user_question = UserQuestion(**request_data.get("user_question")) action.num_bot_responses = request_data.get("num_bot_responses", 5) - action.hyperparameters = request_data.get( - "hyperparameters", Utility.get_llm_hyperparameters() - ) + action.hyperparameters = request_data.get("hyperparameters") action.llm_prompts = [ LlmPrompt(**prompt) for prompt in request_data.get("llm_prompts", []) ] diff --git a/kairon/shared/llm/base.py b/kairon/shared/llm/base.py index 4babc6a23..f07eceda0 100644 --- a/kairon/shared/llm/base.py +++ b/kairon/shared/llm/base.py @@ -8,9 +8,9 @@ def __init__(self, bot: Text): self.bot = bot @abstractmethod - async def train(self, *args, **kwargs) -> Dict: + async def train(self, user, bot, *args, **kwargs) -> Dict: pass @abstractmethod - async def predict(self, query, *args, **kwargs) -> Dict: + async def predict(self, query, user, bot, *args, **kwargs) -> Dict: pass diff --git a/kairon/shared/llm/clients/__init__.py b/kairon/shared/llm/clients/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/kairon/shared/llm/clients/azure.py b/kairon/shared/llm/clients/azure.py deleted file mode 100644 index 9b980d0d6..000000000 --- a/kairon/shared/llm/clients/azure.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Text - -from kairon.shared.constants import GPT3ResourceTypes -from kairon.shared.llm.clients.gpt3 import GPT3Resources - - -class AzureGPT3Resources(GPT3Resources): - resource_url = "https://kairon.openai.azure.com/openai/deployments" - - def __init__(self, api_key: Text, **kwargs): - super().__init__(api_key) - self.api_key = api_key - self.api_version = kwargs.get("api_version") - self.model_id = { - GPT3ResourceTypes.embeddings.value: kwargs.get("embeddings_model_id"), - GPT3ResourceTypes.chat_completion.value: kwargs.get("chat_completion_model_id") - } - - def get_headers(self): - return {"api-key": self.api_key} - - def get_resource_url(self, resource: Text): - model_id = self.model_id[resource] - resource_url = f"{self.resource_url}/{model_id}/{resource}?api-version={self.api_version}" - return resource_url diff --git a/kairon/shared/llm/clients/base.py b/kairon/shared/llm/clients/base.py deleted file mode 100644 index 71ef7037e..000000000 --- a/kairon/shared/llm/clients/base.py +++ /dev/null @@ -1,8 +0,0 @@ -from abc import ABC -from typing import Text - - -class LLMResources(ABC): - - async def invoke(self, resource: Text, engine: Text, **kwargs): - raise NotImplementedError("Provider not implemented") diff --git a/kairon/shared/llm/clients/factory.py b/kairon/shared/llm/clients/factory.py deleted file mode 100644 index def8d09c3..000000000 --- a/kairon/shared/llm/clients/factory.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import Text -from kairon.exceptions import AppException -from kairon.shared.constants import LLMResourceProvider -from kairon.shared.llm.clients.azure import AzureGPT3Resources -from kairon.shared.llm.clients.gpt3 import GPT3Resources - - -class LLMClientFactory: - __implementations = { - LLMResourceProvider.openai.value: GPT3Resources, - LLMResourceProvider.azure.value: AzureGPT3Resources - } - - @staticmethod - def get_resource_provider(_type: Text): - if not LLMClientFactory.__implementations.get(_type): - raise AppException(f'{_type} client not supported') - return LLMClientFactory.__implementations[_type] diff --git a/kairon/shared/llm/clients/gpt3.py b/kairon/shared/llm/clients/gpt3.py deleted file mode 100644 index d6f2c5679..000000000 --- a/kairon/shared/llm/clients/gpt3.py +++ /dev/null @@ -1,92 +0,0 @@ -import ujson as json -import random -from json import JSONDecodeError -from ujson import JSONDecodeError as UJSONDecodeError -from typing import Text -from loguru import logger -from openai.api_requestor import parse_stream_helper -from kairon.exceptions import AppException -from kairon.shared.constants import GPT3ResourceTypes -from kairon.shared.llm.clients.base import LLMResources -from kairon.shared.rest_client import AioRestClient - - -class GPT3Resources(LLMResources): - resource_url = "https://api.openai.com/v1" - - def __init__(self, api_key: Text, **kwargs): - self.api_key = api_key - - def get_headers(self): - return {"Authorization": f"Bearer {self.api_key}"} - - def get_resource_url(self, resource: Text): - return f"{self.resource_url}/{resource}" - - async def invoke(self, resource: Text, model: Text, **kwargs): - client = None - http_url = self.get_resource_url(resource) - request_body = kwargs.copy() - request_body.update({"model": model}) - is_streaming_resp = kwargs.get("stream", False) - try: - client = AioRestClient(False) - resp = await client.request("POST", http_url, request_body, self.get_headers(), - return_json=False, is_streaming_resp=is_streaming_resp, max_retries=3) - if resp.status != 200: - try: - resp = await resp.json() - logger.debug(f"GPT response error: {resp}") - raise AppException(f"{resp['error'].get('message')}. Request id: {resp['error'].get('id')}") - except JSONDecodeError: - raise AppException(f"Received non 200 status code ({resp.status}): {resp.text}") - - if is_streaming_resp: - resp = client.streaming_response - - data = await self.__parse_response(resource, resp, **kwargs) - finally: - if client: - await client.cleanup() - return data - - async def __parse_response(self, resource: Text, response, **kwargs): - parsers = { - GPT3ResourceTypes.embeddings.value: self._parse_embeddings_response, - GPT3ResourceTypes.chat_completion.value: self.__parse_completion_response - } - return await parsers[resource](response, **kwargs) - - async def _parse_embeddings_response(self, response, **hyperparameters): - raw_response = await response.json() - formatted_response = raw_response["data"][0]["embedding"] - return formatted_response, raw_response - - async def __parse_completion_response(self, response, **kwargs): - if kwargs.get("stream"): - formatted_response = await self._parse_streaming_response(response, kwargs.get("n", 1)) - raw_response = response - else: - formatted_response, raw_response = await self._parse_api_response(response) - return formatted_response, raw_response - - async def _parse_api_response(self, response): - raw_response = await response.json() - msg_choice = random.choice(raw_response['choices']) - formatted_response = msg_choice['message']['content'] - return formatted_response, raw_response - - async def _parse_streaming_response(self, response, num_choices): - formatted_response = '' - msg_choice = random.randint(0, num_choices - 1) - try: - for chunk in response or []: - line = parse_stream_helper(chunk) - if line: - line = json.loads(line) - if line["choices"][0].get("index") == msg_choice and line["choices"][0]['delta'].get('content'): - formatted_response = f"{formatted_response}{line['choices'][0]['delta']['content']}" - except Exception as e: - logger.exception(e) - raise AppException(f"Failed to parse streaming response: {chunk}") - return formatted_response diff --git a/kairon/shared/llm/data_objects.py b/kairon/shared/llm/data_objects.py new file mode 100644 index 000000000..7713444ba --- /dev/null +++ b/kairon/shared/llm/data_objects.py @@ -0,0 +1,13 @@ +from mongoengine import Document, DynamicField, StringField, FloatField, DateTimeField, DictField + + +class LLMLogs(Document): + response = DynamicField() + start_time = DateTimeField() + end_time = DateTimeField() + cost = FloatField() + llm_call_id = StringField() + llm_provider = StringField() + model = StringField() + model_params = DictField() + metadata = DictField() \ No newline at end of file diff --git a/kairon/shared/llm/factory.py b/kairon/shared/llm/factory.py deleted file mode 100644 index 5424d1eea..000000000 --- a/kairon/shared/llm/factory.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Text -from kairon.exceptions import AppException -from kairon.shared.utils import Utility -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - - -class LLMFactory: - __implementations = { - "GPT3_FAQ_EMBED": GPT3FAQEmbedding - } - - @staticmethod - def get_instance(_type: Text): - llm_type = Utility.environment['llm'][_type] - if not LLMFactory.__implementations.get(llm_type): - raise AppException(f'{llm_type} type LLM is not supported') - return LLMFactory.__implementations[llm_type] diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py new file mode 100644 index 000000000..06b720b48 --- /dev/null +++ b/kairon/shared/llm/logger.py @@ -0,0 +1,43 @@ +from litellm.integrations.custom_logger import CustomLogger +from .data_objects import LLMLogs +import ujson as json + + +class LiteLLMLogger(CustomLogger): + def log_pre_api_call(self, model, messages, kwargs): + pass + + def log_post_api_call(self, kwargs, response_obj, start_time, end_time): + pass + + def log_stream_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + async def async_log_stream_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + def __logs_litellm(self, **kwargs): + litellm_params = kwargs['litellm_params'] + self.__save_logs(**{'response': json.loads(kwargs['original_response']), + 'start_time': kwargs['start_time'], + 'end_time': kwargs['end_time'], + 'cost': kwargs["response_cost"], + 'llm_call_id': litellm_params['litellm_call_id'], + 'llm_provider': litellm_params['custom_llm_provider'], + 'model_params': kwargs["additional_args"]["complete_input_dict"], + 'metadata': litellm_params['metadata']}) + + def __save_logs(self, **kwargs): + LLMLogs(**kwargs).save() diff --git a/kairon/shared/llm/gpt3.py b/kairon/shared/llm/processor.py similarity index 73% rename from kairon/shared/llm/gpt3.py rename to kairon/shared/llm/processor.py index 4e991ca7c..ffc48e2eb 100644 --- a/kairon/shared/llm/gpt3.py +++ b/kairon/shared/llm/processor.py @@ -1,9 +1,9 @@ +import random import time - from typing import Text, Dict, List, Tuple from urllib.parse import urljoin -import openai +import litellm from loguru import logger as logging from tiktoken import get_encoding from tqdm import tqdm @@ -13,35 +13,33 @@ from kairon.shared.admin.processor import Sysadmin from kairon.shared.cognition.data_objects import CognitionData from kairon.shared.cognition.processor import CognitionDataProcessor -from kairon.shared.constants import GPT3ResourceTypes from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_CONTEXT_PROMPT from kairon.shared.llm.base import LLMBase -from kairon.shared.llm.clients.factory import LLMClientFactory +from kairon.shared.llm.logger import LiteLLMLogger from kairon.shared.models import CognitionDataType from kairon.shared.rest_client import AioRestClient from kairon.shared.utils import Utility +litellm.callbacks = [LiteLLMLogger()] + -class GPT3FAQEmbedding(LLMBase): +class LLMProcessor(LLMBase): __embedding__ = 1536 - def __init__(self, bot: Text, llm_settings: dict): + def __init__(self, bot: Text): super().__init__(bot) self.db_url = Utility.environment['vector']['db'] self.headers = {} if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.suffix = "_faq_embd" - self.vector_config = {'size': 1536, 'distance': 'Cosine'} - self.llm_settings = llm_settings + self.vector_config = {'size': self.__embedding__, 'distance': 'Cosine'} self.api_key = Sysadmin.get_bot_secret(bot, BotSecretType.gpt_key.value, raise_err=True) - self.client = LLMClientFactory.get_resource_provider(llm_settings["provider"])(self.api_key, - **self.llm_settings) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 self.__logs = [] - async def train(self, *args, **kwargs) -> Dict: + async def train(self, user, bot, *args, **kwargs) -> Dict: await self.__delete_collections() count = 0 processor = CognitionDataProcessor() @@ -51,35 +49,38 @@ async def train(self, *args, **kwargs) -> Dict: {'$project': {'collection': "$_id", 'content': 1, '_id': 0}} ])) for collections in collection_groups: - collection = f"{self.bot}_{collections['collection']}{self.suffix}" if collections['collection'] else f"{self.bot}{self.suffix}" + collection = f"{self.bot}_{collections['collection']}{self.suffix}" if collections[ + 'collection'] else f"{self.bot}{self.suffix}" await self.__create_collection__(collection) for content in tqdm(collections['content'], desc="Training FAQ"): if content['content_type'] == CognitionDataType.json.value: metadata = processor.find_matching_metadata(self.bot, content['data'], content.get('collection')) - search_payload, embedding_payload = Utility.retrieve_search_payload_and_embedding_payload(content['data'], metadata) + search_payload, embedding_payload = Utility.retrieve_search_payload_and_embedding_payload( + content['data'], metadata) else: search_payload, embedding_payload = {'content': content["data"]}, content["data"] - search_payload['collection_name'] = collection - embeddings = await self.__get_embedding(embedding_payload) + #search_payload['collection_name'] = collection + embeddings = await self.get_embedding(embedding_payload, user, bot) points = [{'id': content['vector_id'], 'vector': embeddings, 'payload': search_payload}] - await self.__collection_upsert__(collection, {'points': points}, err_msg="Unable to train FAQ! Contact support") + await self.__collection_upsert__(collection, {'points': points}, + err_msg="Unable to train FAQ! Contact support") count += 1 return {"faq": count} - async def predict(self, query: Text, *args, **kwargs) -> Tuple: + async def predict(self, query: Text, user, bot, *args, **kwargs) -> Tuple: start_time = time.time() embeddings_created = False try: - query_embedding = await self.__get_embedding(query) + query_embedding = await self.get_embedding(query, user, bot) embeddings_created = True system_prompt = kwargs.pop('system_prompt', DEFAULT_SYSTEM_PROMPT) context_prompt = kwargs.pop('context_prompt', DEFAULT_CONTEXT_PROMPT) context = await self.__attach_similarity_prompt_if_enabled(query_embedding, context_prompt, **kwargs) - answer = await self.__get_answer(query, system_prompt, context, **kwargs) + answer = await self.__get_answer(query, system_prompt, context, user, bot, **kwargs) response = {"content": answer} - except openai.error.APIConnectionError as e: + except Exception as e: logging.exception(e) if embeddings_created: failure_stage = "Retrieving chat completion for the provided query." @@ -87,9 +88,6 @@ async def predict(self, query: Text, *args, **kwargs) -> Tuple: failure_stage = "Creating a new embedding for the provided query." self.__logs.append({'error': f"{failure_stage} {str(e)}"}) response = {"is_failure": True, "exception": str(e), "content": None} - except Exception as e: - logging.exception(e) - response = {"is_failure": True, "exception": str(e), "content": None} end_time = time.time() elapsed_time = end_time - start_time @@ -102,13 +100,36 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def __get_embedding(self, text: Text) -> List[float]: + async def get_embedding(self, text: Text, user, bot) -> List[float]: truncated_text = self.truncate_text(text) - result, _ = await self.client.invoke(GPT3ResourceTypes.embeddings.value, model="text-embedding-3-small", - input=truncated_text) - return result + result = await litellm.aembedding(model="text-embedding-3-small", + input=[truncated_text], + metadata={'user': user, 'bot': bot}, + api_key=self.api_key, + num_retries=3) + return result["data"][0]["embedding"] + + async def __parse_completion_response(self, response, **kwargs): + if kwargs.get("stream"): + formatted_response = '' + msg_choice = random.randint(0, kwargs.get("n", 1) - 1) + if response["choices"][0].get("index") == msg_choice and response["choices"][0]['delta'].get('content'): + formatted_response = f"{response['choices'][0]['delta']['content']}" + else: + msg_choice = random.choice(response['choices']) + formatted_response = msg_choice['message']['content'] + return formatted_response + + async def __get_completion(self, messages, hyperparameters, user, bot, **kwargs): + response = await litellm.acompletion(messages=messages, + metadata={'user': user, 'bot': bot}, + api_key=self.api_key, + num_retries=3, + **hyperparameters) + formatted_response = await self.__parse_completion_response(response, **kwargs) + return formatted_response, response - async def __get_answer(self, query, system_prompt: Text, context: Text, **kwargs): + async def __get_answer(self, query, system_prompt: Text, context: Text, user, bot, **kwargs): use_query_prompt = False query_prompt = '' if kwargs.get('query_prompt', {}): @@ -116,12 +137,15 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, **kwargs query_prompt = query_prompt_dict.get('query_prompt', '') use_query_prompt = query_prompt_dict.get('use_query_prompt') previous_bot_responses = kwargs.get('previous_bot_responses') - hyperparameters = kwargs.get('hyperparameters', Utility.get_llm_hyperparameters()) + hyperparameters = kwargs['hyperparameters'] instructions = kwargs.get('instructions', []) instructions = '\n'.join(instructions) if use_query_prompt and query_prompt: - query = await self.__rephrase_query(query, system_prompt, query_prompt, hyperparameters=hyperparameters) + query = await self.__rephrase_query(query, system_prompt, query_prompt, + hyperparameters=hyperparameters, + user=user, + bot=bot) messages = [ {"role": "system", "content": system_prompt}, ] @@ -130,21 +154,25 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, **kwargs messages.append({"role": "user", "content": f"{context} \n{instructions} \nQ: {query} \nA:"}) if instructions \ else messages.append({"role": "user", "content": f"{context} \nQ: {query} \nA:"}) - completion, raw_response = await self.client.invoke(GPT3ResourceTypes.chat_completion.value, messages=messages, - **hyperparameters) + completion, raw_response = await self.__get_completion(messages=messages, + hyperparameters=hyperparameters, + user=user, + bot=bot) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'answer_query', 'hyperparameters': hyperparameters}) return completion - async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, **kwargs): + async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, bot, **kwargs): messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": f"{query_prompt}\n\n Q: {query}\n A:"} ] - hyperparameters = kwargs.get('hyperparameters', Utility.get_llm_hyperparameters()) + hyperparameters = kwargs['hyperparameters'] - completion, raw_response = await self.client.invoke(GPT3ResourceTypes.chat_completion.value, messages=messages, - **hyperparameters) + completion, raw_response = await self.__get_completion(messages=messages, + hyperparameters=hyperparameters, + user=user, + bot=bot) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'rephrase_query', 'hyperparameters': hyperparameters}) return completion @@ -153,9 +181,9 @@ async def __delete_collections(self): client = AioRestClient(False) try: response = await client.request(http_url=urljoin(self.db_url, "/collections"), - request_method="GET", - headers=self.headers, - timeout=5) + request_method="GET", + headers=self.headers, + timeout=5) if response.get('result'): for collection in response['result'].get('collections') or []: if collection['name'].startswith(self.bot): diff --git a/kairon/shared/utils.py b/kairon/shared/utils.py index 24a6eaa58..b81e542ec 100644 --- a/kairon/shared/utils.py +++ b/kairon/shared/utils.py @@ -78,6 +78,7 @@ TOKEN_TYPE, KAIRON_TWO_STAGE_FALLBACK, SLOT_TYPE, + DEFAULT_LLM ) from .data.dto import KaironStoryStep from .models import StoryStepType, LlmPromptType, LlmPromptSource @@ -2050,73 +2051,33 @@ def verify_email(email: Text): raise AppException("Invalid or disposable Email!") @staticmethod - def get_llm_hyperparameters(): + def get_llms(): + return Utility.system_metadata["llm"].keys() + + @staticmethod + def get_default_llm_hyperparameters(): + return Utility.get_llm_hyperparameters(DEFAULT_LLM) + + @staticmethod + def get_llm_hyperparameters(llm_type): hyperparameters = {} - if Utility.environment["llm"]["faq"] in {"GPT3_FAQ_EMBED"}: - for key, value in Utility.system_metadata["llm"]["gpt"].items(): + if llm_type in Utility.system_metadata["llm"].keys(): + for key, value in Utility.system_metadata["llm"][llm_type]['properties'].items(): hyperparameters[key] = value["default"] return hyperparameters - raise AppException("Could not find any hyperparameters for configured LLM.") + raise AppException(f"Could not find any hyperparameters for {llm_type} LLM.") @staticmethod - def validate_llm_hyperparameters(hyperparameters: dict, exception_class): - params = Utility.system_metadata["llm"]["gpt"] - for key, value in hyperparameters.items(): - if ( - key == "temperature" - and not params["temperature"]["min"] - <= value - <= params["temperature"]["max"] - ): - raise exception_class( - f"Temperature must be between {params['temperature']['min']} and {params['temperature']['max']}!" - ) - elif ( - key == "presence_penalty" - and not params["presence_penalty"]["min"] - <= value - <= params["presence_penalty"]["max"] - ): - raise exception_class( - f"Presence penalty must be between {params['presence_penalty']['min']} and {params['presence_penalty']['max']}!" - ) - elif ( - key == "frequency_penalty" - and not params["presence_penalty"]["min"] - <= value - <= params["presence_penalty"]["max"] - ): - raise exception_class( - f"Frequency penalty must be between {params['presence_penalty']['min']} and {params['presence_penalty']['max']}!" - ) - elif ( - key == "top_p" - and not params["top_p"]["min"] <= value <= params["top_p"]["max"] - ): - raise exception_class( - f"top_p must be between {params['top_p']['min']} and {params['top_p']['max']}!" - ) - elif key == "n" and not params["n"]["min"] <= value <= params["n"]["max"]: - raise exception_class( - f"n must be between {params['n']['min']} and {params['n']['max']} and should not be 0!" - ) - elif ( - key == "max_tokens" - and not params["max_tokens"]["min"] - <= value - <= params["max_tokens"]["max"] - ): - raise exception_class( - f"max_tokens must be between {params['max_tokens']['min']} and {params['max_tokens']['max']} and should not be 0!" - ) - elif key == "logit_bias" and not isinstance(value, dict): - raise exception_class("logit_bias must be a dictionary!") - elif key == "stop": - exc_msg = "Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers." - if value and not isinstance(value, (str, int, list)): - raise exception_class(exc_msg) - elif value and (isinstance(value, list) and len(value) > 4): - raise exception_class(exc_msg) + def validate_llm_hyperparameters(hyperparameters: dict, llm_type: str, exception_class): + from jsonschema_rs import JSONSchema, ValidationError + schema = Utility.system_metadata["llm"][llm_type] + try: + validator = JSONSchema(schema) + validator.validate(hyperparameters) + except ValidationError as e: + message = f"{e.instance_path}: {e.message}" + raise exception_class(message) + @staticmethod def create_uuid_from_string(val: str): diff --git a/kairon/shared/vector_embeddings/db/base.py b/kairon/shared/vector_embeddings/db/base.py index 178ee25de..d1c2a1e97 100644 --- a/kairon/shared/vector_embeddings/db/base.py +++ b/kairon/shared/vector_embeddings/db/base.py @@ -8,16 +8,16 @@ class VectorEmbeddingsDbBase(ABC): @abstractmethod - async def embedding_search(self, request_body: Dict): + async def embedding_search(self, request_body: Dict, **kwargs): raise NotImplementedError("Provider not implemented") @abstractmethod - async def payload_search(self, request_body: Dict): + async def payload_search(self, request_body: Dict, **kwargs): raise NotImplementedError("Provider not implemented") - async def perform_operation(self, op_type: Text, request_body: Dict): + async def perform_operation(self, op_type: Text, request_body: Dict, **kwargs): supported_ops = {DbActionOperationType.payload_search.value: self.payload_search, DbActionOperationType.embedding_search.value: self.embedding_search} if op_type not in supported_ops.keys(): raise AppException("Operation type not supported") - return await supported_ops[op_type](request_body) + return await supported_ops[op_type](request_body, **kwargs) diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index d2ff7e69c..893a310ad 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -8,7 +8,7 @@ from kairon.shared.admin.constants import BotSecretType from kairon.shared.admin.processor import Sysadmin from kairon.shared.constants import GPT3ResourceTypes -from kairon.shared.llm.clients.factory import LLMClientFactory +from kairon.shared.llm.processor import LLMProcessor from kairon.shared.vector_embeddings.db.base import VectorEmbeddingsDbBase @@ -25,9 +25,7 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.llm_settings = llm_settings - self.api_key = Sysadmin.get_bot_secret(self.bot, BotSecretType.gpt_key.value, raise_err=True) - self.client = LLMClientFactory.get_resource_provider(llm_settings["provider"])(self.api_key, - **self.llm_settings) + self.llm = LLMProcessor(self.bot) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 @@ -38,18 +36,16 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def __get_embedding(self, text: Text) -> List[float]: - truncated_text = self.truncate_text(text) - result, _ = await self.client.invoke(GPT3ResourceTypes.embeddings.value, model="text-embedding-3-small", - input=truncated_text) + async def __get_embedding(self, text: Text, **kwargs) -> List[float]: + result, _ = await self.llm.get_embedding(text, user=kwargs.get('user'), bot=kwargs.get('bot')) return result - async def embedding_search(self, request_body: Dict): + async def embedding_search(self, request_body: Dict, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points") if request_body.get("text"): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/search") user_msg = request_body.get("text") - vector = await self.__get_embedding(user_msg) + vector = await self.__get_embedding(user_msg, **kwargs) request_body = {'vector': vector, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} embedding_search_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', @@ -57,7 +53,7 @@ async def embedding_search(self, request_body: Dict): request_body=request_body) return embedding_search_result - async def payload_search(self, request_body: Dict): + async def payload_search(self, request_body: Dict, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/scroll") payload_filter_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', diff --git a/kairon/train.py b/kairon/train.py index 05e761dc5..0276f7bc5 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -13,10 +13,10 @@ from kairon.shared.data.constant import EVENT_STATUS from kairon.shared.data.model_processor import ModelProcessor from kairon.shared.data.processor import MongoProcessor -from kairon.shared.llm.factory import LLMFactory from kairon.shared.metering.constants import MetricType from kairon.shared.metering.metering_processor import MeteringProcessor from kairon.shared.utils import Utility +from kairon.shared.llm.processor import LLMProcessor def train_model_for_bot(bot: str): @@ -81,6 +81,7 @@ def train_model_for_bot(bot: str): raise AppException(e) return model + def start_training(bot: str, user: str, token: str = None): """ prevents training of the bot, @@ -100,8 +101,8 @@ def start_training(bot: str, user: str, token: str = None): settings = processor.get_bot_settings(bot, user) settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: - llm = LLMFactory.get_instance("faq")(bot, settings["llm_settings"]) - faqs = asyncio.run(llm.train()) + llm_processor = LLMProcessor(bot) + faqs = asyncio.run(llm_processor.train(user=user, bot=bot)) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) agent_url = Utility.environment['model']['agent'].get('url') diff --git a/metadata/integrations.yml b/metadata/integrations.yml index f65e67b57..227b4c413 100644 --- a/metadata/integrations.yml +++ b/metadata/integrations.yml @@ -95,59 +95,74 @@ live_agents: websocket_url: wss://app.chatwoot.com/cable llm: - gpt: - temperature: - type: float - default: 0.0 - min: 0.0 - max: 2.0 - description: "The temperature hyperparameter controls the creativity or randomness of the generated responses." - max_tokens: - type: int - default: 300 - min: 5 - max: 4096 - description: "The max_tokens hyperparameter limits the length of generated responses in chat completion using ChatGPT." - model: - type: str - default: "gpt-3.5-turbo" - description: "The model hyperparameter is the ID of the model to use such as gpt-2, gpt-3, or a custom model that you have trained or fine-tuned." - top_p: - type: float - default: 0.0 - min: 0.0 - max: 1.0 - description: "The top_p hyperparameter is a value that controls the diversity of the generated responses." - n: - type: int - default: 1 - min: 1 - max: 5 - description: "The n hyperparameter controls the number of different response options that are generated by the model." - stream: - type: bool - default: false - description: "the stream hyperparameter controls whether the model generates a continuous stream of text or a single response." - stop: - type: - - str - - array - - int - default: null - description: "The stop hyperparameter is used to specify a list of tokens that should be used to indicate the end of a generated response." - presence_penalty: - type: float - default: 0.0 - min: -2.0 - max: 2.0 - description: "The presence_penalty hyperparameter penalizes the model for generating words that are not present in the context or input prompt. " - frequency_penalty: - type: float - default: 0.0 - min: -2.0 - max: 2.0 - description: "The frequency_penalty hyperparameter penalizes the model for generating words that have already been generated in the current response." - logit_bias: - type: dict - default: {} - description: "The logit_bias hyperparameter helps prevent GPT-3 from generating unwanted tokens or even to encourage generation of tokens that you do want. " + openai: + $schema: "https://json-schema.org/draft/2020-12/schema" + type: object + description: "Open AI Models for Prompt" + properties: + temperature: + type: number + default: 0.0 + minimum: 0.0 + maximum: 2.0 + description: "The temperature hyperparameter controls the creativity or randomness of the generated responses." + max_tokens: + type: integer + default: 300 + minimum: 5 + maximum: 4096 + description: "The max_tokens hyperparameter limits the length of generated responses in chat completion using ChatGPT." + model: + type: string + default: "gpt-3.5-turbo" + enum: ["gpt-3.5-turbo", "gpt-3.5-turbo-instruct"] + description: "The model hyperparameter is the ID of the model to use such as gpt-2, gpt-3, or a custom model that you have trained or fine-tuned." + top_p: + type: number + default: 0.0 + minimum: 0.0 + maximum: 1.0 + description: "The top_p hyperparameter is a value that controls the diversity of the generated responses." + n: + type: integer + default: 1 + minimum: 1 + maximum: 5 + description: "The n hyperparameter controls the number of different response options that are generated by the model." + stream: + type: boolean + default: false + description: "the stream hyperparameter controls whether the model generates a continuous stream of text or a single response." + stop: + anyOf: + - type: "string" + - type: "array" + maxItems: 4 + items: + type: "string" + - type: "integer" + - type: "null" + + type: + - "string" + - "array" + - "integer" + - "null" + default: null + description: "The stop hyperparameter is used to specify a list of tokens that should be used to indicate the end of a generated response." + presence_penalty: + type: number + default: 0.0 + minimum: -2.0 + maximum: 2.0 + description: "The presence_penalty hyperparameter penalizes the model for generating words that are not present in the context or input prompt. " + frequency_penalty: + type: number + default: 0.0 + minimum: -2.0 + maximum: 2.0 + description: "The frequency_penalty hyperparameter penalizes the model for generating words that have already been generated in the current response." + logit_bias: + type: object + default: {} + description: "The logit_bias hyperparameter helps prevent GPT-3 from generating unwanted tokens or even to encourage generation of tokens that you do want. " diff --git a/requirements/dev.txt b/requirements/dev.txt index d411ab022..19268e061 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,16 +1,15 @@ -r prod.txt -pytest==8.1.1 +pytest==8.2.2 pytest_httpx==0.30.0 -pytest-asyncio==0.23.6 -responses==0.25.0 +pytest-asyncio==0.23.7 +responses==0.25.2 mock==5.1.0 -moto[all]==5.0.5 +moto[all]==5.0.9 mongomock==4.1.2 black==22.12.0 -locust==2.25.0 +locust==2.29.0 deepdiff==7.0.1 pytest-cov==5.0.0 pytest-html==4.1.1 pytest-aioresponses==0.2.0 -aioresponses==0.7.6 -pykwalify==1.8.0 \ No newline at end of file +aioresponses==0.7.6 \ No newline at end of file diff --git a/requirements/prod.txt b/requirements/prod.txt index 19c6817ff..e00d448a9 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1,65 +1,69 @@ rasa[full]==3.6.20 mongoengine==0.28.2 fastapi==0.110.2 -uvicorn[standard]==0.29.0 +uvicorn[standard]==0.30.1 smart-config==0.1.3 fastapi_sso==0.9.1 fastapi-keycloak==1.0.10 pykka==3.1.1 zenpy==2.0.42 -validators==0.28.0 +validators==0.28.3 secure==0.3.0 password-strength==0.0.3.post2 beautifulsoup4==4.12.3 uuid6==2024.01.12 passlib[bcrypt]==1.7.4 -openai==0.28.1 json2html==1.3.0 -google-api-python-client==2.110.0 -jira==3.5.2 +google-api-python-client==2.133.0 +jira==3.8.0 pipedrive-python-lib==1.2.3 -google-cloud-translate==3.13.0 -blinker==1.7.0 -pymupdf==1.23.7 -python-docx==1.1.0 +google-cloud-translate==3.15.3 +blinker==1.8.2 +pymupdf==1.24.5 +python-docx==1.1.2 python-multipart==0.0.9 pandas==2.2.2 -openpyxl==3.1.2 +openpyxl==3.1.4 sentencepiece==0.1.99 -dramatiq==1.15.0 +dramatiq==1.17.0 dramatiq-mongodb==0.8.3 nlpaug==1.1.11 keybert==0.8.4 -pyTelegramBotAPI==4.17.0 +pyTelegramBotAPI==4.19.1 APScheduler==3.9.1.post1 -croniter==2.0.3 +croniter==2.0.5 faiss-cpu==1.8.0 -tiktoken==0.6.0 +tiktoken==0.7.0 RestrictedPython==7.1 -AccessControl==6.3 +AccessControl==7.0 timeout-decorator==0.5.0 -googlesearch-python==1.2.3 +googlesearch-python==1.2.4 aiohttp-retry==2.8.3 pqdict==1.4.0 google-businessmessages==1.0.5 google-apitools==0.5.32 -orjson==3.10.1 -opentelemetry-distro[otlp]==0.45b0 +orjson==3.10.5 +opentelemetry-distro[otlp]==0.46b0 opentelemetry-sdk-extension-aws==2.0.1 opentelemetry-propagator-aws-xray==1.0.1 -opentelemetry-instrumentation-fastapi==0.45b0 -opentelemetry-instrumentation-aiohttp-client==0.45b0 -opentelemetry-instrumentation-asyncio==0.45b0 -opentelemetry-instrumentation-aws-lambda==0.45b0 -opentelemetry-instrumentation-boto==0.45b0 -opentelemetry-instrumentation-botocore==0.45b0 -opentelemetry-instrumentation-httpx==0.45b0 -opentelemetry-instrumentation-logging==0.45b0 -opentelemetry-instrumentation-pymongo==0.45b0 -opentelemetry-instrumentation-requests==0.45b0 -opentelemetry-instrumentation-system-metrics==0.45b0 -opentelemetry-instrumentation-grpc==0.45b0 -opentelemetry-instrumentation-sklearn==0.45b0 -opentelemetry-instrumentation-asgi==0.45b0 +opentelemetry-instrumentation-fastapi==0.46b0 +opentelemetry-instrumentation-aiohttp-client==0.46b0 +opentelemetry-instrumentation-asyncio==0.46b0 +opentelemetry-instrumentation-aws-lambda==0.46b0 +opentelemetry-instrumentation-boto==0.46b0 +opentelemetry-instrumentation-botocore==0.46b0 +opentelemetry-instrumentation-httpx==0.46b0 +opentelemetry-instrumentation-logging==0.46b0 +opentelemetry-instrumentation-pymongo==0.46b0 +opentelemetry-instrumentation-requests==0.46b0 +opentelemetry-instrumentation-system-metrics==0.46b0 +opentelemetry-instrumentation-grpc==0.46b0 +opentelemetry-instrumentation-sklearn==0.46b0 +opentelemetry-instrumentation-asgi==0.46b0 +opentelemetry-instrumentation-requests==0.46b0 +opentelemetry-instrumentation-sklearn==0.46b0 pykwalify==1.8.0 gunicorn==22.0.0 +litellm==1.38.11 +jsonschema_rs==0.18.0 +mongoengine-jsonschema==0.1.3 \ No newline at end of file diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index 2012234ba..1941a8fd6 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -2,6 +2,7 @@ import os from urllib.parse import urlencode, urljoin +import litellm import mock import numpy as np import pytest @@ -10,8 +11,11 @@ from deepdiff import DeepDiff from fastapi.testclient import TestClient from jira import JIRAError -from mock import patch -from mongoengine import connect, DoesNotExist +from mongoengine import connect + +from kairon.shared.utils import Utility + +Utility.load_system_metadata() from kairon.actions.definitions.live_agent import ActionLiveAgent from kairon.actions.definitions.set_slot import ActionSetSlot @@ -32,15 +36,13 @@ DEFAULT_NLU_FALLBACK_RESPONSE from kairon.shared.data.data_objects import Slots, KeyVault, BotSettings, LLMSettings from kairon.shared.data.processor import MongoProcessor -from kairon.shared.llm.clients.gpt3 import GPT3Resources -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding -from kairon.shared.utils import Utility from kairon.shared.vector_embeddings.db.qdrant import Qdrant os.environ['ASYNC_TEST_TIMEOUT'] = "360" os.environ["system_file"] = "./tests/testing_data/system.yaml" client = TestClient(action) +OPENAI_EMBEDDING_OUTPUT = 1536 @pytest.fixture(autouse=True, scope='class') @@ -82,7 +84,8 @@ def test_live_agent_action_execution(aioresponses): aioresponses.add( method="POST", url=f"{Utility.environment['live_agent']['url']}/conversation/request", - payload={"success": True, "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": None }, "message": None, "error_code": 0}, + payload={"success": True, "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": None}, "message": None, + "error_code": 0}, body={'bot_id': '5f50fd0a56b698ca10d35d2z', 'sender_id': 'default', 'channel': 'messenger'}, status=200 ) @@ -185,7 +188,9 @@ def test_live_agent_action_execution_no_agent_available(aioresponses): aioresponses.add( method="POST", url=f"{Utility.environment['live_agent']['url']}/conversation/request", - payload={"success": True, "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": "live agent is not available" }, "message": None, "error_code": 0}, + payload={"success": True, + "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": "live agent is not available"}, + "message": None, "error_code": 0}, body={'bot_id': '5f50fd0a56b698ca10d35d2z', 'sender_id': 'default', 'channel': 'messenger'}, status=200 ) @@ -275,7 +280,6 @@ def test_live_agent_action_execution_no_agent_available(aioresponses): assert response_json['responses'][0]['text'] == 'live agent is not available' - def test_live_agent_action_execution_with_exception(aioresponses): bot_settings = BotSettings(bot='5f50fd0a56b698ca10d35d21', user='user') bot_settings.live_agent_enabled = True @@ -384,7 +388,9 @@ def test_live_agent_action_execution_with_exception(aioresponses): assert response.status_code == 200 assert len(response_json['responses']) == 1 assert response_json['responses'][0]['text'] == 'Connecting to live agent' - assert response_json == {'events': [], 'responses': [{'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}]} + assert response_json == {'events': [], 'responses': [ + {'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, + 'response': None, 'image': None, 'attachment': None}]} def test_live_agent_action_execution_with_exception(aioresponses): @@ -495,11 +501,12 @@ def test_live_agent_action_execution_with_exception(aioresponses): assert response.status_code == 200 assert len(response_json['responses']) == 1 assert response_json['responses'][0]['text'] == 'Connecting to live agent' - assert response_json == {'events': [], 'responses': [{'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}]} + assert response_json == {'events': [], 'responses': [ + {'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, + 'response': None, 'image': None, 'attachment': None}]} def test_retrieve_config_failure(): - patch('kairon.actions.definitions.live_agent.LiveAgentActionConfig.objects().get', side_effect=DoesNotExist) action_live_agent = ActionLiveAgent(bot='test_bot', name='test_action') with pytest.raises(ActionFailure, match="No Live Agent action found for given action and bot"): action_live_agent.retrieve_config() @@ -532,14 +539,22 @@ def test_pyscript_action_execution(): json={"success": True, "data": {"bot_response": {'numbers': [1, 2, 3, 4, 5], 'total': 15, 'i': 5}, "slots": {"location": "Bangalore", "langauge": "Kannada"}, "type": "json"}, "message": None, "error_code": 0}, - match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'chat_log': [], 'intent': 'pyscript_action', - 'kairon_user_msg': None, 'key_vault': {}, 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, - 'sender_id': 'default', 'session_started': None, - 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'langauge': 'Kannada', 'location': 'Bangalore'}, - 'user_message': 'get intents'} - - })] + match=[responses.matchers.json_params_matcher({'source_code': script, + 'predefined_objects': {'chat_log': [], + 'intent': 'pyscript_action', + 'kairon_user_msg': None, 'key_vault': {}, + 'latest_message': {'intent_ranking': [ + {'name': 'pyscript_action'}], + 'text': 'get intents'}, + 'sender_id': 'default', + 'session_started': None, + 'slot': { + 'bot': '5f50fd0a56b698ca10d35d2z', + 'langauge': 'Kannada', + 'location': 'Bangalore'}, + 'user_message': 'get intents'} + + })] ) request_object = { @@ -665,6 +680,7 @@ def test_pyscript_action_execution_with_multiple_utterances(): assert response_json['responses'][0]['custom'] == {'text': 'Hello!'} assert response_json['responses'][1]['text'] == 'How can I help you?' + @responses.activate def test_pyscript_action_execution_with_multiple_integer_utterances(): import textwrap @@ -778,7 +794,9 @@ def test_pyscript_action_execution_with_bot_response_none(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -853,7 +871,9 @@ def test_pyscript_action_execution_with_type_json_bot_response_none(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -928,7 +948,9 @@ def test_pyscript_action_execution_with_type_json_bot_response_str(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1005,7 +1027,9 @@ def test_pyscript_action_execution_with_other_type(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1080,7 +1104,9 @@ def test_pyscript_action_execution_with_slots_not_dict_type(): "slots": "invalid slots values"}, "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1175,7 +1201,7 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url(mock_trigger_l }, "version": "version" } - with patch("kairon.shared.utils.Utility.environment", new=mock_environment): + with mock.patch("kairon.shared.utils.Utility.environment", new=mock_environment): mock_trigger_lambda.return_value = \ {"Payload": {"body": {"bot_response": "Successfully Evaluated the pyscript", "slots": {"location": "Bangalore", "langauge": "Kannada"}}}, "StatusCode": 200} @@ -1185,15 +1211,17 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url(mock_trigger_l assert len(response_json['events']) == 3 assert len(response_json['responses']) == 1 assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'location', 'value': 'Bangalore'}, - {'event': 'slot', 'timestamp': None, 'name': 'langauge', 'value': 'Kannada'}, - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "Successfully Evaluated the pyscript"}] + {'event': 'slot', 'timestamp': None, 'name': 'location', 'value': 'Bangalore'}, + {'event': 'slot', 'timestamp': None, 'name': 'langauge', 'value': 'Kannada'}, + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "Successfully Evaluated the pyscript"}] assert response_json['responses'][0]['text'] == "Successfully Evaluated the pyscript" called_args = mock_trigger_lambda.call_args assert called_args.args[1] == \ {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {"bot": "5f50fd0a56b698ca10d35d2z", "location": "Bangalore", "langauge": "Kannada"}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, @@ -1252,7 +1280,7 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url_raise_exceptio }, "version": "version" } - with patch("kairon.shared.utils.Utility.environment", new=mock_environment): + with mock.patch("kairon.shared.utils.Utility.environment", new=mock_environment): mock_trigger_lambda.return_value = {"Payload": {"body": "Failed to evaluated the pyscript"}, "StatusCode": 422} response = client.post("/webhook", json=request_object) response_json = response.json() @@ -1260,8 +1288,8 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url_raise_exceptio assert len(response_json['events']) == 1 assert len(response_json['responses']) == 1 assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "I have failed to process your request"}] + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "I have failed to process your request"}] log = ActionServerLogs.objects(action=action_name).get().to_mongo().to_dict() assert log['exception'] == "Failed to evaluated the pyscript" @@ -1296,7 +1324,9 @@ def raise_custom_exception(request): "POST", Utility.environment['evaluator']['pyscript']['url'], callback=raise_custom_exception, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1307,7 +1337,7 @@ def raise_custom_exception(request): "tracker": { "sender_id": "default", "conversation_id": "default", - "slots": {"bot": "5f50fd0a56b698ca10d35d2z", "location": "Bangalore", "langauge": "Kannada"}, + "slots": {"bot": "5f50fd0a56b698ca10d35d2z", "location": "Bangalore", "langauge": "Kannada"}, "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'pyscript_action'}]}, "latest_event_time": 1537645578.314389, "followup_action": "action_listen", @@ -1367,7 +1397,9 @@ def test_pyscript_action_execution_with_invalid_response(): "error_code": 422}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1409,7 +1441,8 @@ def test_pyscript_action_execution_with_invalid_response(): {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': 'I have failed to process your request'}] log = ActionServerLogs.objects(action=action_name).get().to_mongo().to_dict() - assert log['exception'] == 'Pyscript evaluation failed: {\'success\': False, \'data\': None, \'message\': \'Script execution error: ("Line 2: SyntaxError: invalid syntax at statement: for i in 10",)\', \'error_code\': 422}' + assert log[ + 'exception'] == 'Pyscript evaluation failed: {\'success\': False, \'data\': None, \'message\': \'Script execution error: ("Line 2: SyntaxError: invalid syntax at statement: for i in 10",)\', \'error_code\': 422}' def test_http_action_execution(aioresponses): @@ -1509,8 +1542,42 @@ def test_http_action_execution(aioresponses): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'slots', 'data': [{'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, {'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', 'evaluation_type': 'expression', 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot_response_log': ['evaluation_type: expression', 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, {'type': 'api_call', 'headers': {'botid': '**********************2e', 'userid': '****', 'tag': '******ot', 'email': '*******************om'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot', 'name': 'udit', 'contact': ''}, 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'status_code': 200}, {'type': 'params_list', 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot', 'name': 'udit', 'contact': ''}, 'request_params': {'bot': '**********************2e', 'user': '1011', 'tag': '******ot', 'name': '****', 'contact': None}}, {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', 'expression: ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'response: red']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'slots', 'data': [ + {'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, + {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, + {'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + 'evaluation_type': 'expression', + 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot_response_log': ['evaluation_type: expression', + 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, + {'type': 'api_call', + 'headers': {'botid': '**********************2e', 'userid': '****', 'tag': '******ot', + 'email': '*******************om'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot', 'name': 'udit', + 'contact': ''}, + 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, + 'status_code': 200}, {'type': 'params_list', + 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', + 'tag': 'from_bot', 'name': 'udit', 'contact': ''}, + 'request_params': {'bot': '**********************2e', 'user': '1011', + 'tag': '******ot', 'name': '****', 'contact': None}}, + {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, + 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', + 'expression: ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', + 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'response: red']}] + assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} def test_http_action_execution_returns_custom_json(aioresponses): @@ -1973,8 +2040,41 @@ def test_http_action_execution_no_response_dispatch(aioresponses): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'slots', 'data': [{'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, {'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', 'evaluation_type': 'expression', 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot_response_log': ['evaluation_type: expression', 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot'}, 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'status_code': 200}, {'type': 'params_list', 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot'}, 'request_params': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': '******ot'}}, {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', 'expression: ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'response: red']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_no_response_dispatch', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'slots', 'data': [ + {'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, + {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, + {'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', + 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + 'evaluation_type': 'expression', + 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot_response_log': ['evaluation_type: expression', + 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, + {'type': 'api_call', + 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, + 'method': 'GET', 'url': 'http://localhost:8081/mock', + 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot'}, + 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, + 'status_code': 200}, {'type': 'params_list', + 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', + 'tag': 'from_bot'}, + 'request_params': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', + 'tag': '******ot'}}, + {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, + 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', + 'expression: ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', + 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'response: red']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_no_response_dispatch', 'sender': 'default', 'headers': {}, + 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2073,8 +2173,22 @@ def test_http_action_execution_script_evaluation(aioresponses): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'params_list', 'request_body': {}, 'request_params': {}}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {}, 'response': {'a': 10, 'b': { + 'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, + {'type': 'params_list', 'request_body': {}, 'request_params': {}}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation', 'sender': 'default', 'headers': {}, + 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2199,8 +2313,34 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_post(aiores if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'POST', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_post', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'POST', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'POST', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_post', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'POST', + 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', + 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2324,8 +2464,34 @@ def test_http_action_execution_script_evaluation_with_dynamic_params(aioresponse if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params', 'sender': 'default', + 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', + 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2450,8 +2616,31 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_returns_cus if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'json', 'data': 'bot_response = data', 'evaluation_type': 'script', 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'bot_response_log': ['evaluation_type: script', 'script: bot_response = data', "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_returns_custom_json', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': "{'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}", 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [ + {'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'json', 'data': 'bot_response = data', + 'evaluation_type': 'script', + 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, + 'bot_response_log': ['evaluation_type: script', 'script: bot_response = data', + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, + {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, + 'method': 'GET', 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, + 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_returns_custom_json', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': "{'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}", + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2576,8 +2765,34 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_no_response if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_no_response_dispatch', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_no_response_dispatch', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', + 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2835,7 +3050,7 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_and_params_ resp_msg = json.dumps(data_obj) aioresponses.add( method=responses.GET, - url=http_url+"?intent=test_run&sender_id=default&user_message=get+intents", + url=http_url + "?intent=test_run&sender_id=default&user_message=get+intents", body=resp_msg, status=200 ) @@ -2900,8 +3115,35 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_and_params_ if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert not DeepDiff(log, {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_and_params_list', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200}, ignore_order=True) + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert not DeepDiff(log, {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_and_params_list', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', + 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', + 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', + 'http_status_code': 200}, ignore_order=True) @responses.activate @@ -2941,7 +3183,7 @@ def test_http_action_execution_script_evaluation_failure_no_dispatch(aioresponse aioresponses.add( method=responses.GET, - url=http_url+"?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", + url=http_url + "?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", body=resp_msg, status=200 ) @@ -3039,7 +3281,7 @@ def test_http_action_execution_script_evaluation_failure_and_dispatch(aiorespons aioresponses.add( method=responses.GET, - url=http_url+"?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", + url=http_url + "?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", body=resp_msg, status=200, ) @@ -3198,12 +3440,12 @@ def test_http_action_execution_script_evaluation_failure_and_dispatch_2(aiorespo assert response_json['events'] == [ {"event": "slot", "timestamp": None, "name": "kairon_action_response", "value": "I have failed to process your request"}, - {"event": "slot", "timestamp": None, "name": "http_status_code", "value": 200},] + {"event": "slot", "timestamp": None, "name": "http_status_code", "value": 200}, ] assert response_json['responses'][0]['text'] == "I have failed to process your request" -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.http.ActionHTTP.retrieve_config") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.http.ActionHTTP.retrieve_config") @mock.patch("kairon.shared.rest_client.AioRestClient._AioRestClient__trigger", autospec=True) def test_http_action_failed_execution(mock_trigger_request, mock_action_config, mock_action): action_name = "test_run_with_get" @@ -3271,8 +3513,18 @@ def _get_action(*arge, **kwargs): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': 'The value of ${a.b.3} in ${a.b.d.0} is ${a.b.d}', 'evaluation_type': 'expression', 'exception': 'I have failed to process your request'}, {'type': 'api_call', 'headers': {}, 'method': 'GET', 'url': 'http://localhost:8800/mock', 'payload': {}, 'response': None, 'status_code': 408, 'exception': "Got non-200 status code:408 http_response:{'data': None, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 408}"}, {'type': 'params_list', 'request_body': {}, 'request_params': {}}, {'type': 'filled_slots'}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_run_with_get', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8800/mock', 'request_method': 'GET', 'bot_response': 'I have failed to process your request', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'FAILURE', 'fail_reason': 'Got non-200 status code:408 http_response:None', 'user_msg': 'get intents', 'time_elapsed': 0, 'http_status_code': 408} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': 'The value of ${a.b.3} in ${a.b.d.0} is ${a.b.d}', 'evaluation_type': 'expression', + 'exception': 'I have failed to process your request'}, + {'type': 'api_call', 'headers': {}, 'method': 'GET', 'url': 'http://localhost:8800/mock', + 'payload': {}, 'response': None, 'status_code': 408, + 'exception': "Got non-200 status code:408 http_response:{'data': None, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 408}"}, + {'type': 'params_list', 'request_body': {}, 'request_params': {}}, {'type': 'filled_slots'}] + assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_run_with_get', 'sender': 'default', + 'headers': {}, 'url': 'http://localhost:8800/mock', 'request_method': 'GET', + 'bot_response': 'I have failed to process your request', 'bot': '5f50fd0a56b698ca10d35d2e', + 'status': 'FAILURE', 'fail_reason': 'Got non-200 status code:408 http_response:None', + 'user_msg': 'get intents', 'time_elapsed': 0, 'http_status_code': 408} def test_http_action_missing_action_name(): @@ -3532,7 +3784,7 @@ def test_vectordb_action_execution_embedding_search_from_value(mock_embedding): BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value", bot="5f50fd0a56b698ca10d75d2e", user="user").save() embedding = list(np.random.random(Qdrant.__embedding__)) - mock_embedding.return_value = embedding + mock_embedding.return_value = {'data': [{'embedding': embedding}]} http_url = 'http://localhost:6333/collections/5f50fd0a56b698ca10d75d2e_test_vectordb_action_execution_faq_embd/points' resp_msg = json.dumps( @@ -3975,8 +4227,8 @@ def test_vectordb_action_execution_invalid_operation_type(): log.pop('timestamp') -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.database.ActionDatabase.retrieve_config") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.database.ActionDatabase.retrieve_config") def test_vectordb_action_failed_execution(mock_action_config, mock_action): action_name = "test_run_with_get_action" payload_body = {"ids": [0], "with_payload": True, "with_vector": True} @@ -3996,7 +4248,6 @@ def test_vectordb_action_failed_execution(mock_action_config, mock_action): BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value", bot="5f50fd0a56b697ca10d35d2e", user="user").save() - def _get_action_config(*arge, **kwargs): return action_config.to_mongo().to_dict(), bot_settings.to_mongo().to_dict() @@ -4166,9 +4417,9 @@ def _get_action(*arge, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "get_action") as mock_action: + with mock.patch.object(ActionUtility, "get_action") as mock_action: mock_action.side_effect = _get_action - with patch.object(ActionSetSlot, "retrieve_config") as mocked: + with mock.patch.object(ActionSetSlot, "retrieve_config") as mocked: mocked.side_effect = _get_action_config response = client.post("/webhook", json=request_object) response_json = response.json() @@ -4229,9 +4480,9 @@ def _get_action(*arge, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "get_action") as mock_action: + with mock.patch.object(ActionUtility, "get_action") as mock_action: mock_action.side_effect = _get_action - with patch.object(ActionSetSlot, "retrieve_config") as mocked: + with mock.patch.object(ActionSetSlot, "retrieve_config") as mocked: mocked.side_effect = _get_action_config response = client.post("/webhook", json=request_object) response_json = response.json() @@ -4291,9 +4542,9 @@ def _get_action(*arge, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "get_action") as mock_action: + with mock.patch.object(ActionUtility, "get_action") as mock_action: mock_action.side_effect = _get_action - with patch.object(ActionSetSlot, "retrieve_config") as mocked: + with mock.patch.object(ActionSetSlot, "retrieve_config") as mocked: mocked.side_effect = _get_action_config response = client.post("/webhook", json=request_object) response_json = response.json() @@ -5294,9 +5545,9 @@ def test_form_validation_action_with_is_required_true_and_semantics(): @responses.activate -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_script_evaluation(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['custom_text_mail'] = open('template/emails/custom_text_mail.html', 'rb').read().decode() @@ -5379,9 +5630,9 @@ def _get_action_config(*arge, **kwargs): assert str(args[2]).__contains__("Content-Type: text/html") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -5531,9 +5782,9 @@ def _get_action_config(*arge, **kwargs): assert str(args[2]).__contains__("Subject: default test") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_varied_utterances(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -5987,8 +6238,8 @@ def _get_action_config(*arge, **kwargs): assert logs.status == "SUCCESS" -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") def test_email_action_failed_execution(mock_action_config, mock_action): action_name = "test_run_email_action" action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") @@ -6374,7 +6625,7 @@ def _run_action(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6403,7 +6654,7 @@ def _run_action(*args, **kwargs): 'intent_ranking': [{'name': 'test_run'}], "entities": [{"value": "my custom text", "entity": KAIRON_USER_MSG_ENTITY}] } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6430,7 +6681,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["latest_message"] = { 'text': '/action_google_search', 'intent_ranking': [{'name': 'test_run'}] } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6469,7 +6720,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "what is Kanban?" - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6504,7 +6755,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "what is Kanban?" - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6541,7 +6792,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "what is Kanban?" - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6641,7 +6892,7 @@ def _run_action(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6744,7 +6995,7 @@ def _run_action(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6942,7 +7193,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7183,7 +7434,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7302,7 +7553,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7313,9 +7564,9 @@ def _perform_web_search(*args, **kwargs): {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': 'Data science combines math, statistics, programming, analytics, AI, and machine learning to uncover insights from data. Learn how data science works, what it entails, and how it differs from data science and BI.\nTo know more, please visit: What is Data Science? | IBM\n\nData science is an interdisciplinary field that uses algorithms, procedures, and processes to examine large amounts of data in order to uncover hidden patterns, generate insights, and direct decision-making.\nTo know more, please visit: What Is Data Science? Definition, Examples, Jobs, and More'}], 'responses': [{ - 'text': 'Data science combines math, statistics, programming, analytics, AI, and machine learning to uncover insights from data. Learn how data science works, what it entails, and how it differs from data science and BI.\nTo know more, please visit: What is Data Science? | IBM\n\nData science is an interdisciplinary field that uses algorithms, procedures, and processes to examine large amounts of data in order to uncover hidden patterns, generate insights, and direct decision-making.\nTo know more, please visit: What Is Data Science? Definition, Examples, Jobs, and More', - 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, - 'image': None, 'attachment': None}]} + 'text': 'Data science combines math, statistics, programming, analytics, AI, and machine learning to uncover insights from data. Learn how data science works, what it entails, and how it differs from data science and BI.\nTo know more, please visit: What is Data Science? | IBM\n\nData science is an interdisciplinary field that uses algorithms, procedures, and processes to examine large amounts of data in order to uncover hidden patterns, generate insights, and direct decision-making.\nTo know more, please visit: What Is Data Science? Definition, Examples, Jobs, and More', + 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, + 'image': None, 'attachment': None}]} log = ActionServerLogs.objects(bot=bot, type=ActionType.web_search_action.value, status="SUCCESS").get() assert log['user_msg'] == '/action_public_search' @@ -7343,7 +7594,7 @@ def _perform_web_search(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "What is Python?" - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7446,7 +7697,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7550,7 +7801,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7573,7 +7824,7 @@ def test_process_jira_action(): def _mock_response(*args, **kwargs): return None - with patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_response): + with mock.patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_response): Actions(name=action_name, type=ActionType.jira_action.value, bot=bot, user=user).save() JiraAction( name=action_name, bot=bot, user=user, url='https://test-digite.atlassian.net', @@ -7660,7 +7911,7 @@ def _mock_response(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "create_jira_issue") as mocked: + with mock.patch.object(ActionUtility, "create_jira_issue") as mocked: mocked.side_effect = _mock_response response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7682,7 +7933,7 @@ def _mock_validation(*args, **kwargs): def _mock_response(*args, **kwargs): raise JIRAError(status_code=404, url='https://test-digite.atlassian.net') - with patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_validation): + with mock.patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_validation): Actions(name=action_name, type=ActionType.jira_action.value, bot=bot, user='test_user').save() JiraAction( name=action_name, bot=bot, user=user, url='https://test-digite.atlassian.net', @@ -7769,7 +8020,7 @@ def _mock_response(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "create_jira_issue") as mocked: + with mock.patch.object(ActionUtility, "create_jira_issue") as mocked: mocked.side_effect = _mock_response response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7965,7 +8216,7 @@ def test_process_zendesk_action(): user = 'test_user' Actions(name=action_name, type=ActionType.zendesk_action.value, bot=bot, user='test_user').save() - with patch('zenpy.Zenpy'): + with mock.patch('zenpy.Zenpy'): ZendeskAction(name=action_name, subdomain='digite751', user_name='udit.pandey@digite.com', api_token=CustomActionRequestParameters(value='1234567890'), subject='new ticket', response='ticket created', @@ -8050,7 +8301,7 @@ def test_process_zendesk_action(): "version": "version" } - with patch('zenpy.Zenpy'): + with mock.patch('zenpy.Zenpy'): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -8067,7 +8318,7 @@ def test_process_zendesk_action_failure(): user = 'test_user' Actions(name=action_name, type=ActionType.zendesk_action.value, bot=bot, user='test_user').save() - with patch('zenpy.Zenpy'): + with mock.patch('zenpy.Zenpy') as zen: ZendeskAction(name=action_name, subdomain='digite751', user_name='udit.pandey@digite.com', api_token=CustomActionRequestParameters(value='1234567890'), subject='new ticket', response='ticket created', @@ -8156,8 +8407,8 @@ def __mock_zendesk_error(*args, **kwargs): from zenpy.lib.exception import APIException raise APIException({"error": {"title": "No help desk at digite751.zendesk.com"}}) - with patch('zenpy.Zenpy') as mock: - mock.side_effect = __mock_zendesk_error + with mock.patch('zenpy.Zenpy') as zen: + zen.side_effect = __mock_zendesk_error response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -8263,7 +8514,7 @@ def test_process_pipedrive_leads_action(): user = 'test_user' Actions(name=action_name, type=ActionType.pipedrive_leads_action.value, bot=bot, user='test_user').save() - with patch('pipedrive.client.Client'): + with mock.patch('pipedrive.client.Client'): metadata = {'name': 'name', 'org_name': 'organization', 'email': 'email', 'phone': 'phone'} PipedriveLeadsAction(name=action_name, domain='https://digite751.pipedrive.com/', api_token=CustomActionRequestParameters(value='1234567890'), @@ -8362,10 +8613,10 @@ def __mock_create_leads(*args, **kwargs): def __mock_create_note(*args, **kwargs): return {"success": True, "data": {"id": 2}} - with patch('pipedrive.organizations.Organizations.create_organization', __mock_create_organization): - with patch('pipedrive.persons.Persons.create_person', __mock_create_person): - with patch('pipedrive.leads.Leads.create_lead', __mock_create_leads): - with patch('pipedrive.notes.Notes.create_note', __mock_create_note): + with mock.patch('pipedrive.organizations.Organizations.create_organization', __mock_create_organization): + with mock.patch('pipedrive.persons.Persons.create_person', __mock_create_person): + with mock.patch('pipedrive.leads.Leads.create_lead', __mock_create_leads): + with mock.patch('pipedrive.notes.Notes.create_note', __mock_create_note): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -8539,7 +8790,7 @@ def test_process_pipedrive_leads_action_failure(): user = 'test_user' Actions(name=action_name, type=ActionType.pipedrive_leads_action.value, bot=bot, user='test_user').save() - with patch('pipedrive.client.Client'): + with mock.patch('pipedrive.client.Client'): metadata = {'name': 'name', 'org_name': 'organization', 'email': 'email', 'phone': 'phone'} PipedriveLeadsAction(name=action_name, domain='https://digite751.pipedrive.com/', api_token=CustomActionRequestParameters(value='1234567890'), @@ -8629,7 +8880,7 @@ def __mock_pipedrive_error(*args, **kwargs): from pipedrive.exceptions import BadRequestError raise BadRequestError('Invalid request raised', {'error_code': 402}) - with patch('pipedrive.organizations.Organizations.create_organization', __mock_pipedrive_error): + with mock.patch('pipedrive.organizations.Organizations.create_organization', __mock_pipedrive_error): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -9074,7 +9325,7 @@ def _mock_search(*args, **kwargs): {"text": "yes", "payload": "yes"}]: yield result - with patch.object(MongoProcessor, "search_training_examples") as mock_action: + with mock.patch.object(MongoProcessor, "search_training_examples") as mock_action: mock_action.side_effect = _mock_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -9086,7 +9337,7 @@ def _mock_search(*args, **kwargs): for _ in []: yield - with patch.object(MongoProcessor, "search_training_examples") as mock_action: + with mock.patch.object(MongoProcessor, "search_training_examples") as mock_action: mock_action.side_effect = _mock_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -9977,7 +10228,7 @@ def __mock_error(*args, **kwargs): "e2e_actions": []}, "version": "2.8.15" } - with patch.object(ActionUtility, "trigger_rephrase") as mock_utils: + with mock.patch.object(ActionUtility, "trigger_rephrase") as mock_utils: mock_utils.side_effect = __mock_error response = client.post("/webhook", json=request_object) @@ -10166,7 +10417,7 @@ async def mock_process_actions(*args, **kwargs): from rasa_sdk import ActionExecutionRejection raise ActionExecutionRejection("Action Execution Rejection") - with patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): + with mock.patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'error': "Custom action 'Action Execution Rejection' rejected execution.", @@ -10176,18 +10427,17 @@ async def mock_process_actions(*args, **kwargs): from rasa_sdk.interfaces import ActionNotFoundException raise ActionNotFoundException("Action Not Found Exception") - with patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): + with mock.patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'error': "No registered action found for name 'Action Not Found Exception'.", 'action_name': 'Action Not Found Exception'} -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_prompt_question_from_slot(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_response_action_with_prompt_question_from_slot" @@ -10211,9 +10461,9 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10241,31 +10491,20 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[ - 3] == "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n" - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': []} - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action" @@ -10289,9 +10528,9 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10319,32 +10558,21 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[ - 3] == "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n" - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': []} - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_bot_responses_with_instructions(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_with_bot_responses_with_instructions" @@ -10369,9 +10597,9 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10400,31 +10628,20 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[ - 3] == "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n" - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': ['Answer in a short way.', 'Keep it simple.']} - - -@mock.patch.object(GPT3Resources, "invoke", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_response_action_with_query_prompt" @@ -10448,20 +10665,18 @@ def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embed 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}, {'name': 'Query Prompt', - 'data': 'If there is no specific query, assume that user is aking about java programming.', + 'data': 'If there is no specific query, assume that user is asking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True} ] - mock_completion_for_query_prompt = rephrased_query, { - 'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]} + mock_completion_for_query_prompt = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]} - mock_completion_for_answer = generated_text, { - 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion_for_answer = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_completion.side_effect = [mock_completion_for_query_prompt, mock_completion_for_answer] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10496,10 +10711,9 @@ def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embed 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Explain python is called high level programming language in laymen terms? \nA:"}] -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = GPT_LLM_FAQ @@ -10521,7 +10735,7 @@ def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): }, {'name': 'Data science prompt', 'instructions': 'Answer question based on the context above.', 'type': 'user', 'source': 'bot_content', - 'data': 'data_science'} + 'data': 'data_science'}, ] aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -10537,11 +10751,14 @@ def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): status=200, payload={ 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content_two}}]}) - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() - PromptAction(name=action_name, bot=bot, user=user, llm_prompts=llm_prompts).save() + PromptAction(name=action_name, + bot=bot, + user=user, + llm_prompts=llm_prompts).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() @@ -10561,11 +10778,10 @@ def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): ] -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_response_action_with_instructions(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = 'test_prompt_response_action_with_instructions' @@ -10586,9 +10802,9 @@ def test_prompt_response_action_with_instructions(mock_search, mock_embedding, m 'is_enabled': True } ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10612,11 +10828,10 @@ def test_prompt_response_action_with_instructions(mock_search, mock_embedding, m ] -@mock.patch.object(GPT3Resources, "invoke", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = GPT_LLM_FAQ @@ -10642,9 +10857,9 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text, generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10666,22 +10881,16 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', - 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above.\n \nQ: What kind of language is python? \nA:"}], - 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': True, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} - assert mock_completion.call_args.args[1] == 'chat/completions' - - -@patch("kairon.shared.llm.gpt3.openai.ChatCompletion.create", autospec=True) -@patch("kairon.shared.llm.gpt3.openai.Embedding.create", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) -def test_prompt_response_action_failure(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandeyy', 'bot': '5f50k90a56b698ca10d35d2e'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': True, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args.kwargs, expected, ignore_order=True) + + +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +def test_prompt_response_action_failure(mock_search): from uuid6 import uuid7 action_name = GPT_LLM_FAQ @@ -10690,10 +10899,7 @@ def test_prompt_response_action_failure(mock_search, mock_embedding, mock_comple user_msg = "What kind of language is python?" bot_content = "Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected." generated_text = "I don't know." - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = convert_to_openai_object(OpenAIResponse({'data': [{'embedding': embedding}]}, {})) - mock_completion.return_value = convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10764,13 +10970,10 @@ def test_prompt_action_response_action_does_not_exists(): assert len(response_json['responses']) == 0 -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_static_user_prompt(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -10799,8 +11002,7 @@ def test_prompt_action_response_action_with_static_user_prompt(mock_search, mock ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_search_cache(*args, **kwargs): return {'result': []} @@ -10813,9 +11015,9 @@ def __mock_cache_result(*args, **kwargs): mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.side_effect = [__mock_search_cache(), __mock_fetch_similar(), __mock_cache_result()] Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -10845,13 +11047,10 @@ def __mock_cache_result(*args, **kwargs): @responses.activate -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.llm.gpt3.GPT3FAQEmbedding.__collection_search__", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.llm.processor.LLMProcessor.__collection_search__", autospec=True) def test_prompt_action_response_action_with_action_prompt(mock_search, mock_embedding, mock_completion, aioresponses): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -10918,16 +11117,15 @@ def test_prompt_action_response_action_with_action_prompt(mock_search, mock_embe ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -10955,24 +11153,29 @@ def __mock_fetch_similar(*args, **kwargs): assert response_json['responses'] == [ {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}] - log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, status="SUCCESS").get() - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - with open('tests/testing_data/actions/action_prompt.txt', 'r') as file: - prompt_data = file.read() - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[3] == prompt_data - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, + status="SUCCESS").get().to_mongo().to_dict() + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) @mock.patch.object(ActionUtility, "perform_google_search", autospec=True) def test_kairon_faq_response_with_google_search_prompt(mock_google_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse - action_name = "kairon_faq_action" google_action_name = "custom_search_action" bot = "5u08kd0a56b698ca10hgjgjkhgjks" @@ -11013,12 +11216,11 @@ def _run_action(*args, **kwargs): PromptAction(name=action_name, bot=bot, user=user, llm_prompts=llm_prompts).save() def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - mock_completion.return_value = generated_text - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_google_search.side_effect = _run_action request_object = json.load(open("tests/testing_data/actions/action-request.json")) @@ -11035,12 +11237,24 @@ def mock_completion_for_answer(*args, **kwargs): 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}] - log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, status="SUCCESS").get() - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What is kanban' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - assert mock_completion.call_args.args[ - 3] == 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n' + log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, + status="SUCCESS").get().to_mongo().to_dict() + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Kanban is a workflow management tool which visualizes both the process (the workflow) and the actual work passing through that process.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], + 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) def test_prompt_response_action_with_action_not_found(): @@ -11072,13 +11286,10 @@ def test_prompt_response_action_with_action_not_found(): log['exception'] = 'No action found for given bot and name' -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_dispatch_response_disabled(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -11102,17 +11313,16 @@ def test_prompt_action_dispatch_response_disabled(mock_search, mock_embedding, m ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -11154,23 +11364,28 @@ def __mock_fetch_similar(*args, **kwargs): {'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.'} }, {'type': 'slots_to_fill', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']} ] - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What is the name of prompt?' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - with open('tests/testing_data/actions/slot_prompt.txt', 'r') as file: - prompt_data = file.read() - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[3] == prompt_data - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.actions.utils.ActionUtility.compose_response", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.compose_response", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_set_slots(mock_search, mock_slot_set, mock_mock_embedding, mock_completion): - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse - action_name = "kairon_faq_action" bot = "5u80fd0a56c908ca10d35d2sjhjhjhj" user = "udit.pandey" @@ -11193,11 +11408,10 @@ def test_prompt_action_set_slots(mock_search, mock_slot_set, mock_mock_embedding ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_completion.return_value = mock_completion_for_answer() - mock_completion.return_value = generated_text + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} log1 = ['Slot: api_type', 'evaluation_type: expression', f"data: {generated_text}", 'response: filter'] log2 = ['Slot: query', 'evaluation_type: expression', f"data: {generated_text}", 'response: {\"must\": [{\"key\": \"Date Added\", \"match\": {\"value\": 1673721000.0}}]}'] @@ -11246,26 +11460,38 @@ def mock_completion_for_answer(*args, **kwargs): assert events == [ {'type': 'llm_response', 'response': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', - 'llm_response_log': {'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}'}}, - {'type': 'slots_to_fill', 'data': {'api_type': 'filter', 'query': '{"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}'}, - 'slot_eval_log': ['initiating slot evaluation', 'Slot: api_type', 'Slot: api_type', 'evaluation_type: expression', + 'llm_response_log': { + 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}'}}, + {'type': 'slots_to_fill', + 'data': {'api_type': 'filter', 'query': '{"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}'}, + 'slot_eval_log': ['initiating slot evaluation', 'Slot: api_type', 'Slot: api_type', + 'evaluation_type: expression', 'data: {"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'response: filter', 'Slot: query', 'Slot: query', 'evaluation_type: expression', 'data: {"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'response: {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}']} ] - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == user_msg - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], + 'raw_completion_response': {'choices': [{'message': { + 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_slot_prompt(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -11289,17 +11515,16 @@ def test_prompt_action_response_action_slot_prompt(mock_search, mock_embedding, ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -11344,22 +11569,27 @@ def __mock_fetch_similar(*args, **kwargs): {'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.'} }, {'type': 'slots_to_fill', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']} ] - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What is the name of prompt?' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - with open('tests/testing_data/actions/slot_prompt.txt', 'r') as file: - prompt_data = file.read() - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[3] == prompt_data - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_user_message_in_slot(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -11379,17 +11609,16 @@ def test_prompt_action_user_message_in_slot(mock_search, mock_embedding, mock_co ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -11413,28 +11642,23 @@ def __mock_fetch_similar(*args, **kwargs): {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - assert mock_completion.call_args[0][1] == 'Kanban And Scrum Together?' - assert mock_completion.call_args[0][2] == 'You are a personal assistant.\n' - print(mock_completion.call_args[0][3]) - assert mock_completion.call_args[0][3] == """ -Instructions on how to use Similarity Prompt: -['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.'] -Answer question based on the context above, if answer is not in the context go check previous logs. -""" - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) -def test_prompt_action_response_action_when_similarity_is_empty(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from uuid6 import uuid7 + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Kanban And Scrum Together? \nA:"}], + 'metadata': {'user': user, 'bot': bot}, 'api_key': value, + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +def test_prompt_action_response_action_when_similarity_is_empty(mock_search, mock_embedding, mock_completion): action_name = "test_prompt_action_response_action_when_similarity_is_empty" bot = "5f50fd0a56b698ca10d35d2C" user = "udit.pandey" value = "keyvalue" user_msg = "What kind of language is python?" - bot_content = "Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected." generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." llm_prompts = [ {'name': 'System Prompt', @@ -11450,9 +11674,9 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = {'result': []} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -11480,29 +11704,20 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc 'response': None, 'image': None, 'attachment': None} ] - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert not mock_completion.call_args.args[3] - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': []} - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, + {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], + 'metadata': {'user': 'udit.pandey', 'bot': bot}, 'api_key': value, + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_when_similarity_disabled(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_response_action_when_similarity_disabled" @@ -11526,10 +11741,11 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc 'is_enabled': False} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text - mock_search.return_value = {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_search.return_value = { + 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() PromptAction(name=action_name, bot=bot, user=user, num_bot_responses=2, llm_prompts=llm_prompts).save() @@ -11555,17 +11771,11 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert not mock_completion.call_args.args[3] - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [], - 'instructions': []} \ No newline at end of file + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, + {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], + 'metadata': {'user': user, 'bot': bot}, 'api_key': value, + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/integration_test/chat_service_test.py b/tests/integration_test/chat_service_test.py index 45597ff76..77b828a71 100644 --- a/tests/integration_test/chat_service_test.py +++ b/tests/integration_test/chat_service_test.py @@ -5,14 +5,18 @@ from datetime import datetime, timedelta from unittest import mock from urllib.parse import urlencode, quote_plus - -from kairon.shared.live_agent.live_agent import LiveAgentHandler from kairon.shared.utils import Utility - os.environ["system_file"] = "./tests/testing_data/system.yaml" os.environ["ASYNC_TEST_TIMEOUT"] = "3600" Utility.load_environment() +Utility.load_system_metadata() + +from kairon.shared.live_agent.live_agent import LiveAgentHandler + + + + import pytest import responses from mock import patch diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 1dcfac008..1b4219e76 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -23,6 +23,9 @@ from pydantic import SecretStr from rasa.shared.utils.io import read_config_file from slack_sdk.web.slack_response import SlackResponse +from kairon.shared.utils import Utility, MailUtility + +Utility.load_system_metadata() from kairon.api.app.main import app from kairon.events.definitions.multilingual import MultilingualEvent @@ -69,7 +72,6 @@ from kairon.shared.multilingual.utils.translator import Translator from kairon.shared.organization.processor import OrgProcessor from kairon.shared.sso.clients.google import GoogleSSO -from kairon.shared.utils import Utility, MailUtility from urllib.parse import urlencode from deepdiff import DeepDiff @@ -4100,9 +4102,10 @@ def test_get_prompt_action(): assert actual["error_code"] == 0 assert not actual["message"] actual["data"][0].pop("_id") - assert actual["data"] == [ + assert not DeepDiff(actual["data"], [ {'name': 'test_update_prompt_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ @@ -4118,7 +4121,7 @@ def test_get_prompt_action(): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], 'dispatch_response': True, - 'status': True}] + 'status': True}], ignore_order=True) def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(monkeypatch): @@ -4191,10 +4194,11 @@ def _mock_get_bot_settings(*args, **kwargs): assert actual["error_code"] == 0 assert not actual["message"] actual["data"][1].pop("_id") - assert actual["data"][1] == { + assert not DeepDiff(actual["data"][1], { 'name': 'test_add_prompt_action_with_empty_collection_for_bot_content_prompt', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -4212,7 +4216,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], - 'dispatch_response': True, 'status': True} + 'dispatch_response': True, 'status': True}, ignore_order=True) def test_add_prompt_action_with_bot_content_prompt_with_payload(monkeypatch): @@ -4279,10 +4283,11 @@ def _mock_get_bot_settings(*args, **kwargs): ) actual = response.json() actual["data"][2].pop("_id") - assert actual["data"][2] == { + assert not DeepDiff(actual["data"][2], { 'name': 'test_add_prompt_action_with_bot_content_prompt_with_payload', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -4299,7 +4304,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], - 'set_slots': [], 'dispatch_response': True, 'status': True} + 'set_slots': [], 'dispatch_response': True, 'status': True}, ignore_order=True) assert actual["success"] assert actual["error_code"] == 0 assert not actual["message"] @@ -4370,10 +4375,11 @@ def _mock_get_bot_settings(*args, **kwargs): ) actual = response.json() actual["data"][3].pop("_id") - assert actual["data"][3] == { + assert not DeepDiff(actual["data"][3], { 'name': 'test_add_prompt_action_with_bot_content_prompt_with_content', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -4391,7 +4397,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], - 'dispatch_response': True, 'status': True} + 'dispatch_response': True, 'status': True}, ignore_order=True) assert actual["success"] assert actual["error_code"] == 0 assert not actual["message"] diff --git a/tests/unit_test/action/action_test.py b/tests/unit_test/action/action_test.py index 312d46927..715fecc12 100644 --- a/tests/unit_test/action/action_test.py +++ b/tests/unit_test/action/action_test.py @@ -6,6 +6,8 @@ import mock from googleapiclient.http import HttpRequest from pipedrive.exceptions import UnauthorizedError, BadRequestError +from kairon.shared.utils import Utility +Utility.load_system_metadata() from kairon.actions.definitions.email import ActionEmail from kairon.actions.definitions.factory import ActionFactory @@ -41,10 +43,10 @@ from kairon.actions.handlers.processor import ActionProcessor from kairon.shared.actions.utils import ActionUtility from kairon.shared.actions.exception import ActionFailure -from kairon.shared.utils import Utility from unittest.mock import patch from urllib.parse import urlencode + class TestActions: @pytest.fixture(autouse=True, scope='class') @@ -2658,6 +2660,7 @@ def test_get_prompt_action_config(self): 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', @@ -3949,6 +3952,7 @@ def test_get_prompt_action_config_2(self): 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'dispatch_response': True, 'set_slots': [], + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', diff --git a/tests/unit_test/api/api_processor_test.py b/tests/unit_test/api/api_processor_test.py index 34a46ad81..363272358 100644 --- a/tests/unit_test/api/api_processor_test.py +++ b/tests/unit_test/api/api_processor_test.py @@ -7,6 +7,8 @@ from unittest import mock from unittest.mock import patch from urllib.parse import urljoin +from kairon.shared.utils import Utility, MailUtility +Utility.load_system_metadata() import jwt import pytest @@ -22,7 +24,6 @@ from starlette.requests import Request from starlette.responses import RedirectResponse -from kairon.api.app.routers.idp import get_idp_config from kairon.api.models import RegisterAccount, EventConfig, IDPConfig, StoryRequest, HttpActionParameters, Password from kairon.exceptions import AppException from kairon.idp.data_objects import IdpConfig @@ -42,12 +43,6 @@ from kairon.shared.organization.processor import OrgProcessor from kairon.shared.sso.clients.facebook import FacebookSSO from kairon.shared.sso.clients.google import GoogleSSO -from kairon.shared.utils import Utility, MailUtility -from kairon.exceptions import AppException -import time -from kairon.idp.data_objects import IdpConfig -from kairon.api.models import RegisterAccount, EventConfig, IDPConfig, StoryRequest, HttpActionParameters, Password -from mongomock import MongoClient os.environ["system_file"] = "./tests/testing_data/system.yaml" diff --git a/tests/unit_test/augmentation/gpt_augmentation_test.py b/tests/unit_test/augmentation/gpt_augmentation_test.py index dfdb7d42e..e743fb49c 100644 --- a/tests/unit_test/augmentation/gpt_augmentation_test.py +++ b/tests/unit_test/augmentation/gpt_augmentation_test.py @@ -1,7 +1,7 @@ from augmentation.paraphrase.gpt3.generator import GPT3ParaphraseGenerator from augmentation.paraphrase.gpt3.models import GPTRequest from augmentation.paraphrase.gpt3.gpt import GPT -import openai +from openai.resources.completions import Completions import pytest import responses @@ -61,7 +61,7 @@ def test_questions_set_generation(monkeypatch): def test_generate_questions(monkeypatch): - monkeypatch.setattr(openai.Completion, 'create', mock_create) + monkeypatch.setattr(Completions, 'create', mock_create) request_data = GPTRequest(api_key="MockKey", data=["Are there any more test questions?"], num_responses=2) @@ -73,7 +73,7 @@ def test_generate_questions(monkeypatch): def test_generate_questions_empty_api_key(monkeypatch): - monkeypatch.setattr(openai.Completion, 'create', mock_create) + monkeypatch.setattr(Completions, 'create', mock_create) request_data = GPTRequest(api_key="", data=["Are there any more test questions?"], num_responses=2) @@ -84,7 +84,7 @@ def test_generate_questions_empty_api_key(monkeypatch): def test_generate_questions_empty_data(monkeypatch): - monkeypatch.setattr(openai.Completion, 'create', mock_create) + monkeypatch.setattr(Completions, 'create', mock_create) request_data = GPTRequest(api_key="MockKey", data=[], num_responses=2) @@ -125,6 +125,6 @@ def test_generate_questions_invalid_api_key(): data=["Are there any more test questions?"], num_responses=2) gpt3_generator = GPT3ParaphraseGenerator(request_data=request_data) - with pytest.raises(APIError, match=r'.*Incorrect API key provided: InvalidKey. You can find your API key at https://beta.openai.com..*'): + with pytest.raises(APIError, match=r'.*Incorrect API key provided: InvalidKey. You can find your API key at https://platform.openai.com/account/..*'): gpt3_generator.paraphrases() diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index b3b6fc872..b829acf05 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -8,6 +8,11 @@ from datetime import datetime, timedelta, timezone from io import BytesIO from typing import List +from kairon.shared.utils import Utility +os.environ["system_file"] = "./tests/testing_data/system.yaml" +Utility.load_environment() +Utility.load_system_metadata() + from mock import patch import numpy as np @@ -75,19 +80,15 @@ from kairon.shared.data.training_data_generation_processor import TrainingDataGenerationProcessor from kairon.shared.data.utils import DataUtility from kairon.shared.importer.processor import DataImporterLogProcessor -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from kairon.shared.metering.constants import MetricType from kairon.shared.metering.data_object import Metering from kairon.shared.models import StoryEventType, HttpContentType, CognitionDataType from kairon.shared.multilingual.processor import MultilingualLogProcessor from kairon.shared.test.data_objects import ModelTestingLogs from kairon.shared.test.processor import ModelTestingLogProcessor -from kairon.shared.utils import Utility from kairon.train import train_model_for_bot, start_training - -os.environ["system_file"] = "./tests/testing_data/system.yaml" -Utility.load_environment() from deepdiff import DeepDiff +import litellm class TestMongoProcessor: @@ -167,7 +168,7 @@ def test_add_prompt_action_with_invalid_slots(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_slots', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -195,7 +196,7 @@ def test_add_prompt_action_with_invalid_http_action(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_http_action', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -224,7 +225,7 @@ def test_add_prompt_action_with_invalid_similarity_threshold(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_prompt_action_similarity', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -254,7 +255,7 @@ def test_add_prompt_action_with_invalid_top_results(self): user = 'test_user' request = {'name': 'test_prompt_action_invalid_top_results', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -302,7 +303,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): processor.add_prompt_action(request, bot, user) prompt_action = processor.get_prompt_action(bot) prompt_action[0].pop("_id") - assert prompt_action == [ + assert not DeepDiff(prompt_action, [ {'name': 'test_add_prompt_action_with_empty_collection_for_bot_content_prompt', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", @@ -310,6 +311,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'data': 'default', @@ -320,7 +322,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): 'is_enabled': True}, {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True}] + 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True}], ignore_order=True) def test_add_prompt_action_with_bot_content_prompt(self): processor = MongoProcessor() @@ -346,8 +348,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): processor.add_prompt_action(request, bot, user) prompt_action = processor.get_prompt_action(bot) prompt_action[1].pop("_id") - print(prompt_action) - assert prompt_action[1] == { + assert not DeepDiff(prompt_action[1], { 'name': 'test_add_prompt_action_with_bot_content_prompt', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", @@ -355,6 +356,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'data': 'Bot_collection', @@ -365,7 +367,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): 'is_enabled': True}, {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True} + 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True}, ignore_order=True) def test_add_prompt_action_with_invalid_query_prompt(self): processor = MongoProcessor() @@ -550,7 +552,7 @@ def test_add_prompt_action_with_empty_llm_prompts(self): user = 'test_user' request = {'name': 'test_add_prompt_action_with_empty_llm_prompts', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -575,13 +577,13 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) pytest.action_id = processor.add_prompt_action(request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_add_prompt_action_faq_action_with_default_values', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_add_prompt_action_faq_action_with_default_values', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -589,7 +591,7 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [{'name': 'gpt_result', 'value': '${data}', 'evaluation_type': 'expression'}, {'name': 'gpt_result_type', 'value': '${data.type}', - 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}] + 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}], ignore_order=True) def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): processor = MongoProcessor() @@ -598,14 +600,14 @@ def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_temperature_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="Temperature must be between 0.0 and 2.0!"): + with pytest.raises(ValidationError, match=re.escape("['temperature']: 3.0 is greater than the maximum of 2.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_stop_hyperparameter(self): @@ -615,7 +617,7 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_stop_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': ["\n", ".", "?", "!", ";"], 'presence_penalty': 0.0, @@ -624,7 +626,7 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} with pytest.raises(ValidationError, - match="Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers."): + match=re.escape('[\'stop\']: ["\\n",".","?","!",";"] is not valid under any of the schemas listed in the \'anyOf\' keyword')): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): @@ -635,14 +637,14 @@ def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): request = {'name': 'test_add_prompt_action_with_invalid_presence_penalty_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': -3.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="Presence penalty must be between -2.0 and 2.0!"): + with pytest.raises(ValidationError, match=re.escape("['presence_penalty']: -3.0 is less than the minimum of -2.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): @@ -653,14 +655,14 @@ def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): request = {'name': 'test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 3.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="Frequency penalty must be between -2.0 and 2.0!"): + with pytest.raises(ValidationError, match=re.escape("['frequency_penalty']: 3.0 is greater than the maximum of 2.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): @@ -670,14 +672,14 @@ def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_max_tokens_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="max_tokens must be between 5 and 4096 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['max_tokens']: 2 is less than the minimum of 5")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): @@ -687,14 +689,14 @@ def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_zero_max_tokens_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="max_tokens must be between 5 and 4096 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['max_tokens']: 0 is less than the minimum of 5")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): @@ -704,14 +706,14 @@ def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_top_p_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt-3.5-turbo', 'top_p': 3.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="top_p must be between 0.0 and 1.0!"): + with pytest.raises(ValidationError, match=re.escape("['top_p']: 3.0 is greater than the maximum of 1.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_n_hyperparameter(self): @@ -721,14 +723,14 @@ def test_add_prompt_action_with_invalid_n_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_n_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 7, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="n must be between 1 and 5 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['n']: 7 is greater than the maximum of 5")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_zero_n_hyperparameter(self): @@ -738,14 +740,14 @@ def test_add_prompt_action_with_zero_n_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_zero_n_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 0, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="n must be between 1 and 5 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['n']: 0 is less than the minimum of 1")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): @@ -755,14 +757,14 @@ def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_logit_bias_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 2, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': 'a'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="logit_bias must be a dictionary!"): + with pytest.raises(ValidationError, match=re.escape('[\'logit_bias\']: "a" is not of type "object"')): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_faq_action_already_exist(self): @@ -821,7 +823,7 @@ def test_edit_prompt_action_faq_action(self): 'source': 'static', 'is_enabled': True}], "failure_message": "updated_failure_message", "use_query_prompt": True, "use_bot_responses": True, "query_prompt": "updated_query_prompt", - "num_bot_responses": 5, "hyperparameters": Utility.get_llm_hyperparameters(), + "num_bot_responses": 5, "hyperparameters": Utility.get_llm_hyperparameters('openai'), "set_slots": [{"name": "gpt_result", "value": "${data}", "evaluation_type": "expression"}, {"name": "gpt_result_type", "value": "${data.type}", "evaluation_type": "script"}], "dispatch_response": False @@ -829,12 +831,12 @@ def test_edit_prompt_action_faq_action(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -852,7 +854,8 @@ def test_edit_prompt_action_faq_action(self): 'is_enabled': True}], 'instructions': [], 'set_slots': [{'name': 'gpt_result', 'value': '${data}', 'evaluation_type': 'expression'}, {'name': 'gpt_result_type', 'value': '${data.type}', - 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}] + 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}], + ignore_order=True) request = {'name': 'test_edit_prompt_action_faq_action_again', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -861,10 +864,10 @@ def test_edit_prompt_action_faq_action(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_faq_action_again', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action_again', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -872,7 +875,7 @@ def test_edit_prompt_action_faq_action(self): {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], - 'dispatch_response': True, 'status': True}] + 'dispatch_response': True, 'status': True}], ignore_order=True) def test_edit_prompt_action_with_less_hyperparameters(self): processor = MongoProcessor() @@ -905,13 +908,13 @@ def test_edit_prompt_action_with_less_hyperparameters(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -927,7 +930,7 @@ def test_edit_prompt_action_with_less_hyperparameters(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': [], 'set_slots': [], 'dispatch_response': True, - 'status': True}] + 'status': True}], ignore_order=True) def test_get_prompt_action_does_not_exist(self): processor = MongoProcessor() @@ -940,13 +943,13 @@ def test_get_prompt_faq_action(self): bot = 'test_bot' action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -962,8 +965,7 @@ def test_get_prompt_faq_action(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': [], 'set_slots': [], 'dispatch_response': True, - 'status': True}] - + 'status': True}], ignore_order=True) def test_delete_prompt_action(self): processor = MongoProcessor() bot = 'test_bot' @@ -2628,7 +2630,7 @@ def test_start_training_fail(self): assert model_training.__len__() == 1 assert model_training.first().exception in str("Training data does not exists!") - @patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @patch.object(litellm, "aembedding", autospec=True) @patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) @patch("kairon.shared.account.processor.AccountProcessor.get_bot", autospec=True) @patch("kairon.train.train_model_for_bot", autospec=True) @@ -2649,8 +2651,8 @@ def test_start_training_with_llm_faq( settings = BotSettings.objects(bot=bot).get() settings.llm_settings = LLMSettings(enable_faq=True) settings.save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_openai.return_value = embedding + embedding = list(np.random.random(1532)) + mock_openai.return_value = {'data': [{'embedding': embedding}]} mock_bot.return_value = {"account": 1} mock_train.return_value = f"/models/{bot}" start_training(bot, user) diff --git a/tests/unit_test/events/events_test.py b/tests/unit_test/events/events_test.py index 98ae5ff10..4f7a63438 100644 --- a/tests/unit_test/events/events_test.py +++ b/tests/unit_test/events/events_test.py @@ -17,15 +17,15 @@ from rasa.shared.constants import DEFAULT_DOMAIN_PATH, DEFAULT_DATA_PATH, DEFAULT_CONFIG_PATH from rasa.shared.importers.rasa import RasaFileImporter from responses import matchers +from kairon.shared.utils import Utility + +Utility.load_system_metadata() from kairon.shared.channels.broadcast.whatsapp import WhatsappBroadcast from kairon.shared.chat.data_objects import ChannelLogs os.environ["system_file"] = "./tests/testing_data/system.yaml" -from kairon.events.definitions.message_broadcast import MessageBroadcastEvent - -from kairon.shared.chat.broadcast.processor import MessageBroadcastProcessor from kairon.events.definitions.data_importer import TrainingDataImporterEvent from kairon.events.definitions.faq_importer import FaqDataImporterEvent from kairon.events.definitions.history_delete import DeleteHistoryEvent @@ -42,7 +42,6 @@ from kairon.shared.data.processor import MongoProcessor from kairon.shared.importer.processor import DataImporterLogProcessor from kairon.shared.test.processor import ModelTestingLogProcessor -from kairon.shared.utils import Utility from kairon.test.test_models import ModelTester os.environ["system_file"] = "./tests/testing_data/system.yaml" @@ -2020,7 +2019,7 @@ def test_execute_message_broadcast_with_pyscript_failure(self, mock_is_exist, mo bot = 'test_execute_message_broadcast_with_pyscript_failure' user = 'test_user' script = """ - import time + import os """ script = textwrap.dedent(script) config = { @@ -2057,7 +2056,7 @@ def test_execute_message_broadcast_with_pyscript_failure(self, mock_is_exist, mo logs[0][0].pop("timestamp", None) assert logs[0][0] == {"event_id": event_id, 'reference_id': reference_id, 'log_type': 'common', 'bot': bot, 'status': 'Fail', - 'user': user, "exception": "Script execution error: import of 'time' is unauthorized"} + 'user': user, "exception": "Script execution error: import of 'os' is unauthorized"} with pytest.raises(AppException, match="Notification settings not found!"): MessageBroadcastProcessor.get_settings(event_id, bot) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 7494e1c0a..c8f727db3 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -7,16 +7,17 @@ import ujson as json from aiohttp import ClientConnectionError from mongoengine import connect +from kairon.shared.utils import Utility +Utility.load_system_metadata() from kairon.exceptions import AppException from kairon.shared.admin.constants import BotSecretType from kairon.shared.admin.data_objects import BotSecrets from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT -from kairon.shared.data.data_objects import LLMSettings -from kairon.shared.llm.factory import LLMFactory -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding, LLMBase -from kairon.shared.utils import Utility +from kairon.shared.llm.processor import LLMProcessor +import litellm +from deepdiff import DeepDiff class TestLLM: @@ -26,36 +27,9 @@ def init_connection(self): Utility.load_environment() connect(**Utility.mongoengine_connection(Utility.environment['database']["url"])) - def test_llm_base_train(self): - with pytest.raises(Exception): - base = LLMBase("Test") - base.train() - - def test_llm_base_predict(self): - with pytest.raises(Exception): - base = LLMBase('Test') - base.predict("Sample") - - def test_llm_factory_invalid_type(self): - with pytest.raises(Exception): - LLMFactory.get_instance("sample")("test", LLMSettings(provider="openai").to_mongo().to_dict()) - - def test_llm_factory_faq_type(self): - BotSecrets(secret_type=BotSecretType.gpt_key.value, value='value', bot='test', user='test').save() - inst = LLMFactory.get_instance("faq")("test", LLMSettings(provider="openai").to_mongo().to_dict()) - assert isinstance(inst, GPT3FAQEmbedding) - assert inst.db_url == Utility.environment['vector']['db'] - assert inst.headers == {} - - def test_llm_factory_faq_type_set_vector_key(self): - with mock.patch.dict(Utility.environment, {'vector': {"db": "http://test:6333", 'key': 'test'}}): - inst = LLMFactory.get_instance("faq")("test", LLMSettings(provider="openai").to_mongo().to_dict()) - assert isinstance(inst, GPT3FAQEmbedding) - assert inst.db_url == Utility.environment['vector']['db'] - assert inst.headers == {'api-key': Utility.environment['vector']['key']} - @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): bot = "test_embed_faq" user = "test" value = "nupurkhare" @@ -64,19 +38,11 @@ async def test_gpt3_faq_embedding_train(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} - + embedding = list(np.random.random(LLMProcessor.__embedding__)) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -100,23 +66,26 @@ async def test_gpt3_faq_embedding_train(self, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train() + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": test_content.data} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { 'points': [{'id': test_content.vector_id, 'vector': embedding, - 'payload': {"collection_name": f"{gpt3.bot}{gpt3.suffix}", 'content': test_content.data} + 'payload': {'content': test_content.data} }]} + expected = {"model": "text-embedding-3-small", + "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aioresponses): bot = "test_embed_faq_text" user = "test" value = "nupurkhare" @@ -149,18 +118,9 @@ async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]}, - repeat=True - ) - - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + embedding = list(np.random.random(LLMProcessor.__embedding__)) + mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]} + gpt3 = LLMProcessor(bot) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -205,39 +165,36 @@ async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train() + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 3 assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'name': f"{gpt3.bot}_country_details{gpt3.suffix}", 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": '{"country":"Spain","lang":"spanish"}'} - assert list(aioresponses.requests.values())[3][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][1].kwargs['json'] == {"model": "text-embedding-3-small", - 'input': '{"lang":"spanish","role":"ds"}'} - assert list(aioresponses.requests.values())[3][1].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][2].kwargs['json'] == {"model": "text-embedding-3-small", - "input": '{"name":"Nupur","city":"Pune"}'} - assert list(aioresponses.requests.values())[3][2].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[4][0].kwargs['json'] == {'points': [{'id': test_content_two.vector_id, + assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content_two.vector_id, 'vector': embedding, - 'payload': {'collection_name': f"{gpt3.bot}_country_details{gpt3.suffix}", - 'country': 'Spain'}}]} - assert list(aioresponses.requests.values())[4][1].kwargs['json'] == {'points': [{'id': test_content_three.vector_id, + 'payload': {'country': 'Spain'}}]} + assert list(aioresponses.requests.values())[3][1].kwargs['json'] == {'points': [{'id': test_content_three.vector_id, 'vector': embedding, - 'payload': {'collection_name': f"{gpt3.bot}_country_details{gpt3.suffix}", 'role': 'ds'}}]} + 'payload': {'role': 'ds'}}]} - assert list(aioresponses.requests.values())[5][0].kwargs['json'] == {'name': f"{gpt3.bot}_user_details{gpt3.suffix}", + assert list(aioresponses.requests.values())[4][0].kwargs['json'] == {'name': f"{gpt3.bot}_user_details{gpt3.suffix}", 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[6][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + assert list(aioresponses.requests.values())[5][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, 'vector': embedding, - 'payload': {'collection_name': f"{gpt3.bot}_user_details{gpt3.suffix}", - 'name': 'Nupur'}}]} + 'payload': {'name': 'Nupur'}}]} assert response['faq'] == 3 + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + print(mock_embedding.call_args) + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_payload_with_int(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, aioresponses): bot = "test_embed_faq_json" user = "test" value = "nupurkhare" @@ -254,17 +211,11 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} + embedding = list(np.random.random(LLMProcessor.__embedding__)) input = {"name": "Ram", "color": "red"} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), method="PUT", @@ -282,21 +233,25 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, aioresponses): ) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - response = await gpt3.train() + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_embed_faq_json_payload_with_int_faq_embd', 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": json.dumps(input)} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, 'vector': embedding, - 'payload': {'name': 'Ram', 'age': 23, 'color': 'red', "collection_name": "test_embed_faq_json_payload_with_int_faq_embd"} + 'payload': {'name': 'Ram', 'age': 23, 'color': 'red'} }]} + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_int(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): bot = "test_int" user = "test" value = "nupurkhare" @@ -313,18 +268,11 @@ async def test_gpt3_faq_embedding_train_int(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} + embedding = list(np.random.random(LLMProcessor.__embedding__)) input = {"name": "Ram", "color": "red"} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - + mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -354,28 +302,32 @@ async def test_gpt3_faq_embedding_train_int(self, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train() + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_int_embd_int_faq_embd', 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": json.dumps(input)} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header expected_payload = test_content.data - expected_payload['collection_name'] = 'test_int_embd_int_faq_embd' - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + #expected_payload['collection_name'] = 'test_int_embd_int_faq_embd' + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { 'points': [{'id': test_content.vector_id, 'vector': embedding, 'payload': expected_payload }]} + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + def test_gpt3_faq_embedding_train_failure(self): with pytest.raises(AppException, match=f"Bot secret '{BotSecretType.gpt_key.value}' not configured!"): - GPT3FAQEmbedding('test_failure', LLMSettings(provider="openai").to_mongo().to_dict()) + LLMProcessor('test_failure') @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_upsert_error(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aioresponses): bot = "test_embed_faq_not_exists" user = "test" value = "nupurk" @@ -384,19 +336,12 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - - request_header = {"Authorization": "Bearer nupurk"} + embedding = list(np.random.random(LLMProcessor.__embedding__)) - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -423,17 +368,21 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, aioresponses): ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train() + await gpt3.train(user=user, bot=bot) assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": test_content.data} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, 'payload': {'collection_name': f"{bot}{gpt3.suffix}",'content': test_content.data}}]} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + 'vector': embedding, 'payload': {'content': test_content.data}}]} + expected = {"model": "text-embedding-3-small", + "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_embedding, aioresponses): bot = "payload_upsert_error" user = "test" value = "nupurk" @@ -450,19 +399,11 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, aiorespo bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - - request_header = {"Authorization": "Bearer nupurk"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + embedding = list(np.random.random(LLMProcessor.__embedding__)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -489,21 +430,27 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, aiorespo ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train() + await gpt3.train(user=user, bot=bot) assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": json.dumps(test_content.data)} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header expected_payload = test_content.data - expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + #expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, 'vector': embedding, 'payload': expected_payload }]} + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict" user = "test" @@ -516,6 +463,7 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", @@ -523,8 +471,9 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}]} - hyperparameters = Utility.get_llm_hyperparameters() + 'collection': 'python'}], + "hyperparameters": hyperparameters + } mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, @@ -532,24 +481,10 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), @@ -558,24 +493,29 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response['content'] == generated_text - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = value + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_default_collection(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict_with_default_collection" user = "test" @@ -588,15 +528,17 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, aiorespo generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" - + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'default'}]} - hyperparameters = Utility.get_llm_hyperparameters() + 'collection': 'default'}], + 'hyperparameters': hyperparameters + } + mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, @@ -604,24 +546,10 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, aiorespo 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -631,40 +559,52 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, aiorespo {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) assert response['content'] == generated_text - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = value + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_embed_faq_predict", user="test").save() + collection='python', bot="test_gpt3_faq_embedding_predict_with_values", user="test").save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() + key = 'test' + user = "tests" + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}]} + 'collection': 'python'}], + "hyperparameters": hyperparameters + } - hyperparameters = Utility.get_llm_hyperparameters() mock_completion_request = {"messages": [ {"role": "system", "content": "You are a personal assistant. Answer the question according to the below context"}, @@ -672,24 +612,11 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), @@ -698,7 +625,7 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=gpt3.bot, **k_faq_action_config) assert response['content'] == generated_text assert gpt3.logs == [ {'messages': [{'role': 'system', @@ -714,25 +641,36 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': gpt3.bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "test_gpt3_faq_embedding_predict_with_values_with_instructions" + key = 'test' test_content = CognitionData( data="Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ", - collection='java', bot="test_embed_faq_predict", user="test").save() - + collection='java', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", @@ -740,9 +678,10 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, ai 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', "collection": "java"}], - 'instructions': ['Answer in a short way.', 'Keep it simple.']} + 'instructions': ['Answer in a short way.', 'Keep it simple.'], + "hyperparameters": hyperparameters + } - hyperparameters = Utility.get_llm_hyperparameters() mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, @@ -750,186 +689,210 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, ai 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url="https://api.openai.com/v1/chat/completions", + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + payload={'result': [ + {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) - - aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), - method="POST", - payload={'result': [ - {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} - ) - - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response['content'] == generated_text - assert gpt3.logs == [ - {'messages': [{'role': 'system', - 'content': 'You are a personal assistant. Answer the question according to the below context'}, - {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': { - 'choices': [{'message': {'content': 'Python is dynamically typed, garbage-collected, ' - 'high level, general purpose programming.', - 'role': 'assistant'}}]}, - 'type': 'answer_query', - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}}] - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) + assert response['content'] == generated_text + assert gpt3.logs == [ + {'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': { + 'choices': [{'message': {'content': 'Python is dynamically typed, garbage-collected, ' + 'high level, general purpose programming.', + 'role': 'assistant'}}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + embedding = list(np.random.random(LLMProcessor.__embedding__)) + bot = "test_gpt3_faq_embedding_predict_completion_connection_error" + user = 'test' + key = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_embed_faq_predict", user="test").save() + collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() + hyperparameters = Utility.get_default_llm_hyperparameters() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}]} + "collection": 'python'}], + "hyperparameters": hyperparameters + } def __mock_connection_error(*args, **kwargs): - import openai - - raise openai.error.APIConnectionError("Connection reset by peer!") + raise Exception("Connection reset by peer!") - mock_embedding.return_value = embedding + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.side_effect = __mock_connection_error - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) - - aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), - method="POST", - payload={'result': [ - {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} - ) + gpt3 = LLMProcessor(test_content.bot) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - print(mock_completion.call_args.args[3]) + aioresponses.add( + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + method="POST", + payload={'result': [ + {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} + ) - assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} - assert mock_embedding.call_args.args[1] == query + assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == 'You are a personal assistant. Answer the question according to the below context' - assert mock_completion.call_args.args[3] == """Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n""" - assert mock_completion.call_args.kwargs == {'similarity_prompt': [ - {'top_results': 10, 'similarity_threshold': 0.7, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', 'collection': 'python'}]} - assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error'}, + 'api_key': 'test', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio @mock.patch("kairon.shared.rest_client.AioRestClient._AioRestClient__trigger", autospec=True) - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock_llm_request): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user ="test" + bot = "test_gpt3_faq_embedding_predict_exact_match" + key = 'test' test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_embed_faq_predict", user="test").save() + collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}]} + "collection": 'python'}], + "hyperparameters": hyperparameters + } - mock_embedding.return_value = embedding + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_llm_request.side_effect = ClientConnectionError() - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} + response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_exact_match", **k_faq_action_config) + assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} - assert mock_embedding.call_args.args[1] == query - assert gpt3.logs == [] - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_embedding): - import openai - - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "test_gpt3_faq_embedding_predict_embedding_connection_error" + key = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - bot="test_embed_faq_predict", user="test").save() + bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() + hyperparameters = Utility.get_default_llm_hyperparameters() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", - "context_prompt": "Based on below context answer question, if answer not in context check previous logs."} + "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", + "hyperparameters": hyperparameters + } + mock_embedding.side_effect = [Exception("Connection reset by peer!"), {'data': [{'embedding': embedding}]}] - mock_embedding.side_effect = [openai.error.APIConnectionError("Connection reset by peer!"), embedding] + gpt3 = LLMProcessor(test_content.bot) + mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_embedding_connection_error", **k_faq_action_config) + assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} + assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - assert mock_embedding.call_args.args[1] == query - assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) - bot = "test_embed_faq_predict" + bot = "test_gpt3_faq_embedding_predict_with_previous_bot_responses" user = "test" + key = "test" + hyperparameters = Utility.get_default_llm_hyperparameters() test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" @@ -940,9 +903,10 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior ], "similarity_prompt": [{'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}] + "collection": 'python'}], + "hyperparameters": hyperparameters } - hyperparameters = Utility.get_llm_hyperparameters() + mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below'}, {'role': 'user', 'content': 'hello'}, @@ -951,23 +915,10 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior 'content': "Answer question based on the context below, if answer is not in the context go check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), @@ -976,44 +927,55 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - print(list(aioresponses.requests.values())[2][0].kwargs['json']) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) assert response['content'] == generated_text - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) - bot = "test_embed_faq_predict" + bot = "test_gpt3_faq_embedding_predict_with_query_prompt" user = "test" + key = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" rephrased_query = "Explain python is called high level programming language in laymen terms?" + hyperparameters = Utility.get_default_llm_hyperparameters() + k_faq_action_config = {"query_prompt": { "query_prompt": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", "use_query_prompt": True}, - "similarity_prompt": [ - {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}] - } - hyperparameters = Utility.get_llm_hyperparameters() + "similarity_prompt": [ + {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], + "hyperparameters": hyperparameters + } + mock_rephrase_request = {"messages": [ {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, @@ -1029,31 +991,11 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): ]} mock_rephrase_request.update(hyperparameters) mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]} - ) - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, - repeat=True - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), @@ -1062,18 +1004,21 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - print(list(aioresponses.requests.values())[2][1].kwargs['json']) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) assert response['content'] == generated_text - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_rephrase_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[2][1].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][1].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/unit_test/utility_test.py b/tests/unit_test/utility_test.py index d2e9d3621..438633592 100644 --- a/tests/unit_test/utility_test.py +++ b/tests/unit_test/utility_test.py @@ -8,6 +8,9 @@ from io import BytesIO from unittest.mock import patch, MagicMock from urllib.parse import urlencode +from kairon.shared.utils import Utility, MailUtility + +Utility.load_system_metadata() import numpy as np import pandas as pd @@ -36,12 +39,7 @@ from kairon.shared.data.data_objects import EventConfig, Slots, LLMSettings from kairon.shared.data.processor import MongoProcessor from kairon.shared.data.utils import DataUtility -from kairon.shared.llm.clients.azure import AzureGPT3Resources -from kairon.shared.llm.clients.factory import LLMClientFactory -from kairon.shared.llm.clients.gpt3 import GPT3Resources -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from kairon.shared.models import TemplateType -from kairon.shared.utils import Utility, MailUtility from kairon.shared.verification.email import QuickEmailVerification @@ -2106,7 +2104,7 @@ def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, user=user, ws_url="http://localhost:5000/event_url" @@ -2118,14 +2116,14 @@ def publish_auditlog(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user, action="save" ).count() - assert count == 2 + assert count == 1 def test_save_and_publish_auditlog_action_save_another(self, monkeypatch): def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2141,14 +2139,14 @@ def publish_auditlog(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user, action="save" ).count() - assert count == 3 + assert count == 2 def test_save_and_publish_auditlog_action_update(self, monkeypatch): def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2163,14 +2161,14 @@ def publish_auditlog(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user, action="update" ).count() - assert count == 2 + assert count == 1 def test_save_and_publish_auditlog_total_count(self, monkeypatch): def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2192,7 +2190,7 @@ def execute_http_request(*args, **kwargs): return None monkeypatch.setattr(Utility, "execute_http_request", execute_http_request) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2207,11 +2205,11 @@ def execute_http_request(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user ).count() - assert count >= 3 + assert count >= 2 @responses.activate def test_publish_auditlog(self): - bot = "secret" + bot = "publish_auditlog" user = "secret_user" config = { "bot_user_oAuth_token": "xoxb-801939352912-801478018484-v3zq6MYNu62oSs8vammWOY8K", @@ -2247,7 +2245,7 @@ def test_publish_auditlog(self): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user ).count() - assert count == 4 + assert count == 1 @pytest.mark.asyncio async def test_messageConverter_messenger_button_one(self): @@ -2939,7 +2937,7 @@ def test_verify_email_enable_valid_email(self): Utility.verify_email(email) def test_get_llm_hyperparameters(self): - hyperparameters = Utility.get_llm_hyperparameters() + hyperparameters = Utility.get_llm_hyperparameters("openai") assert hyperparameters == { "temperature": 0.0, "max_tokens": 300, @@ -2954,508 +2952,10 @@ def test_get_llm_hyperparameters(self): } def test_get_llm_hyperparameters_not_found(self, monkeypatch): - monkeypatch.setitem(Utility.environment["llm"], "faq", None) - with pytest.raises( - AppException, match="Could not find any hyperparameters for configured LLM." - ): - Utility.get_llm_hyperparameters() - - @pytest.mark.asyncio - async def test_trigger_gpt3_client_completion_with_generated_text( - self, aioresponses - ): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - messages = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = messages - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - - resp = await GPT3Resources("test").invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert resp[0] == generated_text - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gpt3_client_completion_with_response(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - - formatted_response, raw_response = await GPT3Resources("test").invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == { - "choices": [{"message": {"content": generated_text, "role": "assistant"}}] - } - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_completion(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - formatted_response, raw_response = await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == { - "choices": [{"message": {"content": generated_text, "role": "assistant"}}] - } - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_completion_failure(self, aioresponses): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=504, - payload={"error": {"message": "Server unavailable!", "id": 876543456789}}, - ) - with pytest.raises( - AppException, match="Failed to connect to service: api.openai.com" - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=201, - body="openai".encode(), - repeat=True, - ) - with pytest.raises(AppException): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_embedding(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": f"Bearer {api_key}"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={"data": [{"embedding": embedding}]}, - ) - formatted_response, raw_response = await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - assert formatted_response == embedding - assert raw_response == {"data": [{"embedding": embedding}]} - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_embedding_failure(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - request_header = {"Authorization": f"Bearer {api_key}"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", method="POST", status=504 - ) - - with pytest.raises( - AppException, match="Failed to connect to service: api.openai.com" - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=204, - payload={"error": {"message": "Server unavailable!", "id": 876543456789}}, - repeat=True, - ) - - with pytest.raises( - AppException, match="Server unavailable!. Request id: 876543456789" - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - assert list(aioresponses.requests.values())[0][1].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][1].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_streaming_completion(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - content = """data: {"choices": [{"delta": {"role": "assistant"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": "Python"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " is"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " dynamically"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " typed"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " garbage-collected"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " high"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " level"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " general"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " purpose"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " programming"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": "."}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {}, "index": 0, "finish_reason": "stop"}]}\n\n -data: [DONE]\n\n""" - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - body=content.encode(), - content_type="text/event-stream", - ) - - formatted_response, raw_response = await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == [ - b'data: {"choices": [{"delta": {"role": "assistant"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": "Python"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " is"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " dynamically"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " typed"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " garbage-collected"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " high"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " level"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " general"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " purpose"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " programming"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": "."}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {}, "index": 0, "finish_reason": "stop"}]}\n', - b"\n", - b"\n", - b"data: [DONE]\n", - b"\n", - ] - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_streaming_connection_error(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=401, - ) - - with pytest.raises( - AppException, - match=re.escape( - "Failed to execute the url: 401, message='Unauthorized', url=URL('https://api.openai.com/v1/chat/completions')" - ), - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_streaming_completion_failure(self, aioresponses): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - content = "data: {'choices': [{'delta': {'role': 'assistant'}}]}\n\n" - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - body=content.encode(), - content_type="text/event-stream", - ) - with pytest.raises( - AppException, - match=re.escape( - "Failed to parse streaming response: b\"data: {'choices': [{'delta': {'role': 'assistant'}}]}\\n\"" - ), - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_completion_failure_invalid_json( - self, aioresponses - ): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - content = "data: {'choices': [{'delta': {'role': 'assistant'}}]}\n\n" - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=504, - body=content.encode(), - ) with pytest.raises( - AppException, match="Failed to connect to service: api.openai.com" + AppException, match="Could not find any hyperparameters for claude LLM." ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) + Utility.get_llm_hyperparameters("claude") def test_get_client_ip_with_request_client(self): request = MagicMock() @@ -3464,217 +2964,6 @@ def test_get_client_ip_with_request_client(self): ip = Utility.get_client_ip(request) assert "58.0.127.89" == ip - def test_llm_resource_provider_factory(self): - client = LLMClientFactory.get_resource_provider(LLMResourceProvider.azure.value) - assert isinstance(client("test"), AzureGPT3Resources) - - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.openai.value - ) - assert isinstance(client("test"), GPT3Resources) - - def test_llm_resource_provider_not_implemented(self): - with pytest.raises(AppException, match="aws client not supported"): - LLMClientFactory.get_resource_provider("aws") - - @pytest.mark.asyncio - async def test_trigger_azure_client_completion(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"api-key": api_key} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['chat_completion_model_id']}/{GPT3ResourceTypes.chat_completion.value}?api-version={llm_settings['api_version']}", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - formatted_response, raw_response = await client.invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == { - "choices": [{"message": {"content": generated_text, "role": "assistant"}}] - } - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_azure_client_embedding(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"api-key": api_key} - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['embeddings_model_id']}/{GPT3ResourceTypes.embeddings.value}?api-version={llm_settings['api_version']}", - method="POST", - status=200, - payload={"data": [{"embedding": embedding}]}, - ) - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - formatted_response, raw_response = await client.invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - assert formatted_response == embedding - assert raw_response == {"data": [{"embedding": embedding}]} - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_azure_client_embedding_failure(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - request_header = {"api-key": api_key} - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['embeddings_model_id']}/{GPT3ResourceTypes.embeddings.value}?api-version={llm_settings['api_version']}", - method="POST", - status=504, - ) - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - - with pytest.raises( - AppException, match="Failed to connect to service: kairon.openai.azure.com" - ): - await client.invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_azure_client_completion_failure(self, aioresponses): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"api-key": api_key} - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['chat_completion_model_id']}/{GPT3ResourceTypes.chat_completion.value}?api-version={llm_settings['api_version']}", - method="POST", - status=504, - payload={"error": {"message": "Server unavailable!", "id": 876543456789}}, - ) - - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - with pytest.raises( - AppException, match="Failed to connect to service: kairon.openai.azure.com" - ): - await client.invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) @pytest.mark.asyncio async def test_messageConverter_whatsapp_dropdown(self): diff --git a/tests/unit_test/validator/training_data_validator_test.py b/tests/unit_test/validator/training_data_validator_test.py index 2633c01f1..f3aa8dc66 100644 --- a/tests/unit_test/validator/training_data_validator_test.py +++ b/tests/unit_test/validator/training_data_validator_test.py @@ -3,6 +3,8 @@ import pytest import yaml from mongoengine import connect +from kairon.shared.utils import Utility +Utility.load_system_metadata() from kairon.exceptions import AppException from kairon.importer.validator.file_validator import TrainingDataValidator @@ -797,55 +799,34 @@ def test_validate_custom_actions_with_errors(self): assert len(error_summary['google_search_actions']) == 2 assert len(error_summary['zendesk_actions']) == 2 assert len(error_summary['pipedrive_leads_actions']) == 3 - assert len(error_summary['prompt_actions']) == 49 + assert len(error_summary['prompt_actions']) == 36 assert len(error_summary['razorpay_actions']) == 3 assert len(error_summary['pyscript_actions']) == 3 assert len(error_summary['database_actions']) == 6 - required_fields_error = error_summary["prompt_actions"][21] - assert re.match(r"Required fields .* not found in action: prompt_action_with_no_llm_prompts", required_fields_error) - del error_summary["prompt_actions"][21] - print(error_summary['prompt_actions']) - assert error_summary['prompt_actions'] == ['top_results should not be greater than 30 and of type int!', - 'similarity_threshold should be within 0.3 and 1.0 and of type int or float!', - 'Collection is required for bot content prompts!', - 'System prompt is required', 'Query prompt must have static source', - 'Name cannot be empty', 'System prompt is required', - 'num_bot_responses should not be greater than 5 and of type int: prompt_action_invalid_num_bot_responses', - 'Collection is required for bot content prompts!', - 'data field in prompts should of type string.', - 'data is required for static prompts', - 'Temperature must be between 0.0 and 2.0!', - 'max_tokens must be between 5 and 4096!', - 'top_p must be between 0.0 and 1.0!', 'n must be between 1 and 5!', - 'presence_penality must be between -2.0 and 2.0!', - 'frequency_penalty must be between -2.0 and 2.0!', - 'logit_bias must be a dictionary!', - 'System prompt must have static source', - 'Collection is required for bot content prompts!', - 'Collection is required for bot content prompts!', - 'Duplicate action found: test_add_prompt_action_one', - 'Invalid action configuration format. Dictionary expected.', - 'Temperature must be between 0.0 and 2.0!', - 'max_tokens must be between 5 and 4096!', - 'top_p must be between 0.0 and 1.0!', 'n must be between 1 and 5!', - 'Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers.', - 'presence_penality must be between -2.0 and 2.0!', - 'frequency_penalty must be between -2.0 and 2.0!', - 'logit_bias must be a dictionary!', - 'Only one system prompt can be present', 'Invalid prompt type', - 'Invalid prompt source', 'Only one system prompt can be present', - 'Invalid prompt type', 'Invalid prompt source', - 'type in LLM Prompts should be of type string.', - 'source in LLM Prompts should be of type string.', - 'Instructions in LLM Prompts should be of type string.', - 'Only one system prompt can be present', - 'Data must contain action name', - 'Only one system prompt can be present', - 'Data must contain slot name', - 'Only one system prompt can be present', - 'Only one system prompt can be present', - 'Only one system prompt can be present', - 'Only one history source can be present'] + expected_errors = ['top_results should not be greater than 30 and of type int!', + 'similarity_threshold should be within 0.3 and 1.0 and of type int or float!', + 'Collection is required for bot content prompts!', 'System prompt is required', + 'Query prompt must have static source', 'Name cannot be empty', 'System prompt is required', + 'num_bot_responses should not be greater than 5 and of type int: prompt_action_invalid_num_bot_responses', + 'Collection is required for bot content prompts!', + 'data field in prompts should of type string.', 'data is required for static prompts', + "['frequency_penalty']: 5 is greater than the maximum of 2.0", + 'System prompt must have static source', 'Collection is required for bot content prompts!', + 'Collection is required for bot content prompts!', + "Required fields ['llm_prompts', 'name'] not found in action: prompt_action_with_no_llm_prompts", + 'Duplicate action found: test_add_prompt_action_one', + 'Invalid action configuration format. Dictionary expected.', + "['frequency_penalty']: 5 is greater than the maximum of 2.0", + 'Only one system prompt can be present', 'Invalid prompt type', + 'Only one system prompt can be present', 'Invalid prompt type', 'Invalid prompt source', + 'type in LLM Prompts should be of type string.', + 'source in LLM Prompts should be of type string.', + 'Instructions in LLM Prompts should be of type string.', + 'Only one system prompt can be present', 'Data must contain action name', + 'Only one system prompt can be present', 'Data must contain slot name', + 'Only one system prompt can be present', 'Only one system prompt can be present', + 'Only one system prompt can be present', 'Only one history source can be present'] + assert not DeepDiff(error_summary['prompt_actions'], expected_errors, ignore_order=True) assert component_count == {'http_actions': 7, 'slot_set_actions': 10, 'form_validation_actions': 9, 'email_actions': 5, 'google_search_actions': 5, 'jira_actions': 6, 'zendesk_actions': 4, 'pipedrive_leads_actions': 5, 'prompt_actions': 8, diff --git a/training_data/ReadMe.md b/training_data/ReadMe.md deleted file mode 100644 index f827d7c61..000000000 --- a/training_data/ReadMe.md +++ /dev/null @@ -1 +0,0 @@ -Trained Data Directory \ No newline at end of file From e126bc2dcbb4b2a8ac6d61112c3be1b9dae71164 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Mon, 24 Jun 2024 19:00:46 +0530 Subject: [PATCH 02/57] 1. added missing test cases 2. removed stream from hyperparameters from prompt action --- kairon/actions/definitions/database.py | 2 +- kairon/actions/definitions/prompt.py | 1 - kairon/api/models.py | 10 +- kairon/shared/actions/utils.py | 3 +- kairon/shared/llm/base.py | 4 +- kairon/shared/llm/processor.py | 41 +- kairon/shared/rest_client.py | 2 +- kairon/shared/utils.py | 4 +- kairon/shared/vector_embeddings/db/base.py | 8 +- kairon/shared/vector_embeddings/db/qdrant.py | 18 +- kairon/train.py | 2 +- metadata/integrations.yml | 4 - tests/conftest.py | 1 - tests/integration_test/action_service_test.py | 35 +- tests/integration_test/services_test.py | 243 +++++++++-- tests/unit_test/action/action_test.py | 4 +- .../data_processor/data_processor_test.py | 44 +- tests/unit_test/llm_test.py | 390 ++++++++++++------ .../vector_embeddings/qdrant_test.py | 31 +- 19 files changed, 584 insertions(+), 263 deletions(-) diff --git a/kairon/actions/definitions/database.py b/kairon/actions/definitions/database.py index ebdf83510..0d54abd49 100644 --- a/kairon/actions/definitions/database.py +++ b/kairon/actions/definitions/database.py @@ -83,7 +83,7 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma request_body = ActionUtility.get_payload(payload, tracker) msg_logger.append(request_body) tracker_data = ActionUtility.build_context(tracker, True) - response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id, bot=self.bot) + response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id) logger.info("response: " + str(response)) response_context = self.__add_user_context_to_http_response(response, tracker_data) bot_response, bot_resp_log, _ = ActionUtility.compose_response(vector_action_config['response'], response_context) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 381e6f543..4c7bf6bc4 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -71,7 +71,6 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma llm_processor = LLMProcessor(self.bot) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, user=tracker.sender_id, - bot=self.bot, **llm_params) status = "FAILURE" if llm_response.get("is_failure", False) is True else status exception = llm_response.get("exception") diff --git a/kairon/api/models.py b/kairon/api/models.py index 435566f34..789873fff 100644 --- a/kairon/api/models.py +++ b/kairon/api/models.py @@ -15,8 +15,7 @@ ACTIVITY_STATUS, INTEGRATION_STATUS, FALLBACK_MESSAGE, - DEFAULT_NLU_FALLBACK_RESPONSE, - DEFAULT_LLM + DEFAULT_NLU_FALLBACK_RESPONSE ) from ..shared.actions.models import ( ActionParameterType, @@ -1037,8 +1036,8 @@ class PromptActionConfigRequest(BaseModel): num_bot_responses: int = 5 failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE user_question: UserQuestionModel = UserQuestionModel() - llm_type: str = DEFAULT_LLM - hyperparameters: dict = None + llm_type: str + hyperparameters: dict llm_prompts: List[LlmPromptRequest] instructions: List[str] = [] set_slots: List[SetSlotsUsingActionResponse] = [] @@ -1067,7 +1066,8 @@ def validate_llm_type(cls, v, values, **kwargs): @validator("hyperparameters") def validate_llm_hyperparameters(cls, v, values, **kwargs): - Utility.validate_llm_hyperparameters(v, kwargs['llm_type'], ValueError) + if values.get('llm_type'): + Utility.validate_llm_hyperparameters(v, values['llm_type'], ValueError) @root_validator def check(cls, values): diff --git a/kairon/shared/actions/utils.py b/kairon/shared/actions/utils.py index 00911f55c..d0c97d72f 100644 --- a/kairon/shared/actions/utils.py +++ b/kairon/shared/actions/utils.py @@ -5,6 +5,8 @@ import re from datetime import datetime from typing import Any, List, Text, Dict +from ..utils import Utility +Utility.load_system_metadata() import requests from aiohttp import ContentTypeError @@ -26,7 +28,6 @@ from ..data.data_objects import Slots, KeyVault from ..plugins.factory import PluginFactory from ..rest_client import AioRestClient -from ..utils import Utility from ...exceptions import AppException diff --git a/kairon/shared/llm/base.py b/kairon/shared/llm/base.py index f07eceda0..006e38a3d 100644 --- a/kairon/shared/llm/base.py +++ b/kairon/shared/llm/base.py @@ -8,9 +8,9 @@ def __init__(self, bot: Text): self.bot = bot @abstractmethod - async def train(self, user, bot, *args, **kwargs) -> Dict: + async def train(self, user, *args, **kwargs) -> Dict: pass @abstractmethod - async def predict(self, query, user, bot, *args, **kwargs) -> Dict: + async def predict(self, query, user, *args, **kwargs) -> Dict: pass diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index ffc48e2eb..c6e7fa8af 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -1,4 +1,4 @@ -import random +from secrets import randbelow, choice import time from typing import Text, Dict, List, Tuple from urllib.parse import urljoin @@ -39,7 +39,7 @@ def __init__(self, bot: Text): self.EMBEDDING_CTX_LENGTH = 8191 self.__logs = [] - async def train(self, user, bot, *args, **kwargs) -> Dict: + async def train(self, user, *args, **kwargs) -> Dict: await self.__delete_collections() count = 0 processor = CognitionDataProcessor() @@ -59,26 +59,25 @@ async def train(self, user, bot, *args, **kwargs) -> Dict: content['data'], metadata) else: search_payload, embedding_payload = {'content': content["data"]}, content["data"] - #search_payload['collection_name'] = collection - embeddings = await self.get_embedding(embedding_payload, user, bot) + embeddings = await self.get_embedding(embedding_payload, user) points = [{'id': content['vector_id'], 'vector': embeddings, 'payload': search_payload}] await self.__collection_upsert__(collection, {'points': points}, err_msg="Unable to train FAQ! Contact support") count += 1 return {"faq": count} - async def predict(self, query: Text, user, bot, *args, **kwargs) -> Tuple: + async def predict(self, query: Text, user, *args, **kwargs) -> Tuple: start_time = time.time() embeddings_created = False try: - query_embedding = await self.get_embedding(query, user, bot) + query_embedding = await self.get_embedding(query, user) embeddings_created = True system_prompt = kwargs.pop('system_prompt', DEFAULT_SYSTEM_PROMPT) context_prompt = kwargs.pop('context_prompt', DEFAULT_CONTEXT_PROMPT) context = await self.__attach_similarity_prompt_if_enabled(query_embedding, context_prompt, **kwargs) - answer = await self.__get_answer(query, system_prompt, context, user, bot, **kwargs) + answer = await self.__get_answer(query, system_prompt, context, user, **kwargs) response = {"content": answer} except Exception as e: logging.exception(e) @@ -100,11 +99,11 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def get_embedding(self, text: Text, user, bot) -> List[float]: + async def get_embedding(self, text: Text, user) -> List[float]: truncated_text = self.truncate_text(text) result = await litellm.aembedding(model="text-embedding-3-small", input=[truncated_text], - metadata={'user': user, 'bot': bot}, + metadata={'user': user, 'bot': self.bot}, api_key=self.api_key, num_retries=3) return result["data"][0]["embedding"] @@ -112,24 +111,25 @@ async def get_embedding(self, text: Text, user, bot) -> List[float]: async def __parse_completion_response(self, response, **kwargs): if kwargs.get("stream"): formatted_response = '' - msg_choice = random.randint(0, kwargs.get("n", 1) - 1) + msg_choice = randbelow(kwargs.get("n", 1)) if response["choices"][0].get("index") == msg_choice and response["choices"][0]['delta'].get('content'): formatted_response = f"{response['choices'][0]['delta']['content']}" else: - msg_choice = random.choice(response['choices']) + msg_choice = choice(response['choices']) formatted_response = msg_choice['message']['content'] return formatted_response - async def __get_completion(self, messages, hyperparameters, user, bot, **kwargs): + async def __get_completion(self, messages, hyperparameters, user, **kwargs): response = await litellm.acompletion(messages=messages, - metadata={'user': user, 'bot': bot}, + metadata={'user': user, 'bot': self.bot}, api_key=self.api_key, num_retries=3, **hyperparameters) - formatted_response = await self.__parse_completion_response(response, **kwargs) + formatted_response = await self.__parse_completion_response(response, + **hyperparameters) return formatted_response, response - async def __get_answer(self, query, system_prompt: Text, context: Text, user, bot, **kwargs): + async def __get_answer(self, query, system_prompt: Text, context: Text, user, **kwargs): use_query_prompt = False query_prompt = '' if kwargs.get('query_prompt', {}): @@ -144,8 +144,7 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, bo if use_query_prompt and query_prompt: query = await self.__rephrase_query(query, system_prompt, query_prompt, hyperparameters=hyperparameters, - user=user, - bot=bot) + user=user) messages = [ {"role": "system", "content": system_prompt}, ] @@ -156,13 +155,12 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, bo completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user, - bot=bot) + user=user) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'answer_query', 'hyperparameters': hyperparameters}) return completion - async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, bot, **kwargs): + async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, **kwargs): messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": f"{query_prompt}\n\n Q: {query}\n A:"} @@ -171,8 +169,7 @@ async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user, - bot=bot) + user=user) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'rephrase_query', 'hyperparameters': hyperparameters}) return completion diff --git a/kairon/shared/rest_client.py b/kairon/shared/rest_client.py index a76faad82..301e11323 100644 --- a/kairon/shared/rest_client.py +++ b/kairon/shared/rest_client.py @@ -60,7 +60,7 @@ async def request(self, request_method: str, http_url: str, request_body: Union[ headers: dict = None, return_json: bool = True, **kwargs): max_retries = kwargs.get("max_retries", 1) - status_forcelist = kwargs.get("status_forcelist", [104, 502, 503, 504]) + status_forcelist = set(kwargs.get("status_forcelist", [104, 502, 503, 504])) timeout = ClientTimeout(total=kwargs['timeout']) if kwargs.get('timeout') else None is_streaming_resp = kwargs.pop("is_streaming_resp", False) content_type = kwargs.pop("content_type", HttpRequestContentType.json.value) diff --git a/kairon/shared/utils.py b/kairon/shared/utils.py index b81e542ec..18082325c 100644 --- a/kairon/shared/utils.py +++ b/kairon/shared/utils.py @@ -2069,12 +2069,12 @@ def get_llm_hyperparameters(llm_type): @staticmethod def validate_llm_hyperparameters(hyperparameters: dict, llm_type: str, exception_class): - from jsonschema_rs import JSONSchema, ValidationError + from jsonschema_rs import JSONSchema, ValidationError as JValidationError schema = Utility.system_metadata["llm"][llm_type] try: validator = JSONSchema(schema) validator.validate(hyperparameters) - except ValidationError as e: + except JValidationError as e: message = f"{e.instance_path}: {e.message}" raise exception_class(message) diff --git a/kairon/shared/vector_embeddings/db/base.py b/kairon/shared/vector_embeddings/db/base.py index d1c2a1e97..887be41bb 100644 --- a/kairon/shared/vector_embeddings/db/base.py +++ b/kairon/shared/vector_embeddings/db/base.py @@ -8,16 +8,16 @@ class VectorEmbeddingsDbBase(ABC): @abstractmethod - async def embedding_search(self, request_body: Dict, **kwargs): + async def embedding_search(self, request_body: Dict, user: str, **kwargs): raise NotImplementedError("Provider not implemented") @abstractmethod - async def payload_search(self, request_body: Dict, **kwargs): + async def payload_search(self, request_body: Dict, user: str, **kwargs): raise NotImplementedError("Provider not implemented") - async def perform_operation(self, op_type: Text, request_body: Dict, **kwargs): + async def perform_operation(self, op_type: Text, request_body: Dict, user: str, **kwargs): supported_ops = {DbActionOperationType.payload_search.value: self.payload_search, DbActionOperationType.embedding_search.value: self.embedding_search} if op_type not in supported_ops.keys(): raise AppException("Operation type not supported") - return await supported_ops[op_type](request_body, **kwargs) + return await supported_ops[op_type](request_body, user, **kwargs) diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index 893a310ad..12b5268e2 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -29,23 +29,15 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 - def truncate_text(self, text: Text) -> Text: - """ - Truncate text to 8191 tokens for openai - """ - tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] - return self.tokenizer.decode(tokens) + async def __get_embedding(self, text: Text, user: str, **kwargs) -> List[float]: + return await self.llm.get_embedding(text, user=user) - async def __get_embedding(self, text: Text, **kwargs) -> List[float]: - result, _ = await self.llm.get_embedding(text, user=kwargs.get('user'), bot=kwargs.get('bot')) - return result - - async def embedding_search(self, request_body: Dict, **kwargs): + async def embedding_search(self, request_body: Dict, user: str, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points") if request_body.get("text"): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/search") user_msg = request_body.get("text") - vector = await self.__get_embedding(user_msg, **kwargs) + vector = await self.__get_embedding(user_msg, user, **kwargs) request_body = {'vector': vector, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} embedding_search_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', @@ -53,7 +45,7 @@ async def embedding_search(self, request_body: Dict, **kwargs): request_body=request_body) return embedding_search_result - async def payload_search(self, request_body: Dict, **kwargs): + async def payload_search(self, request_body: Dict, user, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/scroll") payload_filter_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', diff --git a/kairon/train.py b/kairon/train.py index 0276f7bc5..3ddcf9eb0 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -102,7 +102,7 @@ def start_training(bot: str, user: str, token: str = None): settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: llm_processor = LLMProcessor(bot) - faqs = asyncio.run(llm_processor.train(user=user, bot=bot)) + faqs = asyncio.run(llm_processor.train(user=user)) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) agent_url = Utility.environment['model']['agent'].get('url') diff --git a/metadata/integrations.yml b/metadata/integrations.yml index 227b4c413..6ba78b15f 100644 --- a/metadata/integrations.yml +++ b/metadata/integrations.yml @@ -129,10 +129,6 @@ llm: minimum: 1 maximum: 5 description: "The n hyperparameter controls the number of different response options that are generated by the model." - stream: - type: boolean - default: false - description: "the stream hyperparameter controls whether the model generates a continuous stream of text or a single response." stop: anyOf: - type: "string" diff --git a/tests/conftest.py b/tests/conftest.py index c613d74a5..10bd6434c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ from kairon.shared.concurrency.actors.factory import ActorFactory import pytest -from mock import patch import os diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index 1941a8fd6..e54daa27e 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -10497,7 +10497,7 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10564,7 +10564,7 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10634,7 +10634,7 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10859,7 +10859,7 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion.return_value = {'choices': [{'delta': {'role': 'assistant', 'content': generated_text}, 'finish_reason': None, 'index': 0}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10875,6 +10875,7 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m response = client.post("/webhook", json=request_object) response_json = response.json() + print(response_json['events']) assert response_json['events'] == [ {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': generated_text}] assert response_json['responses'] == [ @@ -11161,14 +11162,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11245,7 +11246,7 @@ def mock_completion_for_answer(*args, **kwargs): 'content': 'Kanban is a workflow management tool which visualizes both the process (the workflow) and the actual work passing through that process.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) @@ -11253,7 +11254,7 @@ def mock_completion_for_answer(*args, **kwargs): 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11370,14 +11371,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11477,14 +11478,14 @@ def mock_completion_for_answer(*args, **kwargs): 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11575,14 +11576,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11646,7 +11647,7 @@ def __mock_fetch_similar(*args, **kwargs): 'content': "\nInstructions on how to use Similarity Prompt:\n['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Kanban And Scrum Together? \nA:"}], 'metadata': {'user': user, 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11710,7 +11711,7 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], 'metadata': {'user': 'udit.pandey', 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11777,5 +11778,5 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], 'metadata': {'user': user, 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 1b4219e76..8d99a6c06 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -47,6 +47,7 @@ KAIRON_TWO_STAGE_FALLBACK, FeatureMappings, DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_LLM ) from kairon.shared.data.data_objects import ( Stories, @@ -1573,7 +1574,6 @@ def test_get_live_agent_with_no_live_agent(): def test_enable_live_agent(): - bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.live_agent_enabled = True bot_settings.save() @@ -1596,7 +1596,6 @@ def test_enable_live_agent(): assert actual["success"] - def test_get_live_agent_after_enabled_no_bot_settings_enabled(): bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.live_agent_enabled = False @@ -1613,6 +1612,7 @@ def test_get_live_agent_after_enabled_no_bot_settings_enabled(): assert not actual["message"] assert actual["success"] + def test_get_live_agent_after_enabled(): bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.live_agent_enabled = True @@ -2349,7 +2349,9 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters()} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -3086,6 +3088,8 @@ def _mock_get_bot_settings(*args, **kwargs): ], "num_bot_responses": 5, "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3143,6 +3147,8 @@ def _mock_get_bot_settings(*args, **kwargs): ], "num_bot_responses": 5, "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3187,6 +3193,8 @@ def test_add_prompt_action_with_invalid_query_prompt(): ], "num_bot_responses": 5, "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3244,6 +3252,8 @@ def test_add_prompt_action_with_invalid_num_bot_responses(): ], "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 10, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3300,7 +3310,9 @@ def test_add_prompt_action_with_invalid_system_prompt_source(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3365,7 +3377,9 @@ def test_add_prompt_action_with_multiple_system_prompt(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3422,7 +3436,9 @@ def test_add_prompt_action_with_empty_llm_prompt_name(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3479,7 +3495,9 @@ def test_add_prompt_action_with_empty_data_for_static_prompt(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3540,7 +3558,9 @@ def test_add_prompt_action_with_multiple_history_source_prompts(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3600,6 +3620,8 @@ def test_add_prompt_action_with_gpt_feature_disabled(): "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3616,6 +3638,138 @@ def test_add_prompt_action_with_gpt_feature_disabled(): assert actual["error_code"] == 422 +def test_add_prompt_action_with_invalid_llm_type(monkeypatch): + def _mock_get_bot_settings(*args, **kwargs): + return BotSettings( + bot=pytest.bot, + user="integration@demo.ai", + llm_settings=LLMSettings(enable_faq=True), + ) + + monkeypatch.setattr(MongoProcessor, "get_bot_settings", _mock_get_bot_settings) + action = { + "name": "test_add_prompt_action_with_invalid_llm_type", 'user_question': {'type': 'from_user_message'}, + "llm_prompts": [ + { + "name": "System Prompt", + "data": "You are a personal assistant.", + "type": "system", + "source": "static", + "is_enabled": True, + }, + { + "name": "Similarity Prompt", + "data": "Bot_collection", + "instructions": "Answer question based on the context above, if answer is not in the context go check previous logs.", + "type": "user", + "source": "bot_content", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "If there is no specific query, assume that user is aking about java programming.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + ], + "instructions": ["Answer in a short manner.", "Keep it simple."], + "num_bot_responses": 5, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": "test", + "hyperparameters": Utility.get_default_llm_hyperparameters() + } + response = client.post( + f"/api/bot/{pytest.bot}/action/prompt", + json=action, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + assert not DeepDiff(actual["message"], + [{'loc': ['body', 'llm_type'], 'msg': 'Invalid llm type', 'type': 'value_error'}], + ignore_order=True) + assert not actual["success"] + assert not actual["data"] + assert actual["error_code"] == 422 + + +def test_add_prompt_action_with_invalid_hyperameters(monkeypatch): + temp = Utility.get_default_llm_hyperparameters() + temp['temperature'] = 3.0 + + def _mock_get_bot_settings(*args, **kwargs): + return BotSettings( + bot=pytest.bot, + user="integration@demo.ai", + llm_settings=LLMSettings(enable_faq=True), + ) + + monkeypatch.setattr(MongoProcessor, "get_bot_settings", _mock_get_bot_settings) + action = { + "name": "test_add_prompt_action_with_invalid_hyperameters", 'user_question': {'type': 'from_user_message'}, + "llm_prompts": [ + { + "name": "System Prompt", + "data": "You are a personal assistant.", + "type": "system", + "source": "static", + "is_enabled": True, + }, + { + "name": "Similarity Prompt", + "data": "Bot_collection", + "instructions": "Answer question based on the context above, if answer is not in the context go check previous logs.", + "type": "user", + "source": "bot_content", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "If there is no specific query, assume that user is aking about java programming.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + ], + "instructions": ["Answer in a short manner.", "Keep it simple."], + "num_bot_responses": 5, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": temp + } + response = client.post( + f"/api/bot/{pytest.bot}/action/prompt", + json=action, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + assert not DeepDiff(actual["message"], + [{'loc': ['body', 'hyperparameters'], + 'msg': "['temperature']: 3.0 is greater than the maximum of 2.0", 'type': 'value_error'}], + ignore_order=True) + assert not actual["success"] + assert not actual["data"] + assert actual["error_code"] == 422 + + def test_add_prompt_action(monkeypatch): def _mock_get_bot_settings(*args, **kwargs): return BotSettings( @@ -3662,7 +3816,9 @@ def _mock_get_bot_settings(*args, **kwargs): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3723,7 +3879,9 @@ def _mock_get_bot_settings(*args, **kwargs): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3774,7 +3932,9 @@ def test_update_prompt_action_does_not_exist(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/61512cc2c6219f0aae7bba3d", @@ -3826,7 +3986,9 @@ def test_update_prompt_action_with_invalid_similarity_threshold(): }, ], "num_bot_responses": 5, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -3871,7 +4033,9 @@ def test_update_prompt_action_with_invalid_top_results(): }, ], "num_bot_responses": 5, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -3915,7 +4079,9 @@ def test_update_prompt_action_with_invalid_num_bot_responses(): }, ], "num_bot_responses": 50, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -3964,6 +4130,8 @@ def test_update_prompt_action_with_invalid_query_prompt(): "num_bot_responses": 5, "use_query_prompt": True, "query_prompt": "", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4021,6 +4189,8 @@ def test_update_prompt_action_with_query_prompt_with_false(): }, ], "dispatch_response": False, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4077,7 +4247,9 @@ def test_update_prompt_action(): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4107,7 +4279,7 @@ def test_get_prompt_action(): 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity_analytical Prompt', 'data': 'Bot_collection', @@ -4171,7 +4343,9 @@ def _mock_get_bot_settings(*args, **kwargs): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -4200,7 +4374,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'user_question': {'type': 'from_user_message'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -4269,7 +4443,10 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() + } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -4289,7 +4466,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'user_question': {'type': 'from_user_message'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -4361,7 +4538,10 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() + } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -4381,7 +4561,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'user_question': {'type': 'from_user_message'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -5036,7 +5216,8 @@ def test_get_data_importer_logs(): assert actual['data']["logs"][3]['event_status'] == EVENT_STATUS.COMPLETED.value assert actual['data']["logs"][3]['status'] == 'Failure' assert set(actual['data']["logs"][3]['files_received']) == {'rules', 'stories', 'nlu', 'domain', 'config', - 'actions', 'chat_client_config', 'multiflow_stories', 'bot_content'} + 'actions', 'chat_client_config', 'multiflow_stories', + 'bot_content'} assert actual['data']["logs"][3]['is_data_uploaded'] assert actual['data']["logs"][3]['start_timestamp'] assert actual['data']["logs"][3]['end_timestamp'] @@ -5068,7 +5249,8 @@ def test_get_data_importer_logs(): ] assert actual['data']["logs"][3]['is_data_uploaded'] assert set(actual['data']["logs"][3]['files_received']) == {'rules', 'stories', 'nlu', 'config', 'domain', - 'actions', 'chat_client_config', 'multiflow_stories','bot_content'} + 'actions', 'chat_client_config', 'multiflow_stories', + 'bot_content'} @responses.activate @@ -6197,6 +6379,7 @@ def test_add_story_lone_intent(): } ] + def test_add_story_consecutive_intents(): response = client.post( f"/api/bot/{pytest.bot}/stories", @@ -9850,6 +10033,7 @@ def test_login_for_verified(): pytest.access_token = actual["data"]["access_token"] pytest.token_type = actual["data"]["token_type"] + def test_list_bots_for_different_user(): response = client.get( "/api/account/bot", @@ -18696,7 +18880,7 @@ def test_set_templates_with_sysadmin_as_user(): intents = Intents.objects(bot=pytest.bot) intents = [{k: v for k, v in intent.to_mongo().to_dict().items() if k not in ['_id', 'bot', 'timestamp']} for - intent in intents] + intent in intents] assert intents == [ {'name': 'greet', 'user': 'sysadmin', 'status': True, 'is_integration': False, 'use_entities': False}, @@ -18772,7 +18956,6 @@ def test_add_channel_config(monkeypatch): def test_add_bot_with_template_with_sysadmin_as_user(monkeypatch): - def mock_reload_model(*args, **kwargs): mock_reload_model.called_with = (args, kwargs) return None @@ -18811,7 +18994,7 @@ def mock_reload_model(*args, **kwargs): rules = Rules.objects(bot=bot_id) rules = [{k: v for k, v in rule.to_mongo().to_dict().items() if k not in ['_id', 'bot', 'timestamp']} for - rule in rules] + rule in rules] assert rules == [ {'block_name': 'ask the user to rephrase whenever they send a message with low nlu confidence', @@ -18825,7 +19008,7 @@ def mock_reload_model(*args, **kwargs): utterances = Utterances.objects(bot=bot_id) utterances = [{k: v for k, v in utterance.to_mongo().to_dict().items() if k not in ['_id', 'bot', 'timestamp']} for - utterance in utterances] + utterance in utterances] assert utterances == [ {'name': 'utter_please_rephrase', 'user': 'sysadmin', 'status': True}, @@ -19940,7 +20123,7 @@ def test_get_bot_settings(): 'whatsapp': 'meta', 'cognition_collections_limit': 3, 'cognition_columns_per_collection_limit': 5, - 'integrations_per_user_limit':3 } + 'integrations_per_user_limit': 3} def test_update_analytics_settings_with_empty_value(): @@ -20018,7 +20201,7 @@ def test_update_analytics_settings(): 'live_agent_enabled': False, 'cognition_collections_limit': 3, 'cognition_columns_per_collection_limit': 5, - 'integrations_per_user_limit':3 } + 'integrations_per_user_limit': 3} def test_delete_channels_config(): diff --git a/tests/unit_test/action/action_test.py b/tests/unit_test/action/action_test.py index 715fecc12..079c17a64 100644 --- a/tests/unit_test/action/action_test.py +++ b/tests/unit_test/action/action_test.py @@ -2657,7 +2657,7 @@ def test_get_prompt_action_config(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'bot': 'test_action_server', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', @@ -3949,7 +3949,7 @@ def test_get_prompt_action_config_2(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'bot': 'test_bot_action_test', 'user': 'test_user_action_test', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'dispatch_response': True, 'set_slots': [], 'llm_type': 'openai', diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index b829acf05..5a9a0bd30 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -170,7 +170,7 @@ def test_add_prompt_action_with_invalid_slots(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', @@ -198,7 +198,7 @@ def test_add_prompt_action_with_invalid_http_action(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', @@ -227,7 +227,7 @@ def test_add_prompt_action_with_invalid_similarity_threshold(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -257,7 +257,7 @@ def test_add_prompt_action_with_invalid_top_results(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -309,7 +309,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', @@ -354,7 +354,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', @@ -554,7 +554,7 @@ def test_add_prompt_action_with_empty_llm_prompts(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': []} with pytest.raises(ValidationError, match="llm_prompts are required!"): @@ -581,7 +581,7 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -602,7 +602,7 @@ def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -619,7 +619,7 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': ["\n", ".", "?", "!", ";"], + 'n': 1, 'stop': ["\n", ".", "?", "!", ";"], 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -639,7 +639,7 @@ def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': -3.0, + 'n': 1, 'stop': '?', 'presence_penalty': -3.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -657,7 +657,7 @@ def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 3.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -674,7 +674,7 @@ def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -691,7 +691,7 @@ def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -708,7 +708,7 @@ def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt-3.5-turbo', 'top_p': 3.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -725,7 +725,7 @@ def test_add_prompt_action_with_invalid_n_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 7, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 7, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -742,7 +742,7 @@ def test_add_prompt_action_with_zero_n_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 0, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 0, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -759,7 +759,7 @@ def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 2, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 2, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': 'a'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -834,7 +834,7 @@ def test_edit_prompt_action_faq_action(self): assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -869,7 +869,7 @@ def test_edit_prompt_action_faq_action(self): 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -912,7 +912,7 @@ def test_edit_prompt_action_with_less_hyperparameters(self): 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -947,7 +947,7 @@ def test_get_prompt_faq_action(self): 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index c8f727db3..7738dfeb0 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -8,6 +8,7 @@ from aiohttp import ClientConnectionError from mongoengine import connect from kairon.shared.utils import Utility + Utility.load_system_metadata() from kairon.exceptions import AppException @@ -66,7 +67,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, @@ -119,14 +120,16 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() embedding = list(np.random.random(LLMProcessor.__embedding__)) - mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]} + mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, { + 'data': [{'embedding': embedding}]} gpt3 = LLMProcessor(bot) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), method="GET", - payload={"time": 0, "status": "ok", "result": {"collections": [{"name": "test_embed_faq_text_swift_faq_embd"}, - {"name": "example_bot_swift_faq_embd"}]}} + payload={"time": 0, "status": "ok", + "result": {"collections": [{"name": "test_embed_faq_text_swift_faq_embd"}, + {"name": "example_bot_swift_faq_embd"}]}} ) aioresponses.add( @@ -141,19 +144,22 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_user_details{gpt3.suffix}/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_user_details{gpt3.suffix}/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_country_details{gpt3.suffix}"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_country_details{gpt3.suffix}"), method="PUT", status=200 ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_country_details{gpt3.suffix}/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_country_details{gpt3.suffix}/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) @@ -165,24 +171,29 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 3 - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'name': f"{gpt3.bot}_country_details{gpt3.suffix}", - 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'name': f"{gpt3.bot}_country_details{gpt3.suffix}", + 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content_two.vector_id, - 'vector': embedding, - 'payload': {'country': 'Spain'}}]} - assert list(aioresponses.requests.values())[3][1].kwargs['json'] == {'points': [{'id': test_content_three.vector_id, - 'vector': embedding, - 'payload': {'role': 'ds'}}]} + assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + 'points': [{'id': test_content_two.vector_id, + 'vector': embedding, + 'payload': {'country': 'Spain'}}]} + assert list(aioresponses.requests.values())[3][1].kwargs['json'] == { + 'points': [{'id': test_content_three.vector_id, + 'vector': embedding, + 'payload': {'role': 'ds'}}]} - assert list(aioresponses.requests.values())[4][0].kwargs['json'] == {'name': f"{gpt3.bot}_user_details{gpt3.suffix}", - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[5][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, - 'payload': {'name': 'Nupur'}}]} + assert list(aioresponses.requests.values())[4][0].kwargs['json'] == { + 'name': f"{gpt3.bot}_user_details{gpt3.suffix}", + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[5][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, + 'payload': {'name': 'Nupur'}}]} assert response['faq'] == 3 expected = {"model": "text-embedding-3-small", @@ -217,7 +228,8 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a gpt3 = LLMProcessor(bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), method="PUT", status=200 ) @@ -227,18 +239,21 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a payload={"time": 0, "status": "ok", "result": {"collections": []}}) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/test_embed_faq_json_payload_with_int_faq_embd/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 1 - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_embed_faq_json_payload_with_int_faq_embd', - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == { + 'name': 'test_embed_faq_json_payload_with_int_faq_embd', + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, 'vector': embedding, 'payload': {'name': 'Ram', 'age': 23, 'color': 'red'} }]} @@ -302,7 +317,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_int_embd_int_faq_embd', @@ -363,16 +378,18 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aiore url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}{gpt3.suffix}/points"), method="PUT", payload={"result": None, - 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, - "time": 0.003612634} + 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, + "time": 0.003612634} ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train(user=user, bot=bot) + await gpt3.train(user=user) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, 'payload': {'content': test_content.data}}]} + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, 'payload': {'content': test_content.data}}]} expected = {"model": "text-embedding-3-small", "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, @@ -412,33 +429,38 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb aioresponses.add( method="DELETE", - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd"), ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd"), method="PUT", status=200 ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd/points"), method="PUT", payload={"result": None, - 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, - "time": 0.003612634} + 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, + "time": 0.003612634} ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train(user=user, bot=bot) + await gpt3.train(user=user) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == { + 'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} expected_payload = test_content.data #expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, - 'payload': expected_payload - }]} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, + 'payload': expected_payload + }]} expected = {"model": "text-embedding-3-small", "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, @@ -449,7 +471,7 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict" @@ -469,9 +491,9 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], "hyperparameters": hyperparameters } mock_completion_request = {"messages": [ @@ -487,13 +509,14 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, @@ -514,7 +537,8 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict_with_default_collection" @@ -559,7 +583,7 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_emb {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, @@ -599,11 +623,11 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], "hyperparameters": hyperparameters - } + } mock_completion_request = {"messages": [ {"role": "system", @@ -615,17 +639,18 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + with mock.patch.dict(Utility.environment, {'vector': {"key": "test", 'db': "http://localhost:6333"}}): gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=gpt3.bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert gpt3.logs == [ {'messages': [{'role': 'system', @@ -638,10 +663,12 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 @@ -659,25 +686,133 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) - user = "test" - bot = "test_gpt3_faq_embedding_predict_with_values_with_instructions" - key = 'test' + test_content = CognitionData( - data="Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ", - collection='java', bot=bot, user=user).save() - BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() + data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", + collection='python', bot="test_gpt3_faq_embedding_predict_with_values_and_stream", user="test").save() + generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" hyperparameters = Utility.get_default_llm_hyperparameters() + hyperparameters['stream'] = True + key = 'test' + user = "tests" + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": "java"}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], + "hyperparameters": hyperparameters + } + + mock_completion_request = {"messages": [ + {"role": "system", + "content": "You are a personal assistant. Answer the question according to the below context"}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} + ]} + mock_completion_request.update(hyperparameters) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.side_effect = [{'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[29:60]}, + 'finish_reason': None, 'index': 0}]}, + {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[60:]}, + 'finish_reason': 'stop', 'index': 0}]} + ] + + with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + gpt3 = LLMProcessor(test_content.bot) + + aioresponses.add( + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + method="POST", + payload={'result': [ + {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} + ) + + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + assert response['content'] == "Python is dynamically typed, " + assert gpt3.logs == [ + {'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': True, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} + + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': gpt3.bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + @pytest.mark.asyncio + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, + mock_embedding, + mock_completion, + aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "payload_with_instruction" + key = 'test' + CognitionSchema( + metadata=[{"column_name": "name", "data_type": "str", "enable_search": True, "create_embeddings": True}, + {"column_name": "city", "data_type": "str", "enable_search": True, "create_embeddings": True}], + collection_name="User_details", + bot=bot, user=user + ).save() + test_content1 = CognitionData( + data={"name": "Nupur", "city": "Pune"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + test_content2 = CognitionData( + data={"name": "Fahad", "city": "Mumbai"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + test_content3 = CognitionData( + data={"name": "Hitesh", "city": "Mumbai"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=bot, user=user).save() + + generated_text = "Hitesh and Fahad lives in mumbai city." + query = "List all the user lives in mumbai city" + hyperparameters = Utility.get_default_llm_hyperparameters() + k_faq_action_config = { + "system_prompt": "You are a personal assistant. Answer the question according to the below context", + "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", + "similarity_prompt": [{"top_results": 10, + "similarity_threshold": 0.70, + 'use_similarity_prompt': True, + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": "user_details"}], 'instructions': ['Answer in a short way.', 'Keep it simple.'], "hyperparameters": hyperparameters } @@ -686,39 +821,39 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mo {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"} + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n[{'name': 'Fahad', 'city': 'Mumbai'}, {'name': 'Hitesh', 'city': 'Mumbai'}]\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: List all the user lives in mumbai city \nA:"} ]} mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot) - + gpt3 = LLMProcessor(bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content1.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ - {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} + {'id': test_content2.vector_id, 'score': 0.80, "payload": test_content2.data}, + {'id': test_content3.vector_id, 'score': 0.80, "payload": test_content3.data} + ]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text - assert gpt3.logs == [ - {'messages': [{'role': 'system', - 'content': 'You are a personal assistant. Answer the question according to the below context'}, - {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': { - 'choices': [{'message': {'content': 'Python is dynamically typed, garbage-collected, ' - 'high level, general purpose programming.', - 'role': 'assistant'}}]}, - 'type': 'answer_query', - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}}] - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert gpt3.logs == [{'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n[{'name': 'Fahad', 'city': 'Mumbai'}, {'name': 'Hitesh', 'city': 'Mumbai'}]\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: List all the user lives in mumbai city \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Hitesh and Fahad lives in mumbai city.', 'role': 'assistant'}}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', + 'top_p': 0.0, 'n': 1, 'stop': None, + 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 @@ -736,7 +871,8 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mo @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_gpt3_faq_embedding_predict_completion_connection_error" user = 'test' @@ -755,11 +891,11 @@ async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters - } + } def __mock_connection_error(*args, **kwargs): raise Exception("Connection reset by peer!") @@ -770,18 +906,21 @@ def __mock_connection_error(*args, **kwargs): gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", @@ -795,7 +934,7 @@ def __mock_connection_error(*args, **kwargs): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error'}, 'api_key': 'test', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, - 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -804,7 +943,7 @@ def __mock_connection_error(*args, **kwargs): @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock_llm_request): embedding = list(np.random.random(LLMProcessor.__embedding__)) - user ="test" + user = "test" bot = "test_gpt3_faq_embedding_predict_exact_match" key = 'test' test_content = CognitionData( @@ -818,9 +957,9 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters } @@ -829,16 +968,17 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock gpt3 = LLMProcessor(test_content.bot) - response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_exact_match", **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} - assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] + assert gpt3.logs == [ + {'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, - "api_key": key, - "num_retries": 3} + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio @@ -867,7 +1007,7 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ gpt3 = LLMProcessor(test_content.bot) mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] - response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_embedding_connection_error", **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] @@ -882,7 +1022,8 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_gpt3_faq_embedding_predict_with_previous_bot_responses" @@ -921,13 +1062,14 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, @@ -970,11 +1112,11 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding "query_prompt": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", "use_query_prompt": True}, "similarity_prompt": [ - {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters - } + } mock_rephrase_request = {"messages": [ {"role": "system", @@ -993,18 +1135,20 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, { + 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, diff --git a/tests/unit_test/vector_embeddings/qdrant_test.py b/tests/unit_test/vector_embeddings/qdrant_test.py index 7bf166116..667285715 100644 --- a/tests/unit_test/vector_embeddings/qdrant_test.py +++ b/tests/unit_test/vector_embeddings/qdrant_test.py @@ -14,7 +14,9 @@ from kairon.shared.data.data_objects import LLMSettings from kairon.shared.vector_embeddings.db.factory import VectorEmbeddingsDbFactory from kairon.shared.vector_embeddings.db.qdrant import Qdrant - +import litellm +from kairon.shared.llm.processor import LLMProcessor +import numpy as np class TestQdrant: @@ -25,16 +27,21 @@ def init_connection(self): connect(**Utility.mongoengine_connection(Utility.environment['database']["url"])) @pytest.mark.asyncio + @mock.patch.dict(Utility.environment, {'vector': {"key": "TEST", 'db': 'http://localhost:6333'}}) + @mock.patch.object(litellm, "aembedding", autospec=True) @mock.patch.object(ActionUtility, "execute_http_request", autospec=True) - async def test_embedding_search_valid_request_body(self, mock_http_request): + async def test_embedding_search_valid_request_body(self, mock_http_request, mock_embedding): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" Utility.load_environment() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value", bot="5f50fd0a56v098ca10d75d2g", user="user").save() qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) - request_body = {"ids": [0], "with_payload": True, "with_vector": True} + request_body = {"ids": [0], "with_payload": True, "with_vector": True, 'text': "Hi"} mock_http_request.return_value = 'expected_result' - result = await qdrant.embedding_search(request_body) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + result = await qdrant.embedding_search(request_body, user=user) assert result == 'expected_result' @pytest.mark.asyncio @@ -46,29 +53,31 @@ async def test_payload_search_valid_request_body(self, mock_http_request): request_body = {"filter": {"should": [{"key": "city", "match": {"value": "London"}}, {"key": "color", "match": {"value": "red"}}]}} mock_http_request.return_value = 'expected_result' - result = await qdrant.payload_search(request_body) + result = await qdrant.payload_search(request_body, user="test") assert result == 'expected_result' @pytest.mark.asyncio @mock.patch.object(ActionUtility, "execute_http_request", autospec=True) async def test_perform_operation_valid_op_type_and_request_body(self, mock_http_request): Utility.load_environment() + user = "test" qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) request_body = {} mock_http_request.return_value = 'expected_result' - result_embedding = await qdrant.perform_operation('embedding_search', request_body) + result_embedding = await qdrant.perform_operation('embedding_search', request_body, user=user) assert result_embedding == 'expected_result' - result_payload = await qdrant.perform_operation('payload_search', request_body) + result_payload = await qdrant.perform_operation('payload_search', request_body, user=user) assert result_payload == 'expected_result' @pytest.mark.asyncio async def test_embedding_search_empty_request_body(self): Utility.load_environment() + user = "test" qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) with pytest.raises(ActionFailure): - await qdrant.embedding_search({}) + await qdrant.embedding_search({}, user=user) @pytest.mark.asyncio async def test_payload_search_empty_request_body(self): @@ -76,7 +85,7 @@ async def test_payload_search_empty_request_body(self): qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) with pytest.raises(ActionFailure): - await qdrant.payload_search({}) + await qdrant.payload_search({}, user="test") @pytest.mark.asyncio async def test_perform_operation_invalid_op_type(self): @@ -85,7 +94,7 @@ async def test_perform_operation_invalid_op_type(self): LLMSettings(provider="openai").to_mongo().to_dict()) request_body = {} with pytest.raises(AppException, match="Operation type not supported"): - await qdrant.perform_operation("vector_search", request_body) + await qdrant.perform_operation("vector_search", request_body, user="test") def test_get_instance_raises_exception_when_db_not_implemented(self): with pytest.raises(AppException, match="Database not yet implemented!"): @@ -99,7 +108,7 @@ async def test_embedding_search_valid_request_body_payload(self, mock_http_reque LLMSettings(provider="openai").to_mongo().to_dict()) request_body = {'ids': [0], 'with_payload': True, 'with_vector': True} mock_http_request.return_value = 'expected_result' - result = await qdrant.embedding_search(request_body) + result = await qdrant.embedding_search(request_body, user="test") assert result == 'expected_result' mock_http_request.assert_called_once() From 90a47aa00632f3be3c1503bffbdc94b0c6d56f94 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 10:40:04 +0530 Subject: [PATCH 03/57] test cased fixed --- tests/unit_test/data_processor/data_processor_test.py | 6 ++++-- tests/unit_test/utility_test.py | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 5a9a0bd30..08320bc27 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -60,7 +60,7 @@ from kairon.shared.data.constant import UTTERANCE_TYPE, EVENT_STATUS, STORY_EVENT, ALLOWED_DOMAIN_FORMATS, \ ALLOWED_CONFIG_FORMATS, ALLOWED_NLU_FORMATS, ALLOWED_STORIES_FORMATS, ALLOWED_RULES_FORMATS, REQUIREMENTS, \ DEFAULT_NLU_FALLBACK_RULE, SLOT_TYPE, KAIRON_TWO_STAGE_FALLBACK, AuditlogActions, TOKEN_TYPE, GPT_LLM_FAQ, \ - DEFAULT_CONTEXT_PROMPT, DEFAULT_NLU_FALLBACK_RESPONSE, DEFAULT_SYSTEM_PROMPT + DEFAULT_CONTEXT_PROMPT, DEFAULT_NLU_FALLBACK_RESPONSE, DEFAULT_SYSTEM_PROMPT, DEFAULT_LLM from kairon.shared.data.data_objects import (TrainingExamples, Slots, Entities, EntitySynonyms, RegexFeatures, @@ -8488,7 +8488,9 @@ def test_delete_action_with_attached_http_action(self): 'data': 'tester_action', 'instructions': 'Answer according to the context', 'type': 'user', 'source': 'action', - 'is_enabled': True}] + 'is_enabled': True}], + llm_type=DEFAULT_LLM, + hyperparameters=Utility.get_default_llm_hyperparameters() ) processor.add_http_action_config(http_action_config.dict(), user, bot) processor.add_prompt_action(prompt_action_config.dict(), bot, user) diff --git a/tests/unit_test/utility_test.py b/tests/unit_test/utility_test.py index 438633592..cbf42ee63 100644 --- a/tests/unit_test/utility_test.py +++ b/tests/unit_test/utility_test.py @@ -2944,7 +2944,6 @@ def test_get_llm_hyperparameters(self): "model": "gpt-3.5-turbo", "top_p": 0.0, "n": 1, - "stream": False, "stop": None, "presence_penalty": 0.0, "frequency_penalty": 0.0, From 3c8bb1bc6525ba3759290e59290cc9b6145d0c54 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 15:48:54 +0530 Subject: [PATCH 04/57] 1. added missing test case 2. updated litellm --- kairon/shared/llm/logger.py | 27 ++++++++++++--------------- requirements/prod.txt | 2 +- tests/unit_test/llm_test.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py index 06b720b48..dbb93e469 100644 --- a/kairon/shared/llm/logger.py +++ b/kairon/shared/llm/logger.py @@ -1,14 +1,10 @@ from litellm.integrations.custom_logger import CustomLogger from .data_objects import LLMLogs import ujson as json +from loguru import logger class LiteLLMLogger(CustomLogger): - def log_pre_api_call(self, model, messages, kwargs): - pass - - def log_post_api_call(self, kwargs, response_obj, start_time, end_time): - pass def log_stream_event(self, kwargs, response_obj, start_time, end_time): self.__logs_litellm(**kwargs) @@ -29,15 +25,16 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti self.__logs_litellm(**kwargs) def __logs_litellm(self, **kwargs): - litellm_params = kwargs['litellm_params'] - self.__save_logs(**{'response': json.loads(kwargs['original_response']), - 'start_time': kwargs['start_time'], - 'end_time': kwargs['end_time'], - 'cost': kwargs["response_cost"], - 'llm_call_id': litellm_params['litellm_call_id'], - 'llm_provider': litellm_params['custom_llm_provider'], - 'model_params': kwargs["additional_args"]["complete_input_dict"], - 'metadata': litellm_params['metadata']}) + logger.info("logging llms call") + litellm_params = kwargs.get('litellm_params') + self.__save_logs(**{'response': json.loads(kwargs.get('original_response')) if kwargs.get('original_response') else None, + 'start_time': kwargs.get('start_time'), + 'end_time': kwargs.get('end_time'), + 'cost': kwargs.get("response_cost"), + 'llm_call_id': litellm_params.get('litellm_call_id'), + 'llm_provider': litellm_params.get('custom_llm_provider'), + 'model_params': kwargs.get("additional_args", {}).get("complete_input_dict"), + 'metadata': litellm_params.get('metadata')}) def __save_logs(self, **kwargs): - LLMLogs(**kwargs).save() + print(LLMLogs(**kwargs).save().to_mongo().to_dict()) diff --git a/requirements/prod.txt b/requirements/prod.txt index e00d448a9..13fcc40e7 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -64,6 +64,6 @@ opentelemetry-instrumentation-requests==0.46b0 opentelemetry-instrumentation-sklearn==0.46b0 pykwalify==1.8.0 gunicorn==22.0.0 -litellm==1.38.11 +litellm==1.39.5 jsonschema_rs==0.18.0 mongoengine-jsonschema==0.1.3 \ No newline at end of file diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 7738dfeb0..bb446e802 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1,4 +1,5 @@ import os +import time from urllib.parse import urljoin import mock @@ -17,6 +18,7 @@ from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT from kairon.shared.llm.processor import LLMProcessor +from kairon.shared.llm.data_objects import LLMLogs import litellm from deepdiff import DeepDiff @@ -1166,3 +1168,33 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + @pytest.mark.asyncio + async def test_llm_logging(self): + from kairon.shared.llm.logger import LiteLLMLogger + bot = "test_llm_logging" + user = "test" + litellm.callbacks = [LiteLLMLogger()] + + result = await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + metadata={'user': user, 'bot': bot}) + assert result + + result = litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + metadata={'user': user, 'bot': bot}) + assert result + + result = litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + stream=True, + metadata={'user': user, 'bot': bot}) + for chunk in result: + print(chunk["choices"][0]["delta"]["content"]) + assert chunk["choices"][0]["delta"]["content"] + + assert list(LLMLogs.objects(metadata__bot=bot)) From 76ed000af983eadd50df7225a06769aac8bbeed2 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 17:28:09 +0530 Subject: [PATCH 05/57] 1. added missing test case --- tests/unit_test/llm_test.py | 56 +++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index bb446e802..09eea5168 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -688,7 +688,8 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) test_content = CognitionData( @@ -720,7 +721,8 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe ]} mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.side_effect = [{'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + mock_completion.side_effect = [{'choices': [ + {'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[29:60]}, 'finish_reason': None, 'index': 0}]}, {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[60:]}, @@ -745,7 +747,9 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe 'content': 'You are a personal assistant. Answer the question according to the below context'}, {'role': 'user', 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + 'raw_completion_response': {'choices': [ + {'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, + 'index': 0}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': True, 'stop': None, 'presence_penalty': 0.0, @@ -1177,17 +1181,17 @@ async def test_llm_logging(self): litellm.callbacks = [LiteLLMLogger()] result = await litellm.acompletion(messages=["Hi"], - model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", - metadata={'user': user, 'bot': bot}) - assert result - - result = litellm.completion(messages=["Hi"], model="gpt-3.5-turbo", mock_response="Hi, How may i help you?", metadata={'user': user, 'bot': bot}) assert result + result = litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + metadata={'user': user, 'bot': bot}) + assert result + result = litellm.completion(messages=["Hi"], model="gpt-3.5-turbo", mock_response="Hi, How may i help you?", @@ -1197,4 +1201,38 @@ async def test_llm_logging(self): print(chunk["choices"][0]["delta"]["content"]) assert chunk["choices"][0]["delta"]["content"] + result = await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + stream=True, + metadata={'user': user, 'bot': bot}) + async for chunk in result: + print(chunk["choices"][0]["delta"]["content"]) + assert chunk["choices"][0]["delta"]["content"] + assert list(LLMLogs.objects(metadata__bot=bot)) + + with pytest.raises(Exception) as e: + await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + metadata={'user': user, 'bot': bot}) + + assert str(e) == "Authentication error" + + with pytest.raises(Exception) as e: + litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + metadata={'user': user, 'bot': bot}) + + assert str(e) == "Authentication error" + + with pytest.raises(Exception) as e: + await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + stream=True, + metadata={'user': user, 'bot': bot}) + + assert str(e) == "Authentication error" From 16cb58f940c1c800fcbf4009ced7a0d9fd2c4a16 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 18:50:36 +0530 Subject: [PATCH 06/57] 1. added missing test case --- tests/unit_test/llm_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 09eea5168..bcc2ed661 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1210,8 +1210,6 @@ async def test_llm_logging(self): print(chunk["choices"][0]["delta"]["content"]) assert chunk["choices"][0]["delta"]["content"] - assert list(LLMLogs.objects(metadata__bot=bot)) - with pytest.raises(Exception) as e: await litellm.acompletion(messages=["Hi"], model="gpt-3.5-turbo", From 3bfd3681492150791051659ab6e261badd394e04 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 11:10:39 +0530 Subject: [PATCH 07/57] 1. added missing test case 2. removed deprecated api --- kairon/shared/rest_client.py | 29 +++---------------- tests/integration_test/action_service_test.py | 2 +- tests/integration_test/chat_service_test.py | 2 +- tests/integration_test/event_service_test.py | 2 +- .../integration_test/history_services_test.py | 2 +- tests/integration_test/services_test.py | 4 +-- tests/unit_test/action/action_test.py | 1 - tests/unit_test/chat/chat_test.py | 27 +++++++++-------- tests/unit_test/cli_test.py | 17 +++++------ .../data_processor/agent_processor_test.py | 2 +- .../data_processor/data_processor_test.py | 2 +- .../unit_test/data_processor/history_test.py | 2 +- tests/unit_test/events/definitions_test.py | 4 +-- tests/unit_test/events/events_test.py | 2 +- tests/unit_test/events/scheduler_test.py | 2 +- tests/unit_test/idp/test_idp_helper.py | 1 - tests/unit_test/llm_test.py | 7 ++--- tests/unit_test/plugins_test.py | 3 +- tests/unit_test/rest_client_test.py | 17 +++++++++++ tests/unit_test/verification_test.py | 2 +- 20 files changed, 59 insertions(+), 71 deletions(-) diff --git a/kairon/shared/rest_client.py b/kairon/shared/rest_client.py index 301e11323..5c144a0df 100644 --- a/kairon/shared/rest_client.py +++ b/kairon/shared/rest_client.py @@ -32,14 +32,6 @@ def __init__(self, close_session_with_rqst_completion=True): self._time_elapsed = None self._status_code = None - @property - def streaming_response(self): - return self._streaming_response - - @streaming_response.setter - def streaming_response(self, resp): - self._streaming_response = resp - @property def time_elapsed(self): return self._time_elapsed @@ -124,12 +116,9 @@ async def __trigger(self, client, *args, **kwargs) -> ClientResponse: logger.debug(f"Content-type: {response.headers['content-type']}") logger.debug(f"Status code: {str(response.status)}") self.status_code = response.status - if is_streaming_resp: - streaming_resp = await AioRestClient.parse_streaming_response(response) - self.streaming_response = streaming_resp - logger.debug(f"Raw streaming response: {streaming_resp}") - text = await response.text() - logger.debug(f"Raw response: {text}") + if not is_streaming_resp: + text = await response.text() + logger.debug(f"Raw response: {text}") return response def __validate_response(self, response: ClientResponse, **kwargs): @@ -149,14 +138,4 @@ async def cleanup(self): Close underlying connector to release all acquired resources. """ if not self.session.closed: - await self.session.close() - - @staticmethod - async def parse_streaming_response(response): - chunks = [] - async for chunk in response.content: - if not chunk: - break - chunks.append(chunk) - - return chunks + await self.session.close() \ No newline at end of file diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index e54daa27e..cd5e2c63c 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -3,7 +3,7 @@ from urllib.parse import urlencode, urljoin import litellm -import mock +from unittest import mock import numpy as np import pytest import responses diff --git a/tests/integration_test/chat_service_test.py b/tests/integration_test/chat_service_test.py index 77b828a71..04d65276a 100644 --- a/tests/integration_test/chat_service_test.py +++ b/tests/integration_test/chat_service_test.py @@ -19,7 +19,7 @@ import pytest import responses -from mock import patch +from unittest.mock import patch from mongoengine import connect from slack_sdk.web.slack_response import SlackResponse from starlette.exceptions import HTTPException diff --git a/tests/integration_test/event_service_test.py b/tests/integration_test/event_service_test.py index 445a37949..f03f65817 100644 --- a/tests/integration_test/event_service_test.py +++ b/tests/integration_test/event_service_test.py @@ -3,7 +3,7 @@ from dramatiq.brokers.stub import StubBroker from loguru import logger -from mock import patch +from unittest.mock import patch from starlette.testclient import TestClient from kairon.shared.constants import EventClass, EventExecutor diff --git a/tests/integration_test/history_services_test.py b/tests/integration_test/history_services_test.py index 2d8cc807a..46e683b43 100644 --- a/tests/integration_test/history_services_test.py +++ b/tests/integration_test/history_services_test.py @@ -8,7 +8,7 @@ from mongomock import MongoClient from kairon.history.processor import HistoryProcessor from pymongo.collection import Collection -import mock +from unittest import mock from urllib.parse import urlencode diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 8d99a6c06..5efd5c6ca 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -6,11 +6,11 @@ import tempfile from datetime import datetime, timedelta from io import BytesIO -from mock import patch +from unittest.mock import patch from urllib.parse import urljoin from zipfile import ZipFile -import mock +from unittest import mock import pytest import responses from botocore.exceptions import ClientError diff --git a/tests/unit_test/action/action_test.py b/tests/unit_test/action/action_test.py index 079c17a64..beb360969 100644 --- a/tests/unit_test/action/action_test.py +++ b/tests/unit_test/action/action_test.py @@ -3,7 +3,6 @@ import re from unittest import mock -import mock from googleapiclient.http import HttpRequest from pipedrive.exceptions import UnauthorizedError, BadRequestError from kairon.shared.utils import Utility diff --git a/tests/unit_test/chat/chat_test.py b/tests/unit_test/chat/chat_test.py index 016f660d8..b1a730536 100644 --- a/tests/unit_test/chat/chat_test.py +++ b/tests/unit_test/chat/chat_test.py @@ -3,7 +3,7 @@ import ujson as json import os from re import escape -from unittest.mock import patch +from unittest import mock from urllib.parse import urlencode, quote_plus import mongomock @@ -21,7 +21,6 @@ from kairon.shared.data.constant import ACCESS_ROLES, TOKEN_TYPE from kairon.shared.data.utils import DataUtility from kairon.shared.utils import Utility -import mock from pymongo.errors import ServerSelectionTimeoutError @@ -49,7 +48,7 @@ def test_save_channel_config_invalid(self): "test", "test" ) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -93,7 +92,7 @@ def test_save_channel_config_invalid(self): "test" ) - @patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) + @mock.patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) def test_save_channel_config_slack_team_id_error(self, mock_slack_info): mock_slack_info.side_effect = AppException("The request to the Slack API failed. ") with pytest.raises(AppException, match="The request to the Slack API failed.*"): @@ -108,7 +107,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -133,7 +132,7 @@ def __mock_get_bot(*args, **kwargs): "client_secret": "a23456789sfdghhtyutryuivcbn", "is_primary": True}}, "test", "test" ) - @patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) + @mock.patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) def test_save_channel_config_slack_secondary_app_team_id_error(self, mock_slack_info ): mock_slack_info.side_effect = AppException("The request to the Slack API failed. ") with pytest.raises(AppException, match="The request to the Slack API failed.*"): @@ -149,7 +148,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -252,7 +251,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -317,7 +316,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -384,7 +383,7 @@ def test_save_channel_config_telegram(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/telegram/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): ChatDataProcessor.save_channel_config({"connector_type": "telegram", "config": { "access_token": access_token, @@ -405,7 +404,7 @@ def test_save_channel_config_telegram_invalid(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/telegram/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): ChatDataProcessor.save_channel_config({"connector_type": "telegram", "config": { "access_token": access_token, @@ -478,7 +477,7 @@ def test_save_channel_config_business_messages_with_invalid_private_key(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/business_messages/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): channel_endpoint = ChatDataProcessor.save_channel_config( { "connector_type": "business_messages", @@ -506,7 +505,7 @@ def test_save_channel_config_business_messages(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/business_messages/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): channel_endpoint = ChatDataProcessor.save_channel_config( { "connector_type": "business_messages", @@ -594,7 +593,7 @@ def test_get_channel_end_point_whatsapp(self, monkeypatch): def _mock_generate_integration_token(*arge, **kwargs): return "testtoken", "ignore" - with patch.object(Authentication, "generate_integration_token", _mock_generate_integration_token): + with mock.patch.object(Authentication, "generate_integration_token", _mock_generate_integration_token): channel_url = ChatDataProcessor.save_channel_config({ "connector_type": "whatsapp", "config": { "app_secret": "app123", diff --git a/tests/unit_test/cli_test.py b/tests/unit_test/cli_test.py index 80f1e6bf2..1f28f1fd2 100644 --- a/tests/unit_test/cli_test.py +++ b/tests/unit_test/cli_test.py @@ -1,8 +1,10 @@ +import argparse +import os from datetime import datetime -from unittest.mock import patch +from unittest import mock import pytest -import os +from mongoengine import connect from kairon import cli from kairon.cli.conversations_deletion import initiate_history_deletion_archival @@ -10,22 +12,17 @@ from kairon.cli.delete_logs import delete_logs from kairon.cli.importer import validate_and_import from kairon.cli.message_broadcast import send_notifications -from kairon.cli.training import train from kairon.cli.testing import run_tests_on_model +from kairon.cli.training import train from kairon.cli.translator import translate_multilingual_bot from kairon.events.definitions.data_generator import DataGenerationEvent from kairon.events.definitions.data_importer import TrainingDataImporterEvent from kairon.events.definitions.history_delete import DeleteHistoryEvent -from kairon.events.definitions.message_broadcast import MessageBroadcastEvent from kairon.events.definitions.model_testing import ModelTestingEvent from kairon.events.definitions.multilingual import MultilingualEvent from kairon.shared.concurrency.actors.factory import ActorFactory -from kairon.shared.utils import Utility -from mongoengine import connect -import mock -import argparse - from kairon.shared.constants import EventClass +from kairon.shared.utils import Utility class TestTrainingCli: @@ -395,7 +392,7 @@ def test_message_broadcast_no_event_id(self, monkeypatch): return_value=argparse.Namespace(func=send_notifications, bot="test_cli", user="testUser", event_id="65432123456789876543")) def test_message_broadcast_all_arguments(self, mock_namespace): - with patch('kairon.events.definitions.message_broadcast.MessageBroadcastEvent.execute', autospec=True): + with mock.patch('kairon.events.definitions.message_broadcast.MessageBroadcastEvent.execute', autospec=True): cli() for proxy in ActorFactory._ActorFactory__actors.values(): diff --git a/tests/unit_test/data_processor/agent_processor_test.py b/tests/unit_test/data_processor/agent_processor_test.py index ec62c2dec..8d37b1541 100644 --- a/tests/unit_test/data_processor/agent_processor_test.py +++ b/tests/unit_test/data_processor/agent_processor_test.py @@ -16,7 +16,7 @@ from kairon.shared.data.constant import EVENT_STATUS from kairon.shared.data.model_processor import ModelProcessor -from mock import patch +from unittest.mock import patch from mongoengine import connect diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 08320bc27..c79165738 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -14,7 +14,7 @@ Utility.load_system_metadata() -from mock import patch +from unittest.mock import patch import numpy as np import pandas as pd import pytest diff --git a/tests/unit_test/data_processor/history_test.py b/tests/unit_test/data_processor/history_test.py index 3048b9cc3..44dcda270 100644 --- a/tests/unit_test/data_processor/history_test.py +++ b/tests/unit_test/data_processor/history_test.py @@ -2,7 +2,7 @@ import os from datetime import datetime -import mock +from unittest import mock import mongomock import pytest diff --git a/tests/unit_test/events/definitions_test.py b/tests/unit_test/events/definitions_test.py index aff9455f7..988b14072 100644 --- a/tests/unit_test/events/definitions_test.py +++ b/tests/unit_test/events/definitions_test.py @@ -4,11 +4,11 @@ from io import BytesIO from urllib.parse import urljoin -import mock +from unittest import mock import pytest import responses from fastapi import UploadFile -from mock.mock import patch +from unittest.mock import patch from mongoengine import connect from augmentation.utils import WebsiteParser diff --git a/tests/unit_test/events/events_test.py b/tests/unit_test/events/events_test.py index 4f7a63438..7f1448a9d 100644 --- a/tests/unit_test/events/events_test.py +++ b/tests/unit_test/events/events_test.py @@ -8,7 +8,7 @@ from unittest.mock import patch from urllib.parse import urljoin -import mock +from unittest import mock import mongomock import pytest import responses diff --git a/tests/unit_test/events/scheduler_test.py b/tests/unit_test/events/scheduler_test.py index ec6426415..678c11d2e 100644 --- a/tests/unit_test/events/scheduler_test.py +++ b/tests/unit_test/events/scheduler_test.py @@ -1,7 +1,7 @@ import os import re -from mock import patch +from unittest.mock import patch import pytest from apscheduler.jobstores.mongodb import MongoDBJobStore diff --git a/tests/unit_test/idp/test_idp_helper.py b/tests/unit_test/idp/test_idp_helper.py index db6ff9609..0d74cc842 100644 --- a/tests/unit_test/idp/test_idp_helper.py +++ b/tests/unit_test/idp/test_idp_helper.py @@ -17,7 +17,6 @@ from kairon.shared.organization.processor import OrgProcessor from kairon.shared.utils import Utility from stress_test.data_objects import User -from mock import patch def get_user(): diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index bcc2ed661..16daaec64 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1,13 +1,13 @@ import os -import time +from unittest import mock from urllib.parse import urljoin -import mock import numpy as np import pytest import ujson as json from aiohttp import ClientConnectionError from mongoengine import connect + from kairon.shared.utils import Utility Utility.load_system_metadata() @@ -18,7 +18,6 @@ from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT from kairon.shared.llm.processor import LLMProcessor -from kairon.shared.llm.data_objects import LLMLogs import litellm from deepdiff import DeepDiff @@ -1233,4 +1232,4 @@ async def test_llm_logging(self): stream=True, metadata={'user': user, 'bot': bot}) - assert str(e) == "Authentication error" + assert str(e) == "Authentication error" \ No newline at end of file diff --git a/tests/unit_test/plugins_test.py b/tests/unit_test/plugins_test.py index cc24f8f04..bb9de7443 100644 --- a/tests/unit_test/plugins_test.py +++ b/tests/unit_test/plugins_test.py @@ -1,7 +1,7 @@ import os import re -import mock +from unittest import mock import pytest import requests import responses @@ -11,7 +11,6 @@ from kairon.shared.constants import PluginTypes from kairon.shared.plugins.factory import PluginFactory from kairon.shared.utils import Utility -from mongomock import MongoClient class TestUtility: diff --git a/tests/unit_test/rest_client_test.py b/tests/unit_test/rest_client_test.py index 0d6afa160..6daa1e5e4 100644 --- a/tests/unit_test/rest_client_test.py +++ b/tests/unit_test/rest_client_test.py @@ -1,4 +1,5 @@ import asyncio +import ujson as json from unittest import mock import pytest @@ -101,3 +102,19 @@ async def test_aio_rest_client_timeout_error(self, aioresponses): with pytest.raises(AppException, match="Request timed out: Request timed out"): await AioRestClient().request("get", url, request_body={"name": "udit.pandey", "loc": "blr"}, headers={"Authorization": "Bearer sasdfghjkytrtyui"}, max_retries=3) + + @pytest.mark.asyncio + async def test_aio_rest_client_post_request_stream(self, aioresponses): + url = 'http://kairon.com' + aioresponses.post("http://kairon.com", status=200, body=json.dumps({'data': 'hi!'})) + resp = await AioRestClient().request("post", url, request_body={"name": "udit.pandey", "loc": "blr"}, + headers={"Authorization": "Bearer sasdfghjkytrtyui"}, is_streaming_resp=True) + response = '' + async for content in resp.content: + response += content.decode() + + assert json.loads(response) == {"data": "hi!"} + assert list(aioresponses.requests.values())[0][0].kwargs == {'allow_redirects': True, 'headers': { + 'Authorization': 'Bearer sasdfghjkytrtyui'}, 'json': {'loc': 'blr', 'name': 'udit.pandey'}, 'timeout': None, + 'data': None, + 'trace_request_ctx': {'current_attempt': 1}} \ No newline at end of file diff --git a/tests/unit_test/verification_test.py b/tests/unit_test/verification_test.py index 4e6c2b360..6462ae579 100644 --- a/tests/unit_test/verification_test.py +++ b/tests/unit_test/verification_test.py @@ -2,7 +2,7 @@ import responses from kairon.shared.verification.email import QuickEmailVerification from urllib.parse import urlencode -import mock +from unittest import mock from kairon.shared.utils import Utility import os From aba13c8053a84a7416e5c5b900cd63fc089ae31f Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 13:38:29 +0530 Subject: [PATCH 08/57] test cases fixed --- kairon/shared/llm/logger.py | 3 ++- tests/unit_test/llm_test.py | 46 +++++++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py index dbb93e469..3af7a7870 100644 --- a/kairon/shared/llm/logger.py +++ b/kairon/shared/llm/logger.py @@ -37,4 +37,5 @@ def __logs_litellm(self, **kwargs): 'metadata': litellm_params.get('metadata')}) def __save_logs(self, **kwargs): - print(LLMLogs(**kwargs).save().to_mongo().to_dict()) + logs = LLMLogs(**kwargs).save().to_mongo().to_dict() + logger.info(f"llm logs: {logs}") diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 16daaec64..1042be91d 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1179,38 +1179,50 @@ async def test_llm_logging(self): user = "test" litellm.callbacks = [LiteLLMLogger()] - result = await litellm.acompletion(messages=["Hi"], + messages = [{"role":"user", "content":"Hi"}] + expected = "Hi, How may i help you?" + + result = await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, metadata={'user': user, 'bot': bot}) - assert result + assert result['choices'][0]['message']['content'] == expected - result = litellm.completion(messages=["Hi"], + result = litellm.completion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, metadata={'user': user, 'bot': bot}) - assert result + assert result['choices'][0]['message']['content'] == expected - result = litellm.completion(messages=["Hi"], + result = litellm.completion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, stream=True, metadata={'user': user, 'bot': bot}) + response = '' for chunk in result: - print(chunk["choices"][0]["delta"]["content"]) - assert chunk["choices"][0]["delta"]["content"] + content = chunk["choices"][0]["delta"]["content"] + if content: + response = response + content + + assert response == expected - result = await litellm.acompletion(messages=["Hi"], + result = await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, stream=True, metadata={'user': user, 'bot': bot}) + response = '' async for chunk in result: - print(chunk["choices"][0]["delta"]["content"]) - assert chunk["choices"][0]["delta"]["content"] + content = chunk["choices"][0]["delta"]["content"] + print(chunk) + if content: + response += content + + assert response.__contains__(expected) with pytest.raises(Exception) as e: - await litellm.acompletion(messages=["Hi"], + await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), metadata={'user': user, 'bot': bot}) @@ -1218,7 +1230,7 @@ async def test_llm_logging(self): assert str(e) == "Authentication error" with pytest.raises(Exception) as e: - litellm.completion(messages=["Hi"], + litellm.completion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), metadata={'user': user, 'bot': bot}) @@ -1226,7 +1238,7 @@ async def test_llm_logging(self): assert str(e) == "Authentication error" with pytest.raises(Exception) as e: - await litellm.acompletion(messages=["Hi"], + await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), stream=True, From 2dfddd6ecb4bde4e04eb7638d39693d1e0863857 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 14:53:59 +0530 Subject: [PATCH 09/57] removed unused variable --- kairon/actions/definitions/prompt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 4c7bf6bc4..6323589b9 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -66,7 +66,6 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma k_faq_action_config, bot_settings = self.retrieve_config() user_question = k_faq_action_config.get('user_question') user_msg = self.__get_user_msg(tracker, user_question) - llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) llm_processor = LLMProcessor(self.bot) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, From ff7588cf5c8d2c14571a4d8250ffd18ce1d36c49 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 15:15:10 +0530 Subject: [PATCH 10/57] fixed unused variable --- kairon/actions/definitions/prompt.py | 3 +- kairon/shared/llm/processor.py | 3 +- kairon/shared/vector_embeddings/db/qdrant.py | 6 ++-- kairon/train.py | 4 +-- tests/unit_test/llm_test.py | 36 ++++++++++---------- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 6323589b9..6441d607f 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -66,8 +66,9 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma k_faq_action_config, bot_settings = self.retrieve_config() user_question = k_faq_action_config.get('user_question') user_msg = self.__get_user_msg(tracker, user_question) + llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) - llm_processor = LLMProcessor(self.bot) + llm_processor = LLMProcessor(self.bot, llm_type) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, user=tracker.sender_id, **llm_params) diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index c6e7fa8af..adbb039ae 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -26,13 +26,14 @@ class LLMProcessor(LLMBase): __embedding__ = 1536 - def __init__(self, bot: Text): + def __init__(self, bot: Text, llm_type: str): super().__init__(bot) self.db_url = Utility.environment['vector']['db'] self.headers = {} if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.suffix = "_faq_embd" + self.llm_type = llm_type self.vector_config = {'size': self.__embedding__, 'distance': 'Cosine'} self.api_key = Sysadmin.get_bot_secret(bot, BotSecretType.gpt_key.value, raise_err=True) self.tokenizer = get_encoding("cl100k_base") diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index 12b5268e2..454eeb8fe 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -5,10 +5,8 @@ from kairon import Utility from kairon.shared.actions.utils import ActionUtility -from kairon.shared.admin.constants import BotSecretType -from kairon.shared.admin.processor import Sysadmin -from kairon.shared.constants import GPT3ResourceTypes from kairon.shared.llm.processor import LLMProcessor +from kairon.shared.data.constant import DEFAULT_LLM from kairon.shared.vector_embeddings.db.base import VectorEmbeddingsDbBase @@ -25,7 +23,7 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.llm_settings = llm_settings - self.llm = LLMProcessor(self.bot) + self.llm = LLMProcessor(self.bot, DEFAULT_LLM) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 diff --git a/kairon/train.py b/kairon/train.py index 3ddcf9eb0..d8360ce67 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -6,7 +6,7 @@ from rasa.api import train from rasa.model import DEFAULT_MODELS_PATH from rasa.shared.constants import DEFAULT_CONFIG_PATH, DEFAULT_DATA_PATH, DEFAULT_DOMAIN_PATH - +from kairon.shared.data.constant import DEFAULT_LLM from kairon.chat.agent.agent import KaironAgent from kairon.exceptions import AppException from kairon.shared.account.processor import AccountProcessor @@ -101,7 +101,7 @@ def start_training(bot: str, user: str, token: str = None): settings = processor.get_bot_settings(bot, user) settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: - llm_processor = LLMProcessor(bot) + llm_processor = LLMProcessor(bot, DEFAULT_LLM) faqs = asyncio.run(llm_processor.train(user=user)) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 1042be91d..a385106cd 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -16,7 +16,7 @@ from kairon.shared.admin.constants import BotSecretType from kairon.shared.admin.data_objects import BotSecrets from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema -from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT +from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_LLM from kairon.shared.llm.processor import LLMProcessor import litellm from deepdiff import DeepDiff @@ -44,7 +44,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM)[[]] aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -123,7 +123,7 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore embedding = list(np.random.random(LLMProcessor.__embedding__)) mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, { 'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -227,7 +227,7 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a input = {"name": "Ram", "color": "red"} mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), @@ -288,7 +288,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): input = {"name": "Ram", "color": "red"} mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -339,7 +339,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): def test_gpt3_faq_embedding_train_failure(self): with pytest.raises(AppException, match=f"Bot secret '{BotSecretType.gpt_key.value}' not configured!"): - LLMProcessor('test_failure') + LLMProcessor('test_failure', DEFAULT_LLM) @pytest.mark.asyncio @mock.patch.object(litellm, "aembedding", autospec=True) @@ -357,7 +357,7 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aiore mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -421,7 +421,7 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -507,7 +507,7 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -574,7 +574,7 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_emb mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -641,7 +641,7 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'vector': {"key": "test", 'db': "http://localhost:6333"}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -729,7 +729,7 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe ] with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -832,7 +832,7 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content1.collection}{gpt3.suffix}/points/search"), @@ -908,7 +908,7 @@ def __mock_connection_error(*args, **kwargs): mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.side_effect = __mock_connection_error - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -971,7 +971,7 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_llm_request.side_effect = ClientConnectionError() - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} @@ -1009,7 +1009,7 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ } mock_embedding.side_effect = [Exception("Connection reset by peer!"), {'data': [{'embedding': embedding}]}] - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) @@ -1064,7 +1064,7 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -1143,7 +1143,7 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, { 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], From 3ec831549eaafaed6bab85d14a2d9aad9d55bada Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 16:28:11 +0530 Subject: [PATCH 11/57] fixed unused variable --- tests/unit_test/llm_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index a385106cd..53c31bace 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -44,7 +44,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM)[[]] + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), From 6af2708b70b55e8e749d13c8d2eaa26850dbeca6 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 28 Jun 2024 12:56:08 +0530 Subject: [PATCH 12/57] added test cased for fetching logs --- kairon/api/app/routers/bot/bot.py | 23 +++++++++++-- kairon/shared/llm/processor.py | 24 +++++++++++++ tests/integration_test/services_test.py | 45 ++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/kairon/api/app/routers/bot/bot.py b/kairon/api/app/routers/bot/bot.py index 6dfeda5c5..276404377 100644 --- a/kairon/api/app/routers/bot/bot.py +++ b/kairon/api/app/routers/bot/bot.py @@ -26,7 +26,7 @@ from kairon.shared.actions.data_objects import ActionServerLogs from kairon.shared.auth import Authentication from kairon.shared.constants import TESTER_ACCESS, DESIGNER_ACCESS, CHAT_ACCESS, UserActivityType, ADMIN_ACCESS, \ - VIEW_ACCESS, EventClass, AGENT_ACCESS + EventClass, AGENT_ACCESS from kairon.shared.data.assets_processor import AssetsProcessor from kairon.shared.data.audit.processor import AuditDataProcessor from kairon.shared.data.constant import EVENT_STATUS, ENDPOINT_TYPE, TOKEN_TYPE, ModelTestType, \ @@ -38,10 +38,12 @@ from kairon.shared.data.utils import DataUtility from kairon.shared.importer.data_objects import ValidationLogs from kairon.shared.importer.processor import DataImporterLogProcessor +from kairon.shared.live_agent.live_agent import LiveAgentHandler from kairon.shared.models import User, TemplateType from kairon.shared.test.processor import ModelTestingLogProcessor from kairon.shared.utils import Utility -from kairon.shared.live_agent.live_agent import LiveAgentHandler +from kairon.shared.llm.processor import LLMProcessor +from kairon.shared.llm.data_objects import LLMLogs router = APIRouter() @@ -1668,3 +1670,20 @@ async def get_live_agent_token(current_user: User = Security(Authentication.get_ data = await LiveAgentHandler.authenticate_agent(current_user.get_user(), current_user.get_bot()) return Response(data=data) + +@router.get("/llm/logs", response_model=Response) +async def get_llm_logs( + start_idx: int = 0, page_size: int = 10, + current_user: User = Security(Authentication.get_current_user_and_bot, scopes=TESTER_ACCESS) +): + """ + Get data llm event logs. + """ + logs = list(LLMProcessor.get_logs(current_user.get_bot(), start_idx, page_size)) + row_cnt = LLMProcessor.get_row_count(current_user.get_bot()) + data = { + "logs": logs, + "total": row_cnt + } + return Response(data=data) + diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index adbb039ae..7365e0aa7 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -16,6 +16,7 @@ from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_CONTEXT_PROMPT from kairon.shared.llm.base import LLMBase from kairon.shared.llm.logger import LiteLLMLogger +from kairon.shared.llm.data_objects import LLMLogs from kairon.shared.models import CognitionDataType from kairon.shared.rest_client import AioRestClient from kairon.shared.utils import Utility @@ -260,3 +261,26 @@ async def __attach_similarity_prompt_if_enabled(self, query_embedding, context_p similarity_context = f"Instructions on how to use {similarity_prompt_name}:\n{extracted_values}\n{similarity_prompt_instructions}\n" context_prompt = f"{context_prompt}\n{similarity_context}" return context_prompt + + @staticmethod + def get_logs(bot: str, start_idx: int = 0, page_size: int = 10): + """ + Get all logs for data importer event. + @param bot: bot id. + @param start_idx: start index + @param page_size: page size + @return: list of logs. + """ + for log in LLMLogs.objects(metadata__bot=bot).order_by("-start_time").skip(start_idx).limit(page_size): + llm_log = log.to_mongo().to_dict() + llm_log.pop('_id') + yield llm_log + + @staticmethod + def get_row_count(bot: str): + """ + Gets the count of rows in a LLMLogs for a particular bot. + :param bot: bot id + :return: Count of rows + """ + return LLMLogs.objects(metadata__bot=bot).count() diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 5efd5c6ca..62fb261b3 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -1,3 +1,5 @@ +import time + import ujson as json import os import re @@ -23352,6 +23354,47 @@ def test_trigger_widget(): assert actual["error_code"] == 0 assert len(actual["data"]) == 2 assert not actual["message"] + + +def test_get_llm_logs(): + from kairon.shared.llm.logger import LiteLLMLogger + import litellm + import asyncio + + loop = asyncio.new_event_loop() + user = "test" + litellm.callbacks = [LiteLLMLogger()] + + messages = [{"role": "user", "content": "Hi"}] + expected = "Hi, How may i help you?" + + result = loop.run_until_complete(litellm.acompletion(messages=messages, + model="gpt-3.5-turbo", + mock_response=expected, + metadata={'user': user, 'bot': pytest.bot})) + assert result['choices'][0]['message']['content'] == expected + + time.sleep(2) + + response = client.get( + f"/api/bot/{pytest.bot}/llm/logs?start_idx=0&page_size=10", + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + print(actual) + assert actual["success"] + assert actual["error_code"] == 0 + assert len(actual["data"]["logs"]) == 1 + assert actual["data"]["total"] == 1 + assert actual["data"]["logs"][0]['start_time'] + assert actual["data"]["logs"][0]['end_time'] + assert actual["data"]["logs"][0]['cost'] + assert actual["data"]["logs"][0]['llm_call_id'] + assert actual["data"]["logs"][0]["llm_provider"] == "openai" + assert not actual["data"]["logs"][0].get("model") + assert actual["data"]["logs"][0]["model_params"] == {} + assert actual["data"]["logs"][0]["metadata"]['bot'] == pytest.bot + assert actual["data"]["logs"][0]["metadata"]['user'] == "test" def test_add_custom_widget_invalid_config(): @@ -24251,4 +24294,4 @@ def test_list_system_metadata(): actual = response.json() assert actual["error_code"] == 0 assert actual["success"] - assert len(actual["data"]) == 17 + assert len(actual["data"]) == 17 \ No newline at end of file From 6da18bb44a70dc6fb44d19001241c141303ef13b Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 21 Jun 2024 20:21:05 +0530 Subject: [PATCH 13/57] litellm base version --- augmentation/paraphrase/gpt3/gpt.py | 7 +- custom/__init__.py | 0 custom/fallback.py | 58 - custom/ner.py | 169 --- kairon/actions/definitions/database.py | 2 +- kairon/actions/definitions/prompt.py | 31 +- kairon/api/models.py | 24 +- kairon/chat/agent/message_processor.py | 9 - kairon/importer/validator/file_validator.py | 31 +- kairon/shared/actions/data_objects.py | 8 +- .../concurrency/actors/pyscript_runner.py | 10 +- kairon/shared/data/constant.py | 1 + kairon/shared/data/processor.py | 4 +- kairon/shared/llm/base.py | 4 +- kairon/shared/llm/clients/__init__.py | 0 kairon/shared/llm/clients/azure.py | 25 - kairon/shared/llm/clients/base.py | 8 - kairon/shared/llm/clients/factory.py | 18 - kairon/shared/llm/clients/gpt3.py | 92 -- kairon/shared/llm/data_objects.py | 13 + kairon/shared/llm/factory.py | 17 - kairon/shared/llm/logger.py | 43 + kairon/shared/llm/{gpt3.py => processor.py} | 106 +- kairon/shared/utils.py | 85 +- kairon/shared/vector_embeddings/db/base.py | 8 +- kairon/shared/vector_embeddings/db/qdrant.py | 18 +- kairon/train.py | 7 +- metadata/integrations.yml | 127 +- requirements/dev.txt | 13 +- requirements/prod.txt | 68 +- tests/integration_test/action_service_test.py | 1066 ++++++++++------- tests/integration_test/chat_service_test.py | 10 +- tests/integration_test/services_test.py | 24 +- tests/unit_test/action/action_test.py | 6 +- tests/unit_test/api/api_processor_test.py | 9 +- .../augmentation/gpt_augmentation_test.py | 10 +- .../data_processor/data_processor_test.py | 111 +- tests/unit_test/events/events_test.py | 11 +- tests/unit_test/llm_test.py | 791 ++++++------ tests/unit_test/utility_test.py | 745 +----------- .../validator/training_data_validator_test.py | 73 +- training_data/ReadMe.md | 1 - 42 files changed, 1481 insertions(+), 2382 deletions(-) delete mode 100644 custom/__init__.py delete mode 100644 custom/fallback.py delete mode 100644 custom/ner.py delete mode 100644 kairon/shared/llm/clients/__init__.py delete mode 100644 kairon/shared/llm/clients/azure.py delete mode 100644 kairon/shared/llm/clients/base.py delete mode 100644 kairon/shared/llm/clients/factory.py delete mode 100644 kairon/shared/llm/clients/gpt3.py create mode 100644 kairon/shared/llm/data_objects.py delete mode 100644 kairon/shared/llm/factory.py create mode 100644 kairon/shared/llm/logger.py rename kairon/shared/llm/{gpt3.py => processor.py} (73%) delete mode 100644 training_data/ReadMe.md diff --git a/augmentation/paraphrase/gpt3/gpt.py b/augmentation/paraphrase/gpt3/gpt.py index 340c220c0..b11a8eac5 100644 --- a/augmentation/paraphrase/gpt3/gpt.py +++ b/augmentation/paraphrase/gpt3/gpt.py @@ -1,7 +1,7 @@ """Creates the Example and GPT classes for a user to interface with the OpenAI API.""" -import openai +from openai import OpenAI import uuid @@ -95,8 +95,9 @@ def submit_request(self, prompt, num_responses, api_key): """Calls the OpenAI API with the specified parameters.""" if num_responses < 1: num_responses = 1 - response = openai.Completion.create(api_key=api_key, - engine=self.get_engine(), + client = OpenAI(api_key=api_key) + response = client.completions.create( + model=self.get_engine(), prompt=self.craft_query(prompt), max_tokens=self.get_max_tokens(), temperature=self.get_temperature(), diff --git a/custom/__init__.py b/custom/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/custom/fallback.py b/custom/fallback.py deleted file mode 100644 index 4c396c039..000000000 --- a/custom/fallback.py +++ /dev/null @@ -1,58 +0,0 @@ -''' -Custom component to get fallback action intent -Reference: https://forum.rasa.com/t/fallback-intents-for-context-sensitive-fallbacks/963 -''' - -from rasa.nlu.classifiers.classifier import IntentClassifier - -class FallbackIntentFilter(IntentClassifier): - - # Name of the component to be used when integrating it in a - # pipeline. E.g. ``[ComponentA, ComponentB]`` - # will be a proper pipeline definition where ``ComponentA`` - # is the name of the first component of the pipeline. - name = "FallbackIntentFilter" - - # Defines what attributes the pipeline component will - # provide when called. The listed attributes - # should be set by the component on the message object - # during test and train, e.g. - # ```message.set("entities", [...])``` - provides = [] - - # Which attributes on a message are required by this - # component. e.g. if requires contains "tokens", than a - # previous component in the pipeline needs to have "tokens" - # within the above described `provides` property. - requires = [] - - # Defines the default configuration parameters of a component - # these values can be overwritten in the pipeline configuration - # of the model. The component should choose sensible defaults - # and should be able to create reasonable results with the defaults. - defaults = {} - - # Defines what language(s) this component can handle. - # This attribute is designed for instance method: `can_handle_language`. - # Default value is None which means it can handle all languages. - # This is an important feature for backwards compatibility of components. - language_list = None - - def __init__(self, component_config=None, low_threshold=0.3, high_threshold=0.4, fallback_intent="fallback", - out_of_scope_intent="out_of_scope"): - super().__init__(component_config) - self.fb_low_threshold = low_threshold - self.fb_high_threshold = high_threshold - self.fallback_intent = fallback_intent - self.out_of_scope_intent = out_of_scope_intent - - def process(self, message, **kwargs): - message_confidence = message.data['intent']['confidence'] - new_intent = None - if message_confidence <= self.fb_low_threshold: - new_intent = {'name': self.out_of_scope_intent, 'confidence': message_confidence} - elif message_confidence <= self.fb_high_threshold: - new_intent = {'name': self.fallback_intent, 'confidence': message_confidence} - if new_intent is not None: - message.data['intent'] = new_intent - message.data['intent_ranking'].insert(0, new_intent) diff --git a/custom/ner.py b/custom/ner.py deleted file mode 100644 index 1a38897fe..000000000 --- a/custom/ner.py +++ /dev/null @@ -1,169 +0,0 @@ -from rasa.nlu.components import Component -from typing import Any, Optional, Text, Dict, TYPE_CHECKING -import os -import spacy -import pickle -from spacy.matcher import Matcher -from rasa.nlu.extractors.extractor import EntityExtractor - - -if TYPE_CHECKING: - from rasa.nlu.model import Metadata - -PATTERN_NER_FILE = 'pattern_ner.pkl' -class SpacyPatternNER(EntityExtractor): - """A new component""" - name = "pattern_ner_spacy" - # Defines what attributes the pipeline component will - # provide when called. The listed attributes - # should be set by the component on the message object - # during test and train, e.g. - # ```message.set("entities", [...])``` - provides = ["entities"] - - # Which attributes on a message are required by this - # component. e.g. if requires contains "tokens", than a - # previous component in the pipeline needs to have "tokens" - # within the above described `provides` property. - requires = ["tokens"] - - # Defines the default configuration parameters of a component - # these values can be overwritten in the pipeline configuration - # of the model. The component should choose sensible defaults - # and should be able to create reasonable results with the defaults. - defaults = {} - - # Defines what language(s) this component can handle. - # This attribute is designed for instance method: `can_handle_language`. - # Default value is None which means it can handle all languages. - # This is an important feature for backwards compatibility of components. - language_list = None - - def __init__(self, component_config=None, matcher=None): - super(SpacyPatternNER, self).__init__(component_config) - if matcher: - self.matcher = matcher - self.spacy_nlp = spacy.blank('en') - self.spacy_nlp.vocab = self.matcher.vocab - else: - self.spacy_nlp = spacy.blank('en') - self.matcher = Matcher(self.spacy_nlp.vocab) - - def train(self, training_data, cfg, **kwargs): - """Train this component. - - This is the components chance to train itself provided - with the training data. The component can rely on - any context attribute to be present, that gets created - by a call to :meth:`components.Component.pipeline_init` - of ANY component and - on any context attributes created by a call to - :meth:`components.Component.train` - of components previous to this one.""" - for lookup_table in training_data.lookup_tables: - key = lookup_table['name'] - pattern = [] - for element in lookup_table['elements']: - tokens = [{'LOWER': token.lower()} for token in str(element).split()] - pattern.append(tokens) - self.matcher.add(key, pattern) - - def process(self, message, **kwargs): - """Process an incoming message. - - This is the components chance to process an incoming - message. The component can rely on - any context attribute to be present, that gets created - by a call to :meth:`components.Component.pipeline_init` - of ANY component and - on any context attributes created by a call to - :meth:`components.Component.process` - of components previous to this one.""" - entities = [] - - # with plural forms - doc = self.spacy_nlp(message.data['text'].lower()) - matches = self.matcher(doc) - entities = self.getNewEntityObj(doc, matches, entities) - - # Without plural forms - doc = self.spacy_nlp(' '.join([token.lemma_ for token in doc])) - matches = self.matcher(doc) - entities = self.getNewEntityObj(doc, matches, entities) - - # Remove duplicates - seen = set() - new_entities = [] - - for entityObj in entities: - record = tuple(entityObj.items()) - if record not in seen: - seen.add(record) - new_entities.append(entityObj) - - message.set("entities", message.get("entities", []) + new_entities, add_to_output=True) - - - def getNewEntityObj(self, doc, matches, entities): - - for ent_id, start, end in matches: - new_entity_value = doc[start:end].text - new_entity_value_len = len(new_entity_value.split()) - is_add = True - - for old_entity in entities: - old_entity_value = old_entity["value"] - old_entity_value_len = len(old_entity_value.split()) - - if old_entity_value_len > new_entity_value_len and new_entity_value in old_entity_value: - is_add = False - elif old_entity_value_len < new_entity_value_len and old_entity_value in new_entity_value: - entities.remove(old_entity) - - if is_add: - entities.append({ - 'start': start, - 'end': end, - 'value': doc[start:end].text, - 'entity': self.matcher.vocab.strings[ent_id], - 'confidence': None, - 'extractor': self.name - }) - - return entities - - - def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: - """Persist this component to disk for future loading.""" - if self.matcher: - modelFile = os.path.join(model_dir, PATTERN_NER_FILE) - self.saveModel(modelFile) - return {"pattern_ner_file": PATTERN_NER_FILE} - - - @classmethod - def load( - cls, - meta: Dict[Text, Any], - model_dir: Optional[Text] = None, - model_metadata: Optional["Metadata"] = None, - cached_component: Optional["Component"] = None, - **kwargs: Any - ) -> "Component": - """Load this component from file.""" - - file_name = meta.get("pattern_ner_file", PATTERN_NER_FILE) - modelFile = os.path.join(model_dir, file_name) - if os.path.exists(modelFile): - modelLoad = open(modelFile, "rb") - matcher = pickle.load(modelLoad) - modelLoad.close() - return cls(meta, matcher) - else: - return cls(meta) - - - def saveModel(self, modelFile): - modelSave = open(modelFile, "wb") - pickle.dump(self.matcher, modelSave) - modelSave.close() \ No newline at end of file diff --git a/kairon/actions/definitions/database.py b/kairon/actions/definitions/database.py index 6f0e48271..ebdf83510 100644 --- a/kairon/actions/definitions/database.py +++ b/kairon/actions/definitions/database.py @@ -83,7 +83,7 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma request_body = ActionUtility.get_payload(payload, tracker) msg_logger.append(request_body) tracker_data = ActionUtility.build_context(tracker, True) - response = await vector_db.perform_operation(operation_type, request_body) + response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id, bot=self.bot) logger.info("response: " + str(response)) response_context = self.__add_user_context_to_http_response(response, tracker_data) bot_response, bot_resp_log, _ = ActionUtility.compose_response(vector_action_config['response'], response_context) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 3de92f413..381e6f543 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -4,7 +4,6 @@ from rasa_sdk import Tracker from rasa_sdk.executor import CollectingDispatcher -from kairon import Utility from kairon.actions.definitions.base import ActionsBase from kairon.shared.actions.data_objects import ActionServerLogs from kairon.shared.actions.exception import ActionFailure @@ -12,8 +11,8 @@ from kairon.shared.actions.utils import ActionUtility from kairon.shared.constants import FAQ_DISABLED_ERR, KaironSystemSlots, KAIRON_USER_MSG_ENTITY from kairon.shared.data.constant import DEFAULT_NLU_FALLBACK_RESPONSE -from kairon.shared.llm.factory import LLMFactory from kairon.shared.models import LlmPromptType, LlmPromptSource +from kairon.shared.llm.processor import LLMProcessor class ActionPrompt(ActionsBase): @@ -62,14 +61,18 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma time_taken_slots = 0 final_slots = {"type": "slots_to_fill"} llm_response_log = {"type": "llm_response"} - + llm_processor = None try: k_faq_action_config, bot_settings = self.retrieve_config() user_question = k_faq_action_config.get('user_question') user_msg = self.__get_user_msg(tracker, user_question) + llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) - llm = LLMFactory.get_instance("faq")(self.bot, bot_settings["llm_settings"]) - llm_response, time_taken_llm_response = await llm.predict(user_msg, **llm_params) + llm_processor = LLMProcessor(self.bot) + llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, + user=tracker.sender_id, + bot=self.bot, + **llm_params) status = "FAILURE" if llm_response.get("is_failure", False) is True else status exception = llm_response.get("exception") bot_response = llm_response['content'] @@ -93,8 +96,8 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma total_time_elapsed = time_taken_llm_response + time_taken_slots events_to_extend = [llm_response_log, final_slots] events.extend(events_to_extend) - if llm: - llm_logs = llm.logs + if llm_processor: + llm_logs = llm_processor.logs ActionServerLogs( type=ActionType.prompt_action.value, intent=tracker.get_intent_of_latest_message(skip_fallback_intent=False), @@ -119,16 +122,6 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma return slots_to_fill async def __get_llm_params(self, k_faq_action_config: dict, dispatcher: CollectingDispatcher, tracker: Tracker, domain: Dict[Text, Any]): - implementations = { - "GPT3_FAQ_EMBED": self.__get_gpt_params, - } - - llm_type = Utility.environment['llm']["faq"] - if not implementations.get(llm_type): - raise ActionFailure(f'{llm_type} type LLM is not supported') - return await implementations[Utility.environment['llm']["faq"]](k_faq_action_config, dispatcher, tracker, domain) - - async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: CollectingDispatcher, tracker: Tracker, domain: Dict[Text, Any]): from kairon.actions.definitions.factory import ActionFactory system_prompt = None @@ -147,7 +140,7 @@ async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: Collecti history_prompt = ActionUtility.prepare_bot_responses(tracker, num_bot_responses) elif prompt['source'] == LlmPromptSource.bot_content.value and prompt['is_enabled']: use_similarity_prompt = True - hyperparameters = prompt.get('hyperparameters', {}) + hyperparameters = prompt.get("hyperparameters", {}) similarity_prompt.append({'similarity_prompt_name': prompt['name'], 'similarity_prompt_instructions': prompt['instructions'], 'collection': prompt['data'], @@ -179,7 +172,7 @@ async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: Collecti is_query_prompt_enabled = True query_prompt_dict.update({'query_prompt': query_prompt, 'use_query_prompt': is_query_prompt_enabled}) - params["hyperparameters"] = k_faq_action_config.get('hyperparameters', Utility.get_llm_hyperparameters()) + params["hyperparameters"] = k_faq_action_config['hyperparameters'] params["system_prompt"] = system_prompt params["context_prompt"] = context_prompt params["query_prompt"] = query_prompt_dict diff --git a/kairon/api/models.py b/kairon/api/models.py index d2ad8b11b..435566f34 100644 --- a/kairon/api/models.py +++ b/kairon/api/models.py @@ -16,6 +16,7 @@ INTEGRATION_STATUS, FALLBACK_MESSAGE, DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_LLM ) from ..shared.actions.models import ( ActionParameterType, @@ -37,6 +38,7 @@ CognitionDataType, CognitionMetadataType, ) +from kairon.shared.utils import Utility class RecaptchaVerifiedRequest(BaseModel): @@ -1035,6 +1037,7 @@ class PromptActionConfigRequest(BaseModel): num_bot_responses: int = 5 failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE user_question: UserQuestionModel = UserQuestionModel() + llm_type: str = DEFAULT_LLM hyperparameters: dict = None llm_prompts: List[LlmPromptRequest] instructions: List[str] = [] @@ -1056,16 +1059,27 @@ def validate_num_bot_responses(cls, v, values, **kwargs): raise ValueError("num_bot_responses should not be greater than 5") return v + @validator("llm_type") + def validate_llm_type(cls, v, values, **kwargs): + if v not in Utility.get_llms(): + raise ValueError("Invalid llm type") + return v + + @validator("hyperparameters") + def validate_llm_hyperparameters(cls, v, values, **kwargs): + Utility.validate_llm_hyperparameters(v, kwargs['llm_type'], ValueError) + @root_validator def check(cls, values): from kairon.shared.utils import Utility - if not values.get("hyperparameters"): - values["hyperparameters"] = {} + if values.get("llm_type"): + if not values.get("hyperparameters"): + values["hyperparameters"] = {} - for key, value in Utility.get_llm_hyperparameters().items(): - if key not in values["hyperparameters"]: - values["hyperparameters"][key] = value + for key, value in Utility.get_llm_hyperparameters(values.get("llm_type")).items(): + if key not in values["hyperparameters"]: + values["hyperparameters"][key] = value return values diff --git a/kairon/chat/agent/message_processor.py b/kairon/chat/agent/message_processor.py index 827404063..e4ae506d2 100644 --- a/kairon/chat/agent/message_processor.py +++ b/kairon/chat/agent/message_processor.py @@ -294,15 +294,6 @@ def predict_next_with_tracker_if_should( Raises: ActionLimitReached if the limit of actions to predict has been reached. """ - should_predict_another_action = self.should_predict_another_action( - tracker.latest_action_name - ) - - if self.is_action_limit_reached(tracker, should_predict_another_action): - raise ActionLimitReached( - "The limit of actions to predict has been reached." - ) - prediction = self._predict_next_with_tracker(tracker) action = self.action_for_index( diff --git a/kairon/importer/validator/file_validator.py b/kairon/importer/validator/file_validator.py index b55b3f0e6..ad2062c01 100644 --- a/kairon/importer/validator/file_validator.py +++ b/kairon/importer/validator/file_validator.py @@ -695,9 +695,9 @@ def __validate_prompt_actions(prompt_actions: list): data_error.append( f'num_bot_responses should not be greater than 5 and of type int: {action.get("name")}') llm_prompts_errors = TrainingDataValidator.__validate_llm_prompts(action['llm_prompts']) - if action.get('hyperparameters') is not None: - llm_hyperparameters_errors = TrainingDataValidator.__validate_llm_prompts_hyperparamters( - action.get('hyperparameters')) + if action.get('hyperparameters'): + llm_hyperparameters_errors = TrainingDataValidator.__validate_llm_prompts_hyperparameters( + action.get('hyperparameters'), action.get("llm_type", "openai")) data_error.extend(llm_hyperparameters_errors) data_error.extend(llm_prompts_errors) if action['name'] in actions_present: @@ -785,27 +785,12 @@ def __validate_llm_prompts(llm_prompts: dict): return error_list @staticmethod - def __validate_llm_prompts_hyperparamters(hyperparameters: dict): + def __validate_llm_prompts_hyperparameters(hyperparameters: dict, llm_type: str): error_list = [] - for key, value in hyperparameters.items(): - if key == 'temperature' and not 0.0 <= value <= 2.0: - error_list.append("Temperature must be between 0.0 and 2.0!") - elif key == 'presence_penalty' and not -2.0 <= value <= 2.0: - error_list.append("presence_penality must be between -2.0 and 2.0!") - elif key == 'frequency_penalty' and not -2.0 <= value <= 2.0: - error_list.append("frequency_penalty must be between -2.0 and 2.0!") - elif key == 'top_p' and not 0.0 <= value <= 1.0: - error_list.append("top_p must be between 0.0 and 1.0!") - elif key == 'n' and not 1 <= value <= 5: - error_list.append("n must be between 1 and 5!") - elif key == 'max_tokens' and not 5 <= value <= 4096: - error_list.append("max_tokens must be between 5 and 4096!") - elif key == 'logit_bias' and not isinstance(value, dict): - error_list.append("logit_bias must be a dictionary!") - elif key == 'stop': - if value and (not isinstance(value, (str, int, list)) or (isinstance(value, list) and len(value) > 4)): - error_list.append( - "Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers.") + try: + Utility.validate_llm_hyperparameters(hyperparameters, llm_type, AppException) + except AppException as e: + error_list.append(e.__str__()) return error_list @staticmethod diff --git a/kairon/shared/actions/data_objects.py b/kairon/shared/actions/data_objects.py index dc86e79e3..d3f64347f 100644 --- a/kairon/shared/actions/data_objects.py +++ b/kairon/shared/actions/data_objects.py @@ -34,6 +34,7 @@ KAIRON_TWO_STAGE_FALLBACK, FALLBACK_MESSAGE, DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_LLM ) from kairon.shared.data.signals import push_notification, auditlogger from kairon.shared.models import LlmPromptType, LlmPromptSource @@ -772,7 +773,8 @@ class PromptAction(Auditlog): bot = StringField(required=True) user = StringField(required=True) timestamp = DateTimeField(default=datetime.utcnow) - hyperparameters = DictField(default=Utility.get_llm_hyperparameters) + llm_type = StringField(default=DEFAULT_LLM, choices=Utility.get_llms()) + hyperparameters = DictField(default=Utility.get_default_llm_hyperparameters) llm_prompts = EmbeddedDocumentListField(LlmPrompt, required=True) instructions = ListField(StringField()) set_slots = EmbeddedDocumentListField(SetSlotsFromResponse) @@ -782,7 +784,7 @@ class PromptAction(Auditlog): meta = {"indexes": [{"fields": ["bot", ("bot", "name", "status")]}]} def clean(self): - for key, value in Utility.get_llm_hyperparameters().items(): + for key, value in Utility.get_llm_hyperparameters(self.llm_type).items(): if key not in self.hyperparameters: self.hyperparameters.update({key: value}) @@ -800,7 +802,7 @@ def validate(self, clean=True): dict_data["llm_prompts"], ValidationError ) Utility.validate_llm_hyperparameters( - dict_data["hyperparameters"], ValidationError + dict_data["hyperparameters"], self.llm_type, ValidationError ) diff --git a/kairon/shared/concurrency/actors/pyscript_runner.py b/kairon/shared/concurrency/actors/pyscript_runner.py index a68e352c9..bd5286739 100644 --- a/kairon/shared/concurrency/actors/pyscript_runner.py +++ b/kairon/shared/concurrency/actors/pyscript_runner.py @@ -1,20 +1,26 @@ from types import ModuleType from typing import Text, Dict, Optional, Callable +import orjson as json from AccessControl.ZopeGuards import _safe_globals from RestrictedPython import compile_restricted from RestrictedPython.Guards import safer_getattr from loguru import logger from timeout_decorator import timeout_decorator -import orjson as json -from ..actors.base import BaseActor from kairon.exceptions import AppException +from ..actors.base import BaseActor +from AccessControl.SecurityInfo import allow_module + +allow_module("datetime") +allow_module("time") + global_safe = _safe_globals global_safe['_getattr_'] = safer_getattr global_safe['json'] = json + class PyScriptRunner(BaseActor): def execute(self, source_code: Text, predefined_objects: Optional[Dict] = None, **kwargs): diff --git a/kairon/shared/data/constant.py b/kairon/shared/data/constant.py index 00b573548..b80e82ee0 100644 --- a/kairon/shared/data/constant.py +++ b/kairon/shared/data/constant.py @@ -208,6 +208,7 @@ class ModelTestType(str, Enum): DEFAULT_SYSTEM_PROMPT = ( "You are a personal assistant. Answer question based on the context below" ) +DEFAULT_LLM = "openai" class AuditlogActions(str, Enum): diff --git a/kairon/shared/data/processor.py b/kairon/shared/data/processor.py index 826b5b492..82fea0677 100644 --- a/kairon/shared/data/processor.py +++ b/kairon/shared/data/processor.py @@ -7270,9 +7270,7 @@ def edit_prompt_action( action.failure_message = request_data.get("failure_message") action.user_question = UserQuestion(**request_data.get("user_question")) action.num_bot_responses = request_data.get("num_bot_responses", 5) - action.hyperparameters = request_data.get( - "hyperparameters", Utility.get_llm_hyperparameters() - ) + action.hyperparameters = request_data.get("hyperparameters") action.llm_prompts = [ LlmPrompt(**prompt) for prompt in request_data.get("llm_prompts", []) ] diff --git a/kairon/shared/llm/base.py b/kairon/shared/llm/base.py index 4babc6a23..f07eceda0 100644 --- a/kairon/shared/llm/base.py +++ b/kairon/shared/llm/base.py @@ -8,9 +8,9 @@ def __init__(self, bot: Text): self.bot = bot @abstractmethod - async def train(self, *args, **kwargs) -> Dict: + async def train(self, user, bot, *args, **kwargs) -> Dict: pass @abstractmethod - async def predict(self, query, *args, **kwargs) -> Dict: + async def predict(self, query, user, bot, *args, **kwargs) -> Dict: pass diff --git a/kairon/shared/llm/clients/__init__.py b/kairon/shared/llm/clients/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/kairon/shared/llm/clients/azure.py b/kairon/shared/llm/clients/azure.py deleted file mode 100644 index 9b980d0d6..000000000 --- a/kairon/shared/llm/clients/azure.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Text - -from kairon.shared.constants import GPT3ResourceTypes -from kairon.shared.llm.clients.gpt3 import GPT3Resources - - -class AzureGPT3Resources(GPT3Resources): - resource_url = "https://kairon.openai.azure.com/openai/deployments" - - def __init__(self, api_key: Text, **kwargs): - super().__init__(api_key) - self.api_key = api_key - self.api_version = kwargs.get("api_version") - self.model_id = { - GPT3ResourceTypes.embeddings.value: kwargs.get("embeddings_model_id"), - GPT3ResourceTypes.chat_completion.value: kwargs.get("chat_completion_model_id") - } - - def get_headers(self): - return {"api-key": self.api_key} - - def get_resource_url(self, resource: Text): - model_id = self.model_id[resource] - resource_url = f"{self.resource_url}/{model_id}/{resource}?api-version={self.api_version}" - return resource_url diff --git a/kairon/shared/llm/clients/base.py b/kairon/shared/llm/clients/base.py deleted file mode 100644 index 71ef7037e..000000000 --- a/kairon/shared/llm/clients/base.py +++ /dev/null @@ -1,8 +0,0 @@ -from abc import ABC -from typing import Text - - -class LLMResources(ABC): - - async def invoke(self, resource: Text, engine: Text, **kwargs): - raise NotImplementedError("Provider not implemented") diff --git a/kairon/shared/llm/clients/factory.py b/kairon/shared/llm/clients/factory.py deleted file mode 100644 index def8d09c3..000000000 --- a/kairon/shared/llm/clients/factory.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import Text -from kairon.exceptions import AppException -from kairon.shared.constants import LLMResourceProvider -from kairon.shared.llm.clients.azure import AzureGPT3Resources -from kairon.shared.llm.clients.gpt3 import GPT3Resources - - -class LLMClientFactory: - __implementations = { - LLMResourceProvider.openai.value: GPT3Resources, - LLMResourceProvider.azure.value: AzureGPT3Resources - } - - @staticmethod - def get_resource_provider(_type: Text): - if not LLMClientFactory.__implementations.get(_type): - raise AppException(f'{_type} client not supported') - return LLMClientFactory.__implementations[_type] diff --git a/kairon/shared/llm/clients/gpt3.py b/kairon/shared/llm/clients/gpt3.py deleted file mode 100644 index d6f2c5679..000000000 --- a/kairon/shared/llm/clients/gpt3.py +++ /dev/null @@ -1,92 +0,0 @@ -import ujson as json -import random -from json import JSONDecodeError -from ujson import JSONDecodeError as UJSONDecodeError -from typing import Text -from loguru import logger -from openai.api_requestor import parse_stream_helper -from kairon.exceptions import AppException -from kairon.shared.constants import GPT3ResourceTypes -from kairon.shared.llm.clients.base import LLMResources -from kairon.shared.rest_client import AioRestClient - - -class GPT3Resources(LLMResources): - resource_url = "https://api.openai.com/v1" - - def __init__(self, api_key: Text, **kwargs): - self.api_key = api_key - - def get_headers(self): - return {"Authorization": f"Bearer {self.api_key}"} - - def get_resource_url(self, resource: Text): - return f"{self.resource_url}/{resource}" - - async def invoke(self, resource: Text, model: Text, **kwargs): - client = None - http_url = self.get_resource_url(resource) - request_body = kwargs.copy() - request_body.update({"model": model}) - is_streaming_resp = kwargs.get("stream", False) - try: - client = AioRestClient(False) - resp = await client.request("POST", http_url, request_body, self.get_headers(), - return_json=False, is_streaming_resp=is_streaming_resp, max_retries=3) - if resp.status != 200: - try: - resp = await resp.json() - logger.debug(f"GPT response error: {resp}") - raise AppException(f"{resp['error'].get('message')}. Request id: {resp['error'].get('id')}") - except JSONDecodeError: - raise AppException(f"Received non 200 status code ({resp.status}): {resp.text}") - - if is_streaming_resp: - resp = client.streaming_response - - data = await self.__parse_response(resource, resp, **kwargs) - finally: - if client: - await client.cleanup() - return data - - async def __parse_response(self, resource: Text, response, **kwargs): - parsers = { - GPT3ResourceTypes.embeddings.value: self._parse_embeddings_response, - GPT3ResourceTypes.chat_completion.value: self.__parse_completion_response - } - return await parsers[resource](response, **kwargs) - - async def _parse_embeddings_response(self, response, **hyperparameters): - raw_response = await response.json() - formatted_response = raw_response["data"][0]["embedding"] - return formatted_response, raw_response - - async def __parse_completion_response(self, response, **kwargs): - if kwargs.get("stream"): - formatted_response = await self._parse_streaming_response(response, kwargs.get("n", 1)) - raw_response = response - else: - formatted_response, raw_response = await self._parse_api_response(response) - return formatted_response, raw_response - - async def _parse_api_response(self, response): - raw_response = await response.json() - msg_choice = random.choice(raw_response['choices']) - formatted_response = msg_choice['message']['content'] - return formatted_response, raw_response - - async def _parse_streaming_response(self, response, num_choices): - formatted_response = '' - msg_choice = random.randint(0, num_choices - 1) - try: - for chunk in response or []: - line = parse_stream_helper(chunk) - if line: - line = json.loads(line) - if line["choices"][0].get("index") == msg_choice and line["choices"][0]['delta'].get('content'): - formatted_response = f"{formatted_response}{line['choices'][0]['delta']['content']}" - except Exception as e: - logger.exception(e) - raise AppException(f"Failed to parse streaming response: {chunk}") - return formatted_response diff --git a/kairon/shared/llm/data_objects.py b/kairon/shared/llm/data_objects.py new file mode 100644 index 000000000..7713444ba --- /dev/null +++ b/kairon/shared/llm/data_objects.py @@ -0,0 +1,13 @@ +from mongoengine import Document, DynamicField, StringField, FloatField, DateTimeField, DictField + + +class LLMLogs(Document): + response = DynamicField() + start_time = DateTimeField() + end_time = DateTimeField() + cost = FloatField() + llm_call_id = StringField() + llm_provider = StringField() + model = StringField() + model_params = DictField() + metadata = DictField() \ No newline at end of file diff --git a/kairon/shared/llm/factory.py b/kairon/shared/llm/factory.py deleted file mode 100644 index 5424d1eea..000000000 --- a/kairon/shared/llm/factory.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Text -from kairon.exceptions import AppException -from kairon.shared.utils import Utility -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - - -class LLMFactory: - __implementations = { - "GPT3_FAQ_EMBED": GPT3FAQEmbedding - } - - @staticmethod - def get_instance(_type: Text): - llm_type = Utility.environment['llm'][_type] - if not LLMFactory.__implementations.get(llm_type): - raise AppException(f'{llm_type} type LLM is not supported') - return LLMFactory.__implementations[llm_type] diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py new file mode 100644 index 000000000..06b720b48 --- /dev/null +++ b/kairon/shared/llm/logger.py @@ -0,0 +1,43 @@ +from litellm.integrations.custom_logger import CustomLogger +from .data_objects import LLMLogs +import ujson as json + + +class LiteLLMLogger(CustomLogger): + def log_pre_api_call(self, model, messages, kwargs): + pass + + def log_post_api_call(self, kwargs, response_obj, start_time, end_time): + pass + + def log_stream_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + async def async_log_stream_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + def __logs_litellm(self, **kwargs): + litellm_params = kwargs['litellm_params'] + self.__save_logs(**{'response': json.loads(kwargs['original_response']), + 'start_time': kwargs['start_time'], + 'end_time': kwargs['end_time'], + 'cost': kwargs["response_cost"], + 'llm_call_id': litellm_params['litellm_call_id'], + 'llm_provider': litellm_params['custom_llm_provider'], + 'model_params': kwargs["additional_args"]["complete_input_dict"], + 'metadata': litellm_params['metadata']}) + + def __save_logs(self, **kwargs): + LLMLogs(**kwargs).save() diff --git a/kairon/shared/llm/gpt3.py b/kairon/shared/llm/processor.py similarity index 73% rename from kairon/shared/llm/gpt3.py rename to kairon/shared/llm/processor.py index 4e991ca7c..ffc48e2eb 100644 --- a/kairon/shared/llm/gpt3.py +++ b/kairon/shared/llm/processor.py @@ -1,9 +1,9 @@ +import random import time - from typing import Text, Dict, List, Tuple from urllib.parse import urljoin -import openai +import litellm from loguru import logger as logging from tiktoken import get_encoding from tqdm import tqdm @@ -13,35 +13,33 @@ from kairon.shared.admin.processor import Sysadmin from kairon.shared.cognition.data_objects import CognitionData from kairon.shared.cognition.processor import CognitionDataProcessor -from kairon.shared.constants import GPT3ResourceTypes from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_CONTEXT_PROMPT from kairon.shared.llm.base import LLMBase -from kairon.shared.llm.clients.factory import LLMClientFactory +from kairon.shared.llm.logger import LiteLLMLogger from kairon.shared.models import CognitionDataType from kairon.shared.rest_client import AioRestClient from kairon.shared.utils import Utility +litellm.callbacks = [LiteLLMLogger()] + -class GPT3FAQEmbedding(LLMBase): +class LLMProcessor(LLMBase): __embedding__ = 1536 - def __init__(self, bot: Text, llm_settings: dict): + def __init__(self, bot: Text): super().__init__(bot) self.db_url = Utility.environment['vector']['db'] self.headers = {} if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.suffix = "_faq_embd" - self.vector_config = {'size': 1536, 'distance': 'Cosine'} - self.llm_settings = llm_settings + self.vector_config = {'size': self.__embedding__, 'distance': 'Cosine'} self.api_key = Sysadmin.get_bot_secret(bot, BotSecretType.gpt_key.value, raise_err=True) - self.client = LLMClientFactory.get_resource_provider(llm_settings["provider"])(self.api_key, - **self.llm_settings) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 self.__logs = [] - async def train(self, *args, **kwargs) -> Dict: + async def train(self, user, bot, *args, **kwargs) -> Dict: await self.__delete_collections() count = 0 processor = CognitionDataProcessor() @@ -51,35 +49,38 @@ async def train(self, *args, **kwargs) -> Dict: {'$project': {'collection': "$_id", 'content': 1, '_id': 0}} ])) for collections in collection_groups: - collection = f"{self.bot}_{collections['collection']}{self.suffix}" if collections['collection'] else f"{self.bot}{self.suffix}" + collection = f"{self.bot}_{collections['collection']}{self.suffix}" if collections[ + 'collection'] else f"{self.bot}{self.suffix}" await self.__create_collection__(collection) for content in tqdm(collections['content'], desc="Training FAQ"): if content['content_type'] == CognitionDataType.json.value: metadata = processor.find_matching_metadata(self.bot, content['data'], content.get('collection')) - search_payload, embedding_payload = Utility.retrieve_search_payload_and_embedding_payload(content['data'], metadata) + search_payload, embedding_payload = Utility.retrieve_search_payload_and_embedding_payload( + content['data'], metadata) else: search_payload, embedding_payload = {'content': content["data"]}, content["data"] - search_payload['collection_name'] = collection - embeddings = await self.__get_embedding(embedding_payload) + #search_payload['collection_name'] = collection + embeddings = await self.get_embedding(embedding_payload, user, bot) points = [{'id': content['vector_id'], 'vector': embeddings, 'payload': search_payload}] - await self.__collection_upsert__(collection, {'points': points}, err_msg="Unable to train FAQ! Contact support") + await self.__collection_upsert__(collection, {'points': points}, + err_msg="Unable to train FAQ! Contact support") count += 1 return {"faq": count} - async def predict(self, query: Text, *args, **kwargs) -> Tuple: + async def predict(self, query: Text, user, bot, *args, **kwargs) -> Tuple: start_time = time.time() embeddings_created = False try: - query_embedding = await self.__get_embedding(query) + query_embedding = await self.get_embedding(query, user, bot) embeddings_created = True system_prompt = kwargs.pop('system_prompt', DEFAULT_SYSTEM_PROMPT) context_prompt = kwargs.pop('context_prompt', DEFAULT_CONTEXT_PROMPT) context = await self.__attach_similarity_prompt_if_enabled(query_embedding, context_prompt, **kwargs) - answer = await self.__get_answer(query, system_prompt, context, **kwargs) + answer = await self.__get_answer(query, system_prompt, context, user, bot, **kwargs) response = {"content": answer} - except openai.error.APIConnectionError as e: + except Exception as e: logging.exception(e) if embeddings_created: failure_stage = "Retrieving chat completion for the provided query." @@ -87,9 +88,6 @@ async def predict(self, query: Text, *args, **kwargs) -> Tuple: failure_stage = "Creating a new embedding for the provided query." self.__logs.append({'error': f"{failure_stage} {str(e)}"}) response = {"is_failure": True, "exception": str(e), "content": None} - except Exception as e: - logging.exception(e) - response = {"is_failure": True, "exception": str(e), "content": None} end_time = time.time() elapsed_time = end_time - start_time @@ -102,13 +100,36 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def __get_embedding(self, text: Text) -> List[float]: + async def get_embedding(self, text: Text, user, bot) -> List[float]: truncated_text = self.truncate_text(text) - result, _ = await self.client.invoke(GPT3ResourceTypes.embeddings.value, model="text-embedding-3-small", - input=truncated_text) - return result + result = await litellm.aembedding(model="text-embedding-3-small", + input=[truncated_text], + metadata={'user': user, 'bot': bot}, + api_key=self.api_key, + num_retries=3) + return result["data"][0]["embedding"] + + async def __parse_completion_response(self, response, **kwargs): + if kwargs.get("stream"): + formatted_response = '' + msg_choice = random.randint(0, kwargs.get("n", 1) - 1) + if response["choices"][0].get("index") == msg_choice and response["choices"][0]['delta'].get('content'): + formatted_response = f"{response['choices'][0]['delta']['content']}" + else: + msg_choice = random.choice(response['choices']) + formatted_response = msg_choice['message']['content'] + return formatted_response + + async def __get_completion(self, messages, hyperparameters, user, bot, **kwargs): + response = await litellm.acompletion(messages=messages, + metadata={'user': user, 'bot': bot}, + api_key=self.api_key, + num_retries=3, + **hyperparameters) + formatted_response = await self.__parse_completion_response(response, **kwargs) + return formatted_response, response - async def __get_answer(self, query, system_prompt: Text, context: Text, **kwargs): + async def __get_answer(self, query, system_prompt: Text, context: Text, user, bot, **kwargs): use_query_prompt = False query_prompt = '' if kwargs.get('query_prompt', {}): @@ -116,12 +137,15 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, **kwargs query_prompt = query_prompt_dict.get('query_prompt', '') use_query_prompt = query_prompt_dict.get('use_query_prompt') previous_bot_responses = kwargs.get('previous_bot_responses') - hyperparameters = kwargs.get('hyperparameters', Utility.get_llm_hyperparameters()) + hyperparameters = kwargs['hyperparameters'] instructions = kwargs.get('instructions', []) instructions = '\n'.join(instructions) if use_query_prompt and query_prompt: - query = await self.__rephrase_query(query, system_prompt, query_prompt, hyperparameters=hyperparameters) + query = await self.__rephrase_query(query, system_prompt, query_prompt, + hyperparameters=hyperparameters, + user=user, + bot=bot) messages = [ {"role": "system", "content": system_prompt}, ] @@ -130,21 +154,25 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, **kwargs messages.append({"role": "user", "content": f"{context} \n{instructions} \nQ: {query} \nA:"}) if instructions \ else messages.append({"role": "user", "content": f"{context} \nQ: {query} \nA:"}) - completion, raw_response = await self.client.invoke(GPT3ResourceTypes.chat_completion.value, messages=messages, - **hyperparameters) + completion, raw_response = await self.__get_completion(messages=messages, + hyperparameters=hyperparameters, + user=user, + bot=bot) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'answer_query', 'hyperparameters': hyperparameters}) return completion - async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, **kwargs): + async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, bot, **kwargs): messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": f"{query_prompt}\n\n Q: {query}\n A:"} ] - hyperparameters = kwargs.get('hyperparameters', Utility.get_llm_hyperparameters()) + hyperparameters = kwargs['hyperparameters'] - completion, raw_response = await self.client.invoke(GPT3ResourceTypes.chat_completion.value, messages=messages, - **hyperparameters) + completion, raw_response = await self.__get_completion(messages=messages, + hyperparameters=hyperparameters, + user=user, + bot=bot) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'rephrase_query', 'hyperparameters': hyperparameters}) return completion @@ -153,9 +181,9 @@ async def __delete_collections(self): client = AioRestClient(False) try: response = await client.request(http_url=urljoin(self.db_url, "/collections"), - request_method="GET", - headers=self.headers, - timeout=5) + request_method="GET", + headers=self.headers, + timeout=5) if response.get('result'): for collection in response['result'].get('collections') or []: if collection['name'].startswith(self.bot): diff --git a/kairon/shared/utils.py b/kairon/shared/utils.py index 625ebad88..5cb61ce02 100644 --- a/kairon/shared/utils.py +++ b/kairon/shared/utils.py @@ -78,6 +78,7 @@ TOKEN_TYPE, KAIRON_TWO_STAGE_FALLBACK, SLOT_TYPE, + DEFAULT_LLM ) from .data.dto import KaironStoryStep from .models import StoryStepType, LlmPromptType, LlmPromptSource @@ -2050,73 +2051,33 @@ def verify_email(email: Text): raise AppException("Invalid or disposable Email!") @staticmethod - def get_llm_hyperparameters(): + def get_llms(): + return Utility.system_metadata["llm"].keys() + + @staticmethod + def get_default_llm_hyperparameters(): + return Utility.get_llm_hyperparameters(DEFAULT_LLM) + + @staticmethod + def get_llm_hyperparameters(llm_type): hyperparameters = {} - if Utility.environment["llm"]["faq"] in {"GPT3_FAQ_EMBED"}: - for key, value in Utility.system_metadata["llm"]["gpt"].items(): + if llm_type in Utility.system_metadata["llm"].keys(): + for key, value in Utility.system_metadata["llm"][llm_type]['properties'].items(): hyperparameters[key] = value["default"] return hyperparameters - raise AppException("Could not find any hyperparameters for configured LLM.") + raise AppException(f"Could not find any hyperparameters for {llm_type} LLM.") @staticmethod - def validate_llm_hyperparameters(hyperparameters: dict, exception_class): - params = Utility.system_metadata["llm"]["gpt"] - for key, value in hyperparameters.items(): - if ( - key == "temperature" - and not params["temperature"]["min"] - <= value - <= params["temperature"]["max"] - ): - raise exception_class( - f"Temperature must be between {params['temperature']['min']} and {params['temperature']['max']}!" - ) - elif ( - key == "presence_penalty" - and not params["presence_penalty"]["min"] - <= value - <= params["presence_penalty"]["max"] - ): - raise exception_class( - f"Presence penalty must be between {params['presence_penalty']['min']} and {params['presence_penalty']['max']}!" - ) - elif ( - key == "frequency_penalty" - and not params["presence_penalty"]["min"] - <= value - <= params["presence_penalty"]["max"] - ): - raise exception_class( - f"Frequency penalty must be between {params['presence_penalty']['min']} and {params['presence_penalty']['max']}!" - ) - elif ( - key == "top_p" - and not params["top_p"]["min"] <= value <= params["top_p"]["max"] - ): - raise exception_class( - f"top_p must be between {params['top_p']['min']} and {params['top_p']['max']}!" - ) - elif key == "n" and not params["n"]["min"] <= value <= params["n"]["max"]: - raise exception_class( - f"n must be between {params['n']['min']} and {params['n']['max']} and should not be 0!" - ) - elif ( - key == "max_tokens" - and not params["max_tokens"]["min"] - <= value - <= params["max_tokens"]["max"] - ): - raise exception_class( - f"max_tokens must be between {params['max_tokens']['min']} and {params['max_tokens']['max']} and should not be 0!" - ) - elif key == "logit_bias" and not isinstance(value, dict): - raise exception_class("logit_bias must be a dictionary!") - elif key == "stop": - exc_msg = "Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers." - if value and not isinstance(value, (str, int, list)): - raise exception_class(exc_msg) - elif value and (isinstance(value, list) and len(value) > 4): - raise exception_class(exc_msg) + def validate_llm_hyperparameters(hyperparameters: dict, llm_type: str, exception_class): + from jsonschema_rs import JSONSchema, ValidationError + schema = Utility.system_metadata["llm"][llm_type] + try: + validator = JSONSchema(schema) + validator.validate(hyperparameters) + except ValidationError as e: + message = f"{e.instance_path}: {e.message}" + raise exception_class(message) + @staticmethod def create_uuid_from_string(val: str): diff --git a/kairon/shared/vector_embeddings/db/base.py b/kairon/shared/vector_embeddings/db/base.py index 178ee25de..d1c2a1e97 100644 --- a/kairon/shared/vector_embeddings/db/base.py +++ b/kairon/shared/vector_embeddings/db/base.py @@ -8,16 +8,16 @@ class VectorEmbeddingsDbBase(ABC): @abstractmethod - async def embedding_search(self, request_body: Dict): + async def embedding_search(self, request_body: Dict, **kwargs): raise NotImplementedError("Provider not implemented") @abstractmethod - async def payload_search(self, request_body: Dict): + async def payload_search(self, request_body: Dict, **kwargs): raise NotImplementedError("Provider not implemented") - async def perform_operation(self, op_type: Text, request_body: Dict): + async def perform_operation(self, op_type: Text, request_body: Dict, **kwargs): supported_ops = {DbActionOperationType.payload_search.value: self.payload_search, DbActionOperationType.embedding_search.value: self.embedding_search} if op_type not in supported_ops.keys(): raise AppException("Operation type not supported") - return await supported_ops[op_type](request_body) + return await supported_ops[op_type](request_body, **kwargs) diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index d2ff7e69c..893a310ad 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -8,7 +8,7 @@ from kairon.shared.admin.constants import BotSecretType from kairon.shared.admin.processor import Sysadmin from kairon.shared.constants import GPT3ResourceTypes -from kairon.shared.llm.clients.factory import LLMClientFactory +from kairon.shared.llm.processor import LLMProcessor from kairon.shared.vector_embeddings.db.base import VectorEmbeddingsDbBase @@ -25,9 +25,7 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.llm_settings = llm_settings - self.api_key = Sysadmin.get_bot_secret(self.bot, BotSecretType.gpt_key.value, raise_err=True) - self.client = LLMClientFactory.get_resource_provider(llm_settings["provider"])(self.api_key, - **self.llm_settings) + self.llm = LLMProcessor(self.bot) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 @@ -38,18 +36,16 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def __get_embedding(self, text: Text) -> List[float]: - truncated_text = self.truncate_text(text) - result, _ = await self.client.invoke(GPT3ResourceTypes.embeddings.value, model="text-embedding-3-small", - input=truncated_text) + async def __get_embedding(self, text: Text, **kwargs) -> List[float]: + result, _ = await self.llm.get_embedding(text, user=kwargs.get('user'), bot=kwargs.get('bot')) return result - async def embedding_search(self, request_body: Dict): + async def embedding_search(self, request_body: Dict, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points") if request_body.get("text"): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/search") user_msg = request_body.get("text") - vector = await self.__get_embedding(user_msg) + vector = await self.__get_embedding(user_msg, **kwargs) request_body = {'vector': vector, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} embedding_search_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', @@ -57,7 +53,7 @@ async def embedding_search(self, request_body: Dict): request_body=request_body) return embedding_search_result - async def payload_search(self, request_body: Dict): + async def payload_search(self, request_body: Dict, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/scroll") payload_filter_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', diff --git a/kairon/train.py b/kairon/train.py index 05e761dc5..0276f7bc5 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -13,10 +13,10 @@ from kairon.shared.data.constant import EVENT_STATUS from kairon.shared.data.model_processor import ModelProcessor from kairon.shared.data.processor import MongoProcessor -from kairon.shared.llm.factory import LLMFactory from kairon.shared.metering.constants import MetricType from kairon.shared.metering.metering_processor import MeteringProcessor from kairon.shared.utils import Utility +from kairon.shared.llm.processor import LLMProcessor def train_model_for_bot(bot: str): @@ -81,6 +81,7 @@ def train_model_for_bot(bot: str): raise AppException(e) return model + def start_training(bot: str, user: str, token: str = None): """ prevents training of the bot, @@ -100,8 +101,8 @@ def start_training(bot: str, user: str, token: str = None): settings = processor.get_bot_settings(bot, user) settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: - llm = LLMFactory.get_instance("faq")(bot, settings["llm_settings"]) - faqs = asyncio.run(llm.train()) + llm_processor = LLMProcessor(bot) + faqs = asyncio.run(llm_processor.train(user=user, bot=bot)) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) agent_url = Utility.environment['model']['agent'].get('url') diff --git a/metadata/integrations.yml b/metadata/integrations.yml index f65e67b57..227b4c413 100644 --- a/metadata/integrations.yml +++ b/metadata/integrations.yml @@ -95,59 +95,74 @@ live_agents: websocket_url: wss://app.chatwoot.com/cable llm: - gpt: - temperature: - type: float - default: 0.0 - min: 0.0 - max: 2.0 - description: "The temperature hyperparameter controls the creativity or randomness of the generated responses." - max_tokens: - type: int - default: 300 - min: 5 - max: 4096 - description: "The max_tokens hyperparameter limits the length of generated responses in chat completion using ChatGPT." - model: - type: str - default: "gpt-3.5-turbo" - description: "The model hyperparameter is the ID of the model to use such as gpt-2, gpt-3, or a custom model that you have trained or fine-tuned." - top_p: - type: float - default: 0.0 - min: 0.0 - max: 1.0 - description: "The top_p hyperparameter is a value that controls the diversity of the generated responses." - n: - type: int - default: 1 - min: 1 - max: 5 - description: "The n hyperparameter controls the number of different response options that are generated by the model." - stream: - type: bool - default: false - description: "the stream hyperparameter controls whether the model generates a continuous stream of text or a single response." - stop: - type: - - str - - array - - int - default: null - description: "The stop hyperparameter is used to specify a list of tokens that should be used to indicate the end of a generated response." - presence_penalty: - type: float - default: 0.0 - min: -2.0 - max: 2.0 - description: "The presence_penalty hyperparameter penalizes the model for generating words that are not present in the context or input prompt. " - frequency_penalty: - type: float - default: 0.0 - min: -2.0 - max: 2.0 - description: "The frequency_penalty hyperparameter penalizes the model for generating words that have already been generated in the current response." - logit_bias: - type: dict - default: {} - description: "The logit_bias hyperparameter helps prevent GPT-3 from generating unwanted tokens or even to encourage generation of tokens that you do want. " + openai: + $schema: "https://json-schema.org/draft/2020-12/schema" + type: object + description: "Open AI Models for Prompt" + properties: + temperature: + type: number + default: 0.0 + minimum: 0.0 + maximum: 2.0 + description: "The temperature hyperparameter controls the creativity or randomness of the generated responses." + max_tokens: + type: integer + default: 300 + minimum: 5 + maximum: 4096 + description: "The max_tokens hyperparameter limits the length of generated responses in chat completion using ChatGPT." + model: + type: string + default: "gpt-3.5-turbo" + enum: ["gpt-3.5-turbo", "gpt-3.5-turbo-instruct"] + description: "The model hyperparameter is the ID of the model to use such as gpt-2, gpt-3, or a custom model that you have trained or fine-tuned." + top_p: + type: number + default: 0.0 + minimum: 0.0 + maximum: 1.0 + description: "The top_p hyperparameter is a value that controls the diversity of the generated responses." + n: + type: integer + default: 1 + minimum: 1 + maximum: 5 + description: "The n hyperparameter controls the number of different response options that are generated by the model." + stream: + type: boolean + default: false + description: "the stream hyperparameter controls whether the model generates a continuous stream of text or a single response." + stop: + anyOf: + - type: "string" + - type: "array" + maxItems: 4 + items: + type: "string" + - type: "integer" + - type: "null" + + type: + - "string" + - "array" + - "integer" + - "null" + default: null + description: "The stop hyperparameter is used to specify a list of tokens that should be used to indicate the end of a generated response." + presence_penalty: + type: number + default: 0.0 + minimum: -2.0 + maximum: 2.0 + description: "The presence_penalty hyperparameter penalizes the model for generating words that are not present in the context or input prompt. " + frequency_penalty: + type: number + default: 0.0 + minimum: -2.0 + maximum: 2.0 + description: "The frequency_penalty hyperparameter penalizes the model for generating words that have already been generated in the current response." + logit_bias: + type: object + default: {} + description: "The logit_bias hyperparameter helps prevent GPT-3 from generating unwanted tokens or even to encourage generation of tokens that you do want. " diff --git a/requirements/dev.txt b/requirements/dev.txt index d411ab022..19268e061 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,16 +1,15 @@ -r prod.txt -pytest==8.1.1 +pytest==8.2.2 pytest_httpx==0.30.0 -pytest-asyncio==0.23.6 -responses==0.25.0 +pytest-asyncio==0.23.7 +responses==0.25.2 mock==5.1.0 -moto[all]==5.0.5 +moto[all]==5.0.9 mongomock==4.1.2 black==22.12.0 -locust==2.25.0 +locust==2.29.0 deepdiff==7.0.1 pytest-cov==5.0.0 pytest-html==4.1.1 pytest-aioresponses==0.2.0 -aioresponses==0.7.6 -pykwalify==1.8.0 \ No newline at end of file +aioresponses==0.7.6 \ No newline at end of file diff --git a/requirements/prod.txt b/requirements/prod.txt index 19c6817ff..e00d448a9 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1,65 +1,69 @@ rasa[full]==3.6.20 mongoengine==0.28.2 fastapi==0.110.2 -uvicorn[standard]==0.29.0 +uvicorn[standard]==0.30.1 smart-config==0.1.3 fastapi_sso==0.9.1 fastapi-keycloak==1.0.10 pykka==3.1.1 zenpy==2.0.42 -validators==0.28.0 +validators==0.28.3 secure==0.3.0 password-strength==0.0.3.post2 beautifulsoup4==4.12.3 uuid6==2024.01.12 passlib[bcrypt]==1.7.4 -openai==0.28.1 json2html==1.3.0 -google-api-python-client==2.110.0 -jira==3.5.2 +google-api-python-client==2.133.0 +jira==3.8.0 pipedrive-python-lib==1.2.3 -google-cloud-translate==3.13.0 -blinker==1.7.0 -pymupdf==1.23.7 -python-docx==1.1.0 +google-cloud-translate==3.15.3 +blinker==1.8.2 +pymupdf==1.24.5 +python-docx==1.1.2 python-multipart==0.0.9 pandas==2.2.2 -openpyxl==3.1.2 +openpyxl==3.1.4 sentencepiece==0.1.99 -dramatiq==1.15.0 +dramatiq==1.17.0 dramatiq-mongodb==0.8.3 nlpaug==1.1.11 keybert==0.8.4 -pyTelegramBotAPI==4.17.0 +pyTelegramBotAPI==4.19.1 APScheduler==3.9.1.post1 -croniter==2.0.3 +croniter==2.0.5 faiss-cpu==1.8.0 -tiktoken==0.6.0 +tiktoken==0.7.0 RestrictedPython==7.1 -AccessControl==6.3 +AccessControl==7.0 timeout-decorator==0.5.0 -googlesearch-python==1.2.3 +googlesearch-python==1.2.4 aiohttp-retry==2.8.3 pqdict==1.4.0 google-businessmessages==1.0.5 google-apitools==0.5.32 -orjson==3.10.1 -opentelemetry-distro[otlp]==0.45b0 +orjson==3.10.5 +opentelemetry-distro[otlp]==0.46b0 opentelemetry-sdk-extension-aws==2.0.1 opentelemetry-propagator-aws-xray==1.0.1 -opentelemetry-instrumentation-fastapi==0.45b0 -opentelemetry-instrumentation-aiohttp-client==0.45b0 -opentelemetry-instrumentation-asyncio==0.45b0 -opentelemetry-instrumentation-aws-lambda==0.45b0 -opentelemetry-instrumentation-boto==0.45b0 -opentelemetry-instrumentation-botocore==0.45b0 -opentelemetry-instrumentation-httpx==0.45b0 -opentelemetry-instrumentation-logging==0.45b0 -opentelemetry-instrumentation-pymongo==0.45b0 -opentelemetry-instrumentation-requests==0.45b0 -opentelemetry-instrumentation-system-metrics==0.45b0 -opentelemetry-instrumentation-grpc==0.45b0 -opentelemetry-instrumentation-sklearn==0.45b0 -opentelemetry-instrumentation-asgi==0.45b0 +opentelemetry-instrumentation-fastapi==0.46b0 +opentelemetry-instrumentation-aiohttp-client==0.46b0 +opentelemetry-instrumentation-asyncio==0.46b0 +opentelemetry-instrumentation-aws-lambda==0.46b0 +opentelemetry-instrumentation-boto==0.46b0 +opentelemetry-instrumentation-botocore==0.46b0 +opentelemetry-instrumentation-httpx==0.46b0 +opentelemetry-instrumentation-logging==0.46b0 +opentelemetry-instrumentation-pymongo==0.46b0 +opentelemetry-instrumentation-requests==0.46b0 +opentelemetry-instrumentation-system-metrics==0.46b0 +opentelemetry-instrumentation-grpc==0.46b0 +opentelemetry-instrumentation-sklearn==0.46b0 +opentelemetry-instrumentation-asgi==0.46b0 +opentelemetry-instrumentation-requests==0.46b0 +opentelemetry-instrumentation-sklearn==0.46b0 pykwalify==1.8.0 gunicorn==22.0.0 +litellm==1.38.11 +jsonschema_rs==0.18.0 +mongoengine-jsonschema==0.1.3 \ No newline at end of file diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index 2012234ba..1941a8fd6 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -2,6 +2,7 @@ import os from urllib.parse import urlencode, urljoin +import litellm import mock import numpy as np import pytest @@ -10,8 +11,11 @@ from deepdiff import DeepDiff from fastapi.testclient import TestClient from jira import JIRAError -from mock import patch -from mongoengine import connect, DoesNotExist +from mongoengine import connect + +from kairon.shared.utils import Utility + +Utility.load_system_metadata() from kairon.actions.definitions.live_agent import ActionLiveAgent from kairon.actions.definitions.set_slot import ActionSetSlot @@ -32,15 +36,13 @@ DEFAULT_NLU_FALLBACK_RESPONSE from kairon.shared.data.data_objects import Slots, KeyVault, BotSettings, LLMSettings from kairon.shared.data.processor import MongoProcessor -from kairon.shared.llm.clients.gpt3 import GPT3Resources -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding -from kairon.shared.utils import Utility from kairon.shared.vector_embeddings.db.qdrant import Qdrant os.environ['ASYNC_TEST_TIMEOUT'] = "360" os.environ["system_file"] = "./tests/testing_data/system.yaml" client = TestClient(action) +OPENAI_EMBEDDING_OUTPUT = 1536 @pytest.fixture(autouse=True, scope='class') @@ -82,7 +84,8 @@ def test_live_agent_action_execution(aioresponses): aioresponses.add( method="POST", url=f"{Utility.environment['live_agent']['url']}/conversation/request", - payload={"success": True, "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": None }, "message": None, "error_code": 0}, + payload={"success": True, "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": None}, "message": None, + "error_code": 0}, body={'bot_id': '5f50fd0a56b698ca10d35d2z', 'sender_id': 'default', 'channel': 'messenger'}, status=200 ) @@ -185,7 +188,9 @@ def test_live_agent_action_execution_no_agent_available(aioresponses): aioresponses.add( method="POST", url=f"{Utility.environment['live_agent']['url']}/conversation/request", - payload={"success": True, "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": "live agent is not available" }, "message": None, "error_code": 0}, + payload={"success": True, + "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": "live agent is not available"}, + "message": None, "error_code": 0}, body={'bot_id': '5f50fd0a56b698ca10d35d2z', 'sender_id': 'default', 'channel': 'messenger'}, status=200 ) @@ -275,7 +280,6 @@ def test_live_agent_action_execution_no_agent_available(aioresponses): assert response_json['responses'][0]['text'] == 'live agent is not available' - def test_live_agent_action_execution_with_exception(aioresponses): bot_settings = BotSettings(bot='5f50fd0a56b698ca10d35d21', user='user') bot_settings.live_agent_enabled = True @@ -384,7 +388,9 @@ def test_live_agent_action_execution_with_exception(aioresponses): assert response.status_code == 200 assert len(response_json['responses']) == 1 assert response_json['responses'][0]['text'] == 'Connecting to live agent' - assert response_json == {'events': [], 'responses': [{'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}]} + assert response_json == {'events': [], 'responses': [ + {'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, + 'response': None, 'image': None, 'attachment': None}]} def test_live_agent_action_execution_with_exception(aioresponses): @@ -495,11 +501,12 @@ def test_live_agent_action_execution_with_exception(aioresponses): assert response.status_code == 200 assert len(response_json['responses']) == 1 assert response_json['responses'][0]['text'] == 'Connecting to live agent' - assert response_json == {'events': [], 'responses': [{'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}]} + assert response_json == {'events': [], 'responses': [ + {'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, + 'response': None, 'image': None, 'attachment': None}]} def test_retrieve_config_failure(): - patch('kairon.actions.definitions.live_agent.LiveAgentActionConfig.objects().get', side_effect=DoesNotExist) action_live_agent = ActionLiveAgent(bot='test_bot', name='test_action') with pytest.raises(ActionFailure, match="No Live Agent action found for given action and bot"): action_live_agent.retrieve_config() @@ -532,14 +539,22 @@ def test_pyscript_action_execution(): json={"success": True, "data": {"bot_response": {'numbers': [1, 2, 3, 4, 5], 'total': 15, 'i': 5}, "slots": {"location": "Bangalore", "langauge": "Kannada"}, "type": "json"}, "message": None, "error_code": 0}, - match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'chat_log': [], 'intent': 'pyscript_action', - 'kairon_user_msg': None, 'key_vault': {}, 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, - 'sender_id': 'default', 'session_started': None, - 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'langauge': 'Kannada', 'location': 'Bangalore'}, - 'user_message': 'get intents'} - - })] + match=[responses.matchers.json_params_matcher({'source_code': script, + 'predefined_objects': {'chat_log': [], + 'intent': 'pyscript_action', + 'kairon_user_msg': None, 'key_vault': {}, + 'latest_message': {'intent_ranking': [ + {'name': 'pyscript_action'}], + 'text': 'get intents'}, + 'sender_id': 'default', + 'session_started': None, + 'slot': { + 'bot': '5f50fd0a56b698ca10d35d2z', + 'langauge': 'Kannada', + 'location': 'Bangalore'}, + 'user_message': 'get intents'} + + })] ) request_object = { @@ -665,6 +680,7 @@ def test_pyscript_action_execution_with_multiple_utterances(): assert response_json['responses'][0]['custom'] == {'text': 'Hello!'} assert response_json['responses'][1]['text'] == 'How can I help you?' + @responses.activate def test_pyscript_action_execution_with_multiple_integer_utterances(): import textwrap @@ -778,7 +794,9 @@ def test_pyscript_action_execution_with_bot_response_none(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -853,7 +871,9 @@ def test_pyscript_action_execution_with_type_json_bot_response_none(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -928,7 +948,9 @@ def test_pyscript_action_execution_with_type_json_bot_response_str(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1005,7 +1027,9 @@ def test_pyscript_action_execution_with_other_type(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1080,7 +1104,9 @@ def test_pyscript_action_execution_with_slots_not_dict_type(): "slots": "invalid slots values"}, "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1175,7 +1201,7 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url(mock_trigger_l }, "version": "version" } - with patch("kairon.shared.utils.Utility.environment", new=mock_environment): + with mock.patch("kairon.shared.utils.Utility.environment", new=mock_environment): mock_trigger_lambda.return_value = \ {"Payload": {"body": {"bot_response": "Successfully Evaluated the pyscript", "slots": {"location": "Bangalore", "langauge": "Kannada"}}}, "StatusCode": 200} @@ -1185,15 +1211,17 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url(mock_trigger_l assert len(response_json['events']) == 3 assert len(response_json['responses']) == 1 assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'location', 'value': 'Bangalore'}, - {'event': 'slot', 'timestamp': None, 'name': 'langauge', 'value': 'Kannada'}, - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "Successfully Evaluated the pyscript"}] + {'event': 'slot', 'timestamp': None, 'name': 'location', 'value': 'Bangalore'}, + {'event': 'slot', 'timestamp': None, 'name': 'langauge', 'value': 'Kannada'}, + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "Successfully Evaluated the pyscript"}] assert response_json['responses'][0]['text'] == "Successfully Evaluated the pyscript" called_args = mock_trigger_lambda.call_args assert called_args.args[1] == \ {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {"bot": "5f50fd0a56b698ca10d35d2z", "location": "Bangalore", "langauge": "Kannada"}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, @@ -1252,7 +1280,7 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url_raise_exceptio }, "version": "version" } - with patch("kairon.shared.utils.Utility.environment", new=mock_environment): + with mock.patch("kairon.shared.utils.Utility.environment", new=mock_environment): mock_trigger_lambda.return_value = {"Payload": {"body": "Failed to evaluated the pyscript"}, "StatusCode": 422} response = client.post("/webhook", json=request_object) response_json = response.json() @@ -1260,8 +1288,8 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url_raise_exceptio assert len(response_json['events']) == 1 assert len(response_json['responses']) == 1 assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "I have failed to process your request"}] + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "I have failed to process your request"}] log = ActionServerLogs.objects(action=action_name).get().to_mongo().to_dict() assert log['exception'] == "Failed to evaluated the pyscript" @@ -1296,7 +1324,9 @@ def raise_custom_exception(request): "POST", Utility.environment['evaluator']['pyscript']['url'], callback=raise_custom_exception, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1307,7 +1337,7 @@ def raise_custom_exception(request): "tracker": { "sender_id": "default", "conversation_id": "default", - "slots": {"bot": "5f50fd0a56b698ca10d35d2z", "location": "Bangalore", "langauge": "Kannada"}, + "slots": {"bot": "5f50fd0a56b698ca10d35d2z", "location": "Bangalore", "langauge": "Kannada"}, "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'pyscript_action'}]}, "latest_event_time": 1537645578.314389, "followup_action": "action_listen", @@ -1367,7 +1397,9 @@ def test_pyscript_action_execution_with_invalid_response(): "error_code": 422}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1409,7 +1441,8 @@ def test_pyscript_action_execution_with_invalid_response(): {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': 'I have failed to process your request'}] log = ActionServerLogs.objects(action=action_name).get().to_mongo().to_dict() - assert log['exception'] == 'Pyscript evaluation failed: {\'success\': False, \'data\': None, \'message\': \'Script execution error: ("Line 2: SyntaxError: invalid syntax at statement: for i in 10",)\', \'error_code\': 422}' + assert log[ + 'exception'] == 'Pyscript evaluation failed: {\'success\': False, \'data\': None, \'message\': \'Script execution error: ("Line 2: SyntaxError: invalid syntax at statement: for i in 10",)\', \'error_code\': 422}' def test_http_action_execution(aioresponses): @@ -1509,8 +1542,42 @@ def test_http_action_execution(aioresponses): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'slots', 'data': [{'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, {'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', 'evaluation_type': 'expression', 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot_response_log': ['evaluation_type: expression', 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, {'type': 'api_call', 'headers': {'botid': '**********************2e', 'userid': '****', 'tag': '******ot', 'email': '*******************om'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot', 'name': 'udit', 'contact': ''}, 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'status_code': 200}, {'type': 'params_list', 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot', 'name': 'udit', 'contact': ''}, 'request_params': {'bot': '**********************2e', 'user': '1011', 'tag': '******ot', 'name': '****', 'contact': None}}, {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', 'expression: ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'response: red']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'slots', 'data': [ + {'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, + {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, + {'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + 'evaluation_type': 'expression', + 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot_response_log': ['evaluation_type: expression', + 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, + {'type': 'api_call', + 'headers': {'botid': '**********************2e', 'userid': '****', 'tag': '******ot', + 'email': '*******************om'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot', 'name': 'udit', + 'contact': ''}, + 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, + 'status_code': 200}, {'type': 'params_list', + 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', + 'tag': 'from_bot', 'name': 'udit', 'contact': ''}, + 'request_params': {'bot': '**********************2e', 'user': '1011', + 'tag': '******ot', 'name': '****', 'contact': None}}, + {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, + 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', + 'expression: ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', + 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'response: red']}] + assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} def test_http_action_execution_returns_custom_json(aioresponses): @@ -1973,8 +2040,41 @@ def test_http_action_execution_no_response_dispatch(aioresponses): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'slots', 'data': [{'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, {'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', 'evaluation_type': 'expression', 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot_response_log': ['evaluation_type: expression', 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot'}, 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'status_code': 200}, {'type': 'params_list', 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot'}, 'request_params': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': '******ot'}}, {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', 'expression: ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'response: red']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_no_response_dispatch', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'slots', 'data': [ + {'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, + {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, + {'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', + 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + 'evaluation_type': 'expression', + 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot_response_log': ['evaluation_type: expression', + 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, + {'type': 'api_call', + 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, + 'method': 'GET', 'url': 'http://localhost:8081/mock', + 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot'}, + 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, + 'status_code': 200}, {'type': 'params_list', + 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', + 'tag': 'from_bot'}, + 'request_params': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', + 'tag': '******ot'}}, + {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, + 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', + 'expression: ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', + 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'response: red']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_no_response_dispatch', 'sender': 'default', 'headers': {}, + 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2073,8 +2173,22 @@ def test_http_action_execution_script_evaluation(aioresponses): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'params_list', 'request_body': {}, 'request_params': {}}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {}, 'response': {'a': 10, 'b': { + 'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, + {'type': 'params_list', 'request_body': {}, 'request_params': {}}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation', 'sender': 'default', 'headers': {}, + 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2199,8 +2313,34 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_post(aiores if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'POST', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_post', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'POST', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'POST', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_post', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'POST', + 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', + 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2324,8 +2464,34 @@ def test_http_action_execution_script_evaluation_with_dynamic_params(aioresponse if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params', 'sender': 'default', + 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', + 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2450,8 +2616,31 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_returns_cus if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'json', 'data': 'bot_response = data', 'evaluation_type': 'script', 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'bot_response_log': ['evaluation_type: script', 'script: bot_response = data', "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_returns_custom_json', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': "{'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}", 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [ + {'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'json', 'data': 'bot_response = data', + 'evaluation_type': 'script', + 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, + 'bot_response_log': ['evaluation_type: script', 'script: bot_response = data', + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, + {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, + 'method': 'GET', 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, + 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_returns_custom_json', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': "{'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}", + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2576,8 +2765,34 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_no_response if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_no_response_dispatch', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_no_response_dispatch', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', + 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2835,7 +3050,7 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_and_params_ resp_msg = json.dumps(data_obj) aioresponses.add( method=responses.GET, - url=http_url+"?intent=test_run&sender_id=default&user_message=get+intents", + url=http_url + "?intent=test_run&sender_id=default&user_message=get+intents", body=resp_msg, status=200 ) @@ -2900,8 +3115,35 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_and_params_ if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert not DeepDiff(log, {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_and_params_list', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200}, ignore_order=True) + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert not DeepDiff(log, {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_and_params_list', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', + 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', + 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', + 'http_status_code': 200}, ignore_order=True) @responses.activate @@ -2941,7 +3183,7 @@ def test_http_action_execution_script_evaluation_failure_no_dispatch(aioresponse aioresponses.add( method=responses.GET, - url=http_url+"?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", + url=http_url + "?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", body=resp_msg, status=200 ) @@ -3039,7 +3281,7 @@ def test_http_action_execution_script_evaluation_failure_and_dispatch(aiorespons aioresponses.add( method=responses.GET, - url=http_url+"?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", + url=http_url + "?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", body=resp_msg, status=200, ) @@ -3198,12 +3440,12 @@ def test_http_action_execution_script_evaluation_failure_and_dispatch_2(aiorespo assert response_json['events'] == [ {"event": "slot", "timestamp": None, "name": "kairon_action_response", "value": "I have failed to process your request"}, - {"event": "slot", "timestamp": None, "name": "http_status_code", "value": 200},] + {"event": "slot", "timestamp": None, "name": "http_status_code", "value": 200}, ] assert response_json['responses'][0]['text'] == "I have failed to process your request" -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.http.ActionHTTP.retrieve_config") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.http.ActionHTTP.retrieve_config") @mock.patch("kairon.shared.rest_client.AioRestClient._AioRestClient__trigger", autospec=True) def test_http_action_failed_execution(mock_trigger_request, mock_action_config, mock_action): action_name = "test_run_with_get" @@ -3271,8 +3513,18 @@ def _get_action(*arge, **kwargs): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': 'The value of ${a.b.3} in ${a.b.d.0} is ${a.b.d}', 'evaluation_type': 'expression', 'exception': 'I have failed to process your request'}, {'type': 'api_call', 'headers': {}, 'method': 'GET', 'url': 'http://localhost:8800/mock', 'payload': {}, 'response': None, 'status_code': 408, 'exception': "Got non-200 status code:408 http_response:{'data': None, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 408}"}, {'type': 'params_list', 'request_body': {}, 'request_params': {}}, {'type': 'filled_slots'}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_run_with_get', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8800/mock', 'request_method': 'GET', 'bot_response': 'I have failed to process your request', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'FAILURE', 'fail_reason': 'Got non-200 status code:408 http_response:None', 'user_msg': 'get intents', 'time_elapsed': 0, 'http_status_code': 408} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': 'The value of ${a.b.3} in ${a.b.d.0} is ${a.b.d}', 'evaluation_type': 'expression', + 'exception': 'I have failed to process your request'}, + {'type': 'api_call', 'headers': {}, 'method': 'GET', 'url': 'http://localhost:8800/mock', + 'payload': {}, 'response': None, 'status_code': 408, + 'exception': "Got non-200 status code:408 http_response:{'data': None, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 408}"}, + {'type': 'params_list', 'request_body': {}, 'request_params': {}}, {'type': 'filled_slots'}] + assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_run_with_get', 'sender': 'default', + 'headers': {}, 'url': 'http://localhost:8800/mock', 'request_method': 'GET', + 'bot_response': 'I have failed to process your request', 'bot': '5f50fd0a56b698ca10d35d2e', + 'status': 'FAILURE', 'fail_reason': 'Got non-200 status code:408 http_response:None', + 'user_msg': 'get intents', 'time_elapsed': 0, 'http_status_code': 408} def test_http_action_missing_action_name(): @@ -3532,7 +3784,7 @@ def test_vectordb_action_execution_embedding_search_from_value(mock_embedding): BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value", bot="5f50fd0a56b698ca10d75d2e", user="user").save() embedding = list(np.random.random(Qdrant.__embedding__)) - mock_embedding.return_value = embedding + mock_embedding.return_value = {'data': [{'embedding': embedding}]} http_url = 'http://localhost:6333/collections/5f50fd0a56b698ca10d75d2e_test_vectordb_action_execution_faq_embd/points' resp_msg = json.dumps( @@ -3975,8 +4227,8 @@ def test_vectordb_action_execution_invalid_operation_type(): log.pop('timestamp') -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.database.ActionDatabase.retrieve_config") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.database.ActionDatabase.retrieve_config") def test_vectordb_action_failed_execution(mock_action_config, mock_action): action_name = "test_run_with_get_action" payload_body = {"ids": [0], "with_payload": True, "with_vector": True} @@ -3996,7 +4248,6 @@ def test_vectordb_action_failed_execution(mock_action_config, mock_action): BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value", bot="5f50fd0a56b697ca10d35d2e", user="user").save() - def _get_action_config(*arge, **kwargs): return action_config.to_mongo().to_dict(), bot_settings.to_mongo().to_dict() @@ -4166,9 +4417,9 @@ def _get_action(*arge, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "get_action") as mock_action: + with mock.patch.object(ActionUtility, "get_action") as mock_action: mock_action.side_effect = _get_action - with patch.object(ActionSetSlot, "retrieve_config") as mocked: + with mock.patch.object(ActionSetSlot, "retrieve_config") as mocked: mocked.side_effect = _get_action_config response = client.post("/webhook", json=request_object) response_json = response.json() @@ -4229,9 +4480,9 @@ def _get_action(*arge, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "get_action") as mock_action: + with mock.patch.object(ActionUtility, "get_action") as mock_action: mock_action.side_effect = _get_action - with patch.object(ActionSetSlot, "retrieve_config") as mocked: + with mock.patch.object(ActionSetSlot, "retrieve_config") as mocked: mocked.side_effect = _get_action_config response = client.post("/webhook", json=request_object) response_json = response.json() @@ -4291,9 +4542,9 @@ def _get_action(*arge, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "get_action") as mock_action: + with mock.patch.object(ActionUtility, "get_action") as mock_action: mock_action.side_effect = _get_action - with patch.object(ActionSetSlot, "retrieve_config") as mocked: + with mock.patch.object(ActionSetSlot, "retrieve_config") as mocked: mocked.side_effect = _get_action_config response = client.post("/webhook", json=request_object) response_json = response.json() @@ -5294,9 +5545,9 @@ def test_form_validation_action_with_is_required_true_and_semantics(): @responses.activate -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_script_evaluation(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['custom_text_mail'] = open('template/emails/custom_text_mail.html', 'rb').read().decode() @@ -5379,9 +5630,9 @@ def _get_action_config(*arge, **kwargs): assert str(args[2]).__contains__("Content-Type: text/html") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -5531,9 +5782,9 @@ def _get_action_config(*arge, **kwargs): assert str(args[2]).__contains__("Subject: default test") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_varied_utterances(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -5987,8 +6238,8 @@ def _get_action_config(*arge, **kwargs): assert logs.status == "SUCCESS" -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") def test_email_action_failed_execution(mock_action_config, mock_action): action_name = "test_run_email_action" action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") @@ -6374,7 +6625,7 @@ def _run_action(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6403,7 +6654,7 @@ def _run_action(*args, **kwargs): 'intent_ranking': [{'name': 'test_run'}], "entities": [{"value": "my custom text", "entity": KAIRON_USER_MSG_ENTITY}] } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6430,7 +6681,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["latest_message"] = { 'text': '/action_google_search', 'intent_ranking': [{'name': 'test_run'}] } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6469,7 +6720,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "what is Kanban?" - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6504,7 +6755,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "what is Kanban?" - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6541,7 +6792,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "what is Kanban?" - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6641,7 +6892,7 @@ def _run_action(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6744,7 +6995,7 @@ def _run_action(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -6942,7 +7193,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7183,7 +7434,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7302,7 +7553,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7313,9 +7564,9 @@ def _perform_web_search(*args, **kwargs): {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': 'Data science combines math, statistics, programming, analytics, AI, and machine learning to uncover insights from data. Learn how data science works, what it entails, and how it differs from data science and BI.\nTo know more, please visit: What is Data Science? | IBM\n\nData science is an interdisciplinary field that uses algorithms, procedures, and processes to examine large amounts of data in order to uncover hidden patterns, generate insights, and direct decision-making.\nTo know more, please visit: What Is Data Science? Definition, Examples, Jobs, and More'}], 'responses': [{ - 'text': 'Data science combines math, statistics, programming, analytics, AI, and machine learning to uncover insights from data. Learn how data science works, what it entails, and how it differs from data science and BI.\nTo know more, please visit: What is Data Science? | IBM\n\nData science is an interdisciplinary field that uses algorithms, procedures, and processes to examine large amounts of data in order to uncover hidden patterns, generate insights, and direct decision-making.\nTo know more, please visit: What Is Data Science? Definition, Examples, Jobs, and More', - 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, - 'image': None, 'attachment': None}]} + 'text': 'Data science combines math, statistics, programming, analytics, AI, and machine learning to uncover insights from data. Learn how data science works, what it entails, and how it differs from data science and BI.\nTo know more, please visit: What is Data Science? | IBM\n\nData science is an interdisciplinary field that uses algorithms, procedures, and processes to examine large amounts of data in order to uncover hidden patterns, generate insights, and direct decision-making.\nTo know more, please visit: What Is Data Science? Definition, Examples, Jobs, and More', + 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, + 'image': None, 'attachment': None}]} log = ActionServerLogs.objects(bot=bot, type=ActionType.web_search_action.value, status="SUCCESS").get() assert log['user_msg'] == '/action_public_search' @@ -7343,7 +7594,7 @@ def _perform_web_search(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "What is Python?" - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7446,7 +7697,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7550,7 +7801,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7573,7 +7824,7 @@ def test_process_jira_action(): def _mock_response(*args, **kwargs): return None - with patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_response): + with mock.patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_response): Actions(name=action_name, type=ActionType.jira_action.value, bot=bot, user=user).save() JiraAction( name=action_name, bot=bot, user=user, url='https://test-digite.atlassian.net', @@ -7660,7 +7911,7 @@ def _mock_response(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "create_jira_issue") as mocked: + with mock.patch.object(ActionUtility, "create_jira_issue") as mocked: mocked.side_effect = _mock_response response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7682,7 +7933,7 @@ def _mock_validation(*args, **kwargs): def _mock_response(*args, **kwargs): raise JIRAError(status_code=404, url='https://test-digite.atlassian.net') - with patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_validation): + with mock.patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_validation): Actions(name=action_name, type=ActionType.jira_action.value, bot=bot, user='test_user').save() JiraAction( name=action_name, bot=bot, user=user, url='https://test-digite.atlassian.net', @@ -7769,7 +8020,7 @@ def _mock_response(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "create_jira_issue") as mocked: + with mock.patch.object(ActionUtility, "create_jira_issue") as mocked: mocked.side_effect = _mock_response response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7965,7 +8216,7 @@ def test_process_zendesk_action(): user = 'test_user' Actions(name=action_name, type=ActionType.zendesk_action.value, bot=bot, user='test_user').save() - with patch('zenpy.Zenpy'): + with mock.patch('zenpy.Zenpy'): ZendeskAction(name=action_name, subdomain='digite751', user_name='udit.pandey@digite.com', api_token=CustomActionRequestParameters(value='1234567890'), subject='new ticket', response='ticket created', @@ -8050,7 +8301,7 @@ def test_process_zendesk_action(): "version": "version" } - with patch('zenpy.Zenpy'): + with mock.patch('zenpy.Zenpy'): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -8067,7 +8318,7 @@ def test_process_zendesk_action_failure(): user = 'test_user' Actions(name=action_name, type=ActionType.zendesk_action.value, bot=bot, user='test_user').save() - with patch('zenpy.Zenpy'): + with mock.patch('zenpy.Zenpy') as zen: ZendeskAction(name=action_name, subdomain='digite751', user_name='udit.pandey@digite.com', api_token=CustomActionRequestParameters(value='1234567890'), subject='new ticket', response='ticket created', @@ -8156,8 +8407,8 @@ def __mock_zendesk_error(*args, **kwargs): from zenpy.lib.exception import APIException raise APIException({"error": {"title": "No help desk at digite751.zendesk.com"}}) - with patch('zenpy.Zenpy') as mock: - mock.side_effect = __mock_zendesk_error + with mock.patch('zenpy.Zenpy') as zen: + zen.side_effect = __mock_zendesk_error response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -8263,7 +8514,7 @@ def test_process_pipedrive_leads_action(): user = 'test_user' Actions(name=action_name, type=ActionType.pipedrive_leads_action.value, bot=bot, user='test_user').save() - with patch('pipedrive.client.Client'): + with mock.patch('pipedrive.client.Client'): metadata = {'name': 'name', 'org_name': 'organization', 'email': 'email', 'phone': 'phone'} PipedriveLeadsAction(name=action_name, domain='https://digite751.pipedrive.com/', api_token=CustomActionRequestParameters(value='1234567890'), @@ -8362,10 +8613,10 @@ def __mock_create_leads(*args, **kwargs): def __mock_create_note(*args, **kwargs): return {"success": True, "data": {"id": 2}} - with patch('pipedrive.organizations.Organizations.create_organization', __mock_create_organization): - with patch('pipedrive.persons.Persons.create_person', __mock_create_person): - with patch('pipedrive.leads.Leads.create_lead', __mock_create_leads): - with patch('pipedrive.notes.Notes.create_note', __mock_create_note): + with mock.patch('pipedrive.organizations.Organizations.create_organization', __mock_create_organization): + with mock.patch('pipedrive.persons.Persons.create_person', __mock_create_person): + with mock.patch('pipedrive.leads.Leads.create_lead', __mock_create_leads): + with mock.patch('pipedrive.notes.Notes.create_note', __mock_create_note): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -8539,7 +8790,7 @@ def test_process_pipedrive_leads_action_failure(): user = 'test_user' Actions(name=action_name, type=ActionType.pipedrive_leads_action.value, bot=bot, user='test_user').save() - with patch('pipedrive.client.Client'): + with mock.patch('pipedrive.client.Client'): metadata = {'name': 'name', 'org_name': 'organization', 'email': 'email', 'phone': 'phone'} PipedriveLeadsAction(name=action_name, domain='https://digite751.pipedrive.com/', api_token=CustomActionRequestParameters(value='1234567890'), @@ -8629,7 +8880,7 @@ def __mock_pipedrive_error(*args, **kwargs): from pipedrive.exceptions import BadRequestError raise BadRequestError('Invalid request raised', {'error_code': 402}) - with patch('pipedrive.organizations.Organizations.create_organization', __mock_pipedrive_error): + with mock.patch('pipedrive.organizations.Organizations.create_organization', __mock_pipedrive_error): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -9074,7 +9325,7 @@ def _mock_search(*args, **kwargs): {"text": "yes", "payload": "yes"}]: yield result - with patch.object(MongoProcessor, "search_training_examples") as mock_action: + with mock.patch.object(MongoProcessor, "search_training_examples") as mock_action: mock_action.side_effect = _mock_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -9086,7 +9337,7 @@ def _mock_search(*args, **kwargs): for _ in []: yield - with patch.object(MongoProcessor, "search_training_examples") as mock_action: + with mock.patch.object(MongoProcessor, "search_training_examples") as mock_action: mock_action.side_effect = _mock_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -9977,7 +10228,7 @@ def __mock_error(*args, **kwargs): "e2e_actions": []}, "version": "2.8.15" } - with patch.object(ActionUtility, "trigger_rephrase") as mock_utils: + with mock.patch.object(ActionUtility, "trigger_rephrase") as mock_utils: mock_utils.side_effect = __mock_error response = client.post("/webhook", json=request_object) @@ -10166,7 +10417,7 @@ async def mock_process_actions(*args, **kwargs): from rasa_sdk import ActionExecutionRejection raise ActionExecutionRejection("Action Execution Rejection") - with patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): + with mock.patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'error': "Custom action 'Action Execution Rejection' rejected execution.", @@ -10176,18 +10427,17 @@ async def mock_process_actions(*args, **kwargs): from rasa_sdk.interfaces import ActionNotFoundException raise ActionNotFoundException("Action Not Found Exception") - with patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): + with mock.patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'error': "No registered action found for name 'Action Not Found Exception'.", 'action_name': 'Action Not Found Exception'} -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_prompt_question_from_slot(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_response_action_with_prompt_question_from_slot" @@ -10211,9 +10461,9 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10241,31 +10491,20 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[ - 3] == "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n" - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': []} - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action" @@ -10289,9 +10528,9 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10319,32 +10558,21 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[ - 3] == "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n" - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': []} - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_bot_responses_with_instructions(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_with_bot_responses_with_instructions" @@ -10369,9 +10597,9 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10400,31 +10628,20 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[ - 3] == "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n" - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': ['Answer in a short way.', 'Keep it simple.']} - - -@mock.patch.object(GPT3Resources, "invoke", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_response_action_with_query_prompt" @@ -10448,20 +10665,18 @@ def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embed 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}, {'name': 'Query Prompt', - 'data': 'If there is no specific query, assume that user is aking about java programming.', + 'data': 'If there is no specific query, assume that user is asking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True} ] - mock_completion_for_query_prompt = rephrased_query, { - 'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]} + mock_completion_for_query_prompt = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]} - mock_completion_for_answer = generated_text, { - 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion_for_answer = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_completion.side_effect = [mock_completion_for_query_prompt, mock_completion_for_answer] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10496,10 +10711,9 @@ def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embed 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Explain python is called high level programming language in laymen terms? \nA:"}] -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = GPT_LLM_FAQ @@ -10521,7 +10735,7 @@ def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): }, {'name': 'Data science prompt', 'instructions': 'Answer question based on the context above.', 'type': 'user', 'source': 'bot_content', - 'data': 'data_science'} + 'data': 'data_science'}, ] aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -10537,11 +10751,14 @@ def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): status=200, payload={ 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content_two}}]}) - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() - PromptAction(name=action_name, bot=bot, user=user, llm_prompts=llm_prompts).save() + PromptAction(name=action_name, + bot=bot, + user=user, + llm_prompts=llm_prompts).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() @@ -10561,11 +10778,10 @@ def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): ] -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_response_action_with_instructions(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = 'test_prompt_response_action_with_instructions' @@ -10586,9 +10802,9 @@ def test_prompt_response_action_with_instructions(mock_search, mock_embedding, m 'is_enabled': True } ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10612,11 +10828,10 @@ def test_prompt_response_action_with_instructions(mock_search, mock_embedding, m ] -@mock.patch.object(GPT3Resources, "invoke", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = GPT_LLM_FAQ @@ -10642,9 +10857,9 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text, generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10666,22 +10881,16 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', - 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above.\n \nQ: What kind of language is python? \nA:"}], - 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': True, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} - assert mock_completion.call_args.args[1] == 'chat/completions' - - -@patch("kairon.shared.llm.gpt3.openai.ChatCompletion.create", autospec=True) -@patch("kairon.shared.llm.gpt3.openai.Embedding.create", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) -def test_prompt_response_action_failure(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandeyy', 'bot': '5f50k90a56b698ca10d35d2e'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': True, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args.kwargs, expected, ignore_order=True) + + +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +def test_prompt_response_action_failure(mock_search): from uuid6 import uuid7 action_name = GPT_LLM_FAQ @@ -10690,10 +10899,7 @@ def test_prompt_response_action_failure(mock_search, mock_embedding, mock_comple user_msg = "What kind of language is python?" bot_content = "Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected." generated_text = "I don't know." - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = convert_to_openai_object(OpenAIResponse({'data': [{'embedding': embedding}]}, {})) - mock_completion.return_value = convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10764,13 +10970,10 @@ def test_prompt_action_response_action_does_not_exists(): assert len(response_json['responses']) == 0 -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_static_user_prompt(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -10799,8 +11002,7 @@ def test_prompt_action_response_action_with_static_user_prompt(mock_search, mock ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_search_cache(*args, **kwargs): return {'result': []} @@ -10813,9 +11015,9 @@ def __mock_cache_result(*args, **kwargs): mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.side_effect = [__mock_search_cache(), __mock_fetch_similar(), __mock_cache_result()] Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -10845,13 +11047,10 @@ def __mock_cache_result(*args, **kwargs): @responses.activate -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.llm.gpt3.GPT3FAQEmbedding.__collection_search__", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.llm.processor.LLMProcessor.__collection_search__", autospec=True) def test_prompt_action_response_action_with_action_prompt(mock_search, mock_embedding, mock_completion, aioresponses): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -10918,16 +11117,15 @@ def test_prompt_action_response_action_with_action_prompt(mock_search, mock_embe ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -10955,24 +11153,29 @@ def __mock_fetch_similar(*args, **kwargs): assert response_json['responses'] == [ {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}] - log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, status="SUCCESS").get() - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - with open('tests/testing_data/actions/action_prompt.txt', 'r') as file: - prompt_data = file.read() - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[3] == prompt_data - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, + status="SUCCESS").get().to_mongo().to_dict() + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) @mock.patch.object(ActionUtility, "perform_google_search", autospec=True) def test_kairon_faq_response_with_google_search_prompt(mock_google_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse - action_name = "kairon_faq_action" google_action_name = "custom_search_action" bot = "5u08kd0a56b698ca10hgjgjkhgjks" @@ -11013,12 +11216,11 @@ def _run_action(*args, **kwargs): PromptAction(name=action_name, bot=bot, user=user, llm_prompts=llm_prompts).save() def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - mock_completion.return_value = generated_text - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_google_search.side_effect = _run_action request_object = json.load(open("tests/testing_data/actions/action-request.json")) @@ -11035,12 +11237,24 @@ def mock_completion_for_answer(*args, **kwargs): 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}] - log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, status="SUCCESS").get() - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What is kanban' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - assert mock_completion.call_args.args[ - 3] == 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n' + log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, + status="SUCCESS").get().to_mongo().to_dict() + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Kanban is a workflow management tool which visualizes both the process (the workflow) and the actual work passing through that process.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], + 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) def test_prompt_response_action_with_action_not_found(): @@ -11072,13 +11286,10 @@ def test_prompt_response_action_with_action_not_found(): log['exception'] = 'No action found for given bot and name' -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_dispatch_response_disabled(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -11102,17 +11313,16 @@ def test_prompt_action_dispatch_response_disabled(mock_search, mock_embedding, m ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -11154,23 +11364,28 @@ def __mock_fetch_similar(*args, **kwargs): {'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.'} }, {'type': 'slots_to_fill', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']} ] - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What is the name of prompt?' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - with open('tests/testing_data/actions/slot_prompt.txt', 'r') as file: - prompt_data = file.read() - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[3] == prompt_data - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.actions.utils.ActionUtility.compose_response", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.compose_response", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_set_slots(mock_search, mock_slot_set, mock_mock_embedding, mock_completion): - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse - action_name = "kairon_faq_action" bot = "5u80fd0a56c908ca10d35d2sjhjhjhj" user = "udit.pandey" @@ -11193,11 +11408,10 @@ def test_prompt_action_set_slots(mock_search, mock_slot_set, mock_mock_embedding ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_completion.return_value = mock_completion_for_answer() - mock_completion.return_value = generated_text + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} log1 = ['Slot: api_type', 'evaluation_type: expression', f"data: {generated_text}", 'response: filter'] log2 = ['Slot: query', 'evaluation_type: expression', f"data: {generated_text}", 'response: {\"must\": [{\"key\": \"Date Added\", \"match\": {\"value\": 1673721000.0}}]}'] @@ -11246,26 +11460,38 @@ def mock_completion_for_answer(*args, **kwargs): assert events == [ {'type': 'llm_response', 'response': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', - 'llm_response_log': {'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}'}}, - {'type': 'slots_to_fill', 'data': {'api_type': 'filter', 'query': '{"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}'}, - 'slot_eval_log': ['initiating slot evaluation', 'Slot: api_type', 'Slot: api_type', 'evaluation_type: expression', + 'llm_response_log': { + 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}'}}, + {'type': 'slots_to_fill', + 'data': {'api_type': 'filter', 'query': '{"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}'}, + 'slot_eval_log': ['initiating slot evaluation', 'Slot: api_type', 'Slot: api_type', + 'evaluation_type: expression', 'data: {"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'response: filter', 'Slot: query', 'Slot: query', 'evaluation_type: expression', 'data: {"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'response: {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}']} ] - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == user_msg - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], + 'raw_completion_response': {'choices': [{'message': { + 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_slot_prompt(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -11289,17 +11515,16 @@ def test_prompt_action_response_action_slot_prompt(mock_search, mock_embedding, ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -11344,22 +11569,27 @@ def __mock_fetch_similar(*args, **kwargs): {'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.'} }, {'type': 'slots_to_fill', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']} ] - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What is the name of prompt?' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - with open('tests/testing_data/actions/slot_prompt.txt', 'r') as file: - prompt_data = file.read() - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[3] == prompt_data - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_user_message_in_slot(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -11379,17 +11609,16 @@ def test_prompt_action_user_message_in_slot(mock_search, mock_embedding, mock_co ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -11413,28 +11642,23 @@ def __mock_fetch_similar(*args, **kwargs): {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - assert mock_completion.call_args[0][1] == 'Kanban And Scrum Together?' - assert mock_completion.call_args[0][2] == 'You are a personal assistant.\n' - print(mock_completion.call_args[0][3]) - assert mock_completion.call_args[0][3] == """ -Instructions on how to use Similarity Prompt: -['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.'] -Answer question based on the context above, if answer is not in the context go check previous logs. -""" - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) -def test_prompt_action_response_action_when_similarity_is_empty(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from uuid6 import uuid7 + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Kanban And Scrum Together? \nA:"}], + 'metadata': {'user': user, 'bot': bot}, 'api_key': value, + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +def test_prompt_action_response_action_when_similarity_is_empty(mock_search, mock_embedding, mock_completion): action_name = "test_prompt_action_response_action_when_similarity_is_empty" bot = "5f50fd0a56b698ca10d35d2C" user = "udit.pandey" value = "keyvalue" user_msg = "What kind of language is python?" - bot_content = "Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected." generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." llm_prompts = [ {'name': 'System Prompt', @@ -11450,9 +11674,9 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = {'result': []} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -11480,29 +11704,20 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc 'response': None, 'image': None, 'attachment': None} ] - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert not mock_completion.call_args.args[3] - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': []} - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, + {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], + 'metadata': {'user': 'udit.pandey', 'bot': bot}, 'api_key': value, + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_when_similarity_disabled(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_response_action_when_similarity_disabled" @@ -11526,10 +11741,11 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc 'is_enabled': False} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text - mock_search.return_value = {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_search.return_value = { + 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() PromptAction(name=action_name, bot=bot, user=user, num_bot_responses=2, llm_prompts=llm_prompts).save() @@ -11555,17 +11771,11 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert not mock_completion.call_args.args[3] - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [], - 'instructions': []} \ No newline at end of file + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, + {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], + 'metadata': {'user': user, 'bot': bot}, 'api_key': value, + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/integration_test/chat_service_test.py b/tests/integration_test/chat_service_test.py index 45597ff76..77b828a71 100644 --- a/tests/integration_test/chat_service_test.py +++ b/tests/integration_test/chat_service_test.py @@ -5,14 +5,18 @@ from datetime import datetime, timedelta from unittest import mock from urllib.parse import urlencode, quote_plus - -from kairon.shared.live_agent.live_agent import LiveAgentHandler from kairon.shared.utils import Utility - os.environ["system_file"] = "./tests/testing_data/system.yaml" os.environ["ASYNC_TEST_TIMEOUT"] = "3600" Utility.load_environment() +Utility.load_system_metadata() + +from kairon.shared.live_agent.live_agent import LiveAgentHandler + + + + import pytest import responses from mock import patch diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 1dcfac008..1b4219e76 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -23,6 +23,9 @@ from pydantic import SecretStr from rasa.shared.utils.io import read_config_file from slack_sdk.web.slack_response import SlackResponse +from kairon.shared.utils import Utility, MailUtility + +Utility.load_system_metadata() from kairon.api.app.main import app from kairon.events.definitions.multilingual import MultilingualEvent @@ -69,7 +72,6 @@ from kairon.shared.multilingual.utils.translator import Translator from kairon.shared.organization.processor import OrgProcessor from kairon.shared.sso.clients.google import GoogleSSO -from kairon.shared.utils import Utility, MailUtility from urllib.parse import urlencode from deepdiff import DeepDiff @@ -4100,9 +4102,10 @@ def test_get_prompt_action(): assert actual["error_code"] == 0 assert not actual["message"] actual["data"][0].pop("_id") - assert actual["data"] == [ + assert not DeepDiff(actual["data"], [ {'name': 'test_update_prompt_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ @@ -4118,7 +4121,7 @@ def test_get_prompt_action(): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], 'dispatch_response': True, - 'status': True}] + 'status': True}], ignore_order=True) def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(monkeypatch): @@ -4191,10 +4194,11 @@ def _mock_get_bot_settings(*args, **kwargs): assert actual["error_code"] == 0 assert not actual["message"] actual["data"][1].pop("_id") - assert actual["data"][1] == { + assert not DeepDiff(actual["data"][1], { 'name': 'test_add_prompt_action_with_empty_collection_for_bot_content_prompt', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -4212,7 +4216,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], - 'dispatch_response': True, 'status': True} + 'dispatch_response': True, 'status': True}, ignore_order=True) def test_add_prompt_action_with_bot_content_prompt_with_payload(monkeypatch): @@ -4279,10 +4283,11 @@ def _mock_get_bot_settings(*args, **kwargs): ) actual = response.json() actual["data"][2].pop("_id") - assert actual["data"][2] == { + assert not DeepDiff(actual["data"][2], { 'name': 'test_add_prompt_action_with_bot_content_prompt_with_payload', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -4299,7 +4304,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], - 'set_slots': [], 'dispatch_response': True, 'status': True} + 'set_slots': [], 'dispatch_response': True, 'status': True}, ignore_order=True) assert actual["success"] assert actual["error_code"] == 0 assert not actual["message"] @@ -4370,10 +4375,11 @@ def _mock_get_bot_settings(*args, **kwargs): ) actual = response.json() actual["data"][3].pop("_id") - assert actual["data"][3] == { + assert not DeepDiff(actual["data"][3], { 'name': 'test_add_prompt_action_with_bot_content_prompt_with_content', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -4391,7 +4397,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], - 'dispatch_response': True, 'status': True} + 'dispatch_response': True, 'status': True}, ignore_order=True) assert actual["success"] assert actual["error_code"] == 0 assert not actual["message"] diff --git a/tests/unit_test/action/action_test.py b/tests/unit_test/action/action_test.py index 312d46927..715fecc12 100644 --- a/tests/unit_test/action/action_test.py +++ b/tests/unit_test/action/action_test.py @@ -6,6 +6,8 @@ import mock from googleapiclient.http import HttpRequest from pipedrive.exceptions import UnauthorizedError, BadRequestError +from kairon.shared.utils import Utility +Utility.load_system_metadata() from kairon.actions.definitions.email import ActionEmail from kairon.actions.definitions.factory import ActionFactory @@ -41,10 +43,10 @@ from kairon.actions.handlers.processor import ActionProcessor from kairon.shared.actions.utils import ActionUtility from kairon.shared.actions.exception import ActionFailure -from kairon.shared.utils import Utility from unittest.mock import patch from urllib.parse import urlencode + class TestActions: @pytest.fixture(autouse=True, scope='class') @@ -2658,6 +2660,7 @@ def test_get_prompt_action_config(self): 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', @@ -3949,6 +3952,7 @@ def test_get_prompt_action_config_2(self): 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'dispatch_response': True, 'set_slots': [], + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', diff --git a/tests/unit_test/api/api_processor_test.py b/tests/unit_test/api/api_processor_test.py index 34a46ad81..363272358 100644 --- a/tests/unit_test/api/api_processor_test.py +++ b/tests/unit_test/api/api_processor_test.py @@ -7,6 +7,8 @@ from unittest import mock from unittest.mock import patch from urllib.parse import urljoin +from kairon.shared.utils import Utility, MailUtility +Utility.load_system_metadata() import jwt import pytest @@ -22,7 +24,6 @@ from starlette.requests import Request from starlette.responses import RedirectResponse -from kairon.api.app.routers.idp import get_idp_config from kairon.api.models import RegisterAccount, EventConfig, IDPConfig, StoryRequest, HttpActionParameters, Password from kairon.exceptions import AppException from kairon.idp.data_objects import IdpConfig @@ -42,12 +43,6 @@ from kairon.shared.organization.processor import OrgProcessor from kairon.shared.sso.clients.facebook import FacebookSSO from kairon.shared.sso.clients.google import GoogleSSO -from kairon.shared.utils import Utility, MailUtility -from kairon.exceptions import AppException -import time -from kairon.idp.data_objects import IdpConfig -from kairon.api.models import RegisterAccount, EventConfig, IDPConfig, StoryRequest, HttpActionParameters, Password -from mongomock import MongoClient os.environ["system_file"] = "./tests/testing_data/system.yaml" diff --git a/tests/unit_test/augmentation/gpt_augmentation_test.py b/tests/unit_test/augmentation/gpt_augmentation_test.py index dfdb7d42e..e743fb49c 100644 --- a/tests/unit_test/augmentation/gpt_augmentation_test.py +++ b/tests/unit_test/augmentation/gpt_augmentation_test.py @@ -1,7 +1,7 @@ from augmentation.paraphrase.gpt3.generator import GPT3ParaphraseGenerator from augmentation.paraphrase.gpt3.models import GPTRequest from augmentation.paraphrase.gpt3.gpt import GPT -import openai +from openai.resources.completions import Completions import pytest import responses @@ -61,7 +61,7 @@ def test_questions_set_generation(monkeypatch): def test_generate_questions(monkeypatch): - monkeypatch.setattr(openai.Completion, 'create', mock_create) + monkeypatch.setattr(Completions, 'create', mock_create) request_data = GPTRequest(api_key="MockKey", data=["Are there any more test questions?"], num_responses=2) @@ -73,7 +73,7 @@ def test_generate_questions(monkeypatch): def test_generate_questions_empty_api_key(monkeypatch): - monkeypatch.setattr(openai.Completion, 'create', mock_create) + monkeypatch.setattr(Completions, 'create', mock_create) request_data = GPTRequest(api_key="", data=["Are there any more test questions?"], num_responses=2) @@ -84,7 +84,7 @@ def test_generate_questions_empty_api_key(monkeypatch): def test_generate_questions_empty_data(monkeypatch): - monkeypatch.setattr(openai.Completion, 'create', mock_create) + monkeypatch.setattr(Completions, 'create', mock_create) request_data = GPTRequest(api_key="MockKey", data=[], num_responses=2) @@ -125,6 +125,6 @@ def test_generate_questions_invalid_api_key(): data=["Are there any more test questions?"], num_responses=2) gpt3_generator = GPT3ParaphraseGenerator(request_data=request_data) - with pytest.raises(APIError, match=r'.*Incorrect API key provided: InvalidKey. You can find your API key at https://beta.openai.com..*'): + with pytest.raises(APIError, match=r'.*Incorrect API key provided: InvalidKey. You can find your API key at https://platform.openai.com/account/..*'): gpt3_generator.paraphrases() diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index e3f313c8a..7d4a9ad6e 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -8,6 +8,11 @@ from datetime import datetime, timedelta, timezone from io import BytesIO from typing import List +from kairon.shared.utils import Utility +os.environ["system_file"] = "./tests/testing_data/system.yaml" +Utility.load_environment() +Utility.load_system_metadata() + from mock import patch import numpy as np @@ -83,12 +88,9 @@ from kairon.shared.multilingual.processor import MultilingualLogProcessor from kairon.shared.test.data_objects import ModelTestingLogs from kairon.shared.test.processor import ModelTestingLogProcessor -from kairon.shared.utils import Utility from kairon.train import train_model_for_bot, start_training - -os.environ["system_file"] = "./tests/testing_data/system.yaml" -Utility.load_environment() from deepdiff import DeepDiff +import litellm class TestMongoProcessor: @@ -168,7 +170,7 @@ def test_add_prompt_action_with_invalid_slots(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_slots', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -196,7 +198,7 @@ def test_add_prompt_action_with_invalid_http_action(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_http_action', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -225,7 +227,7 @@ def test_add_prompt_action_with_invalid_similarity_threshold(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_prompt_action_similarity', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -255,7 +257,7 @@ def test_add_prompt_action_with_invalid_top_results(self): user = 'test_user' request = {'name': 'test_prompt_action_invalid_top_results', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -303,7 +305,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): processor.add_prompt_action(request, bot, user) prompt_action = processor.get_prompt_action(bot) prompt_action[0].pop("_id") - assert prompt_action == [ + assert not DeepDiff(prompt_action, [ {'name': 'test_add_prompt_action_with_empty_collection_for_bot_content_prompt', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", @@ -311,6 +313,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'data': 'default', @@ -321,7 +324,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): 'is_enabled': True}, {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True}] + 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True}], ignore_order=True) def test_add_prompt_action_with_bot_content_prompt(self): processor = MongoProcessor() @@ -347,8 +350,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): processor.add_prompt_action(request, bot, user) prompt_action = processor.get_prompt_action(bot) prompt_action[1].pop("_id") - print(prompt_action) - assert prompt_action[1] == { + assert not DeepDiff(prompt_action[1], { 'name': 'test_add_prompt_action_with_bot_content_prompt', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", @@ -356,6 +358,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'data': 'Bot_collection', @@ -366,7 +369,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): 'is_enabled': True}, {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True} + 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True}, ignore_order=True) def test_add_prompt_action_with_invalid_query_prompt(self): processor = MongoProcessor() @@ -551,7 +554,7 @@ def test_add_prompt_action_with_empty_llm_prompts(self): user = 'test_user' request = {'name': 'test_add_prompt_action_with_empty_llm_prompts', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -576,13 +579,13 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) pytest.action_id = processor.add_prompt_action(request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_add_prompt_action_faq_action_with_default_values', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_add_prompt_action_faq_action_with_default_values', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -590,7 +593,7 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [{'name': 'gpt_result', 'value': '${data}', 'evaluation_type': 'expression'}, {'name': 'gpt_result_type', 'value': '${data.type}', - 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}] + 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}], ignore_order=True) def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): processor = MongoProcessor() @@ -599,14 +602,14 @@ def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_temperature_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="Temperature must be between 0.0 and 2.0!"): + with pytest.raises(ValidationError, match=re.escape("['temperature']: 3.0 is greater than the maximum of 2.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_stop_hyperparameter(self): @@ -616,7 +619,7 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_stop_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': ["\n", ".", "?", "!", ";"], 'presence_penalty': 0.0, @@ -625,7 +628,7 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} with pytest.raises(ValidationError, - match="Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers."): + match=re.escape('[\'stop\']: ["\\n",".","?","!",";"] is not valid under any of the schemas listed in the \'anyOf\' keyword')): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): @@ -636,14 +639,14 @@ def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): request = {'name': 'test_add_prompt_action_with_invalid_presence_penalty_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': -3.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="Presence penalty must be between -2.0 and 2.0!"): + with pytest.raises(ValidationError, match=re.escape("['presence_penalty']: -3.0 is less than the minimum of -2.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): @@ -654,14 +657,14 @@ def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): request = {'name': 'test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 3.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="Frequency penalty must be between -2.0 and 2.0!"): + with pytest.raises(ValidationError, match=re.escape("['frequency_penalty']: 3.0 is greater than the maximum of 2.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): @@ -671,14 +674,14 @@ def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_max_tokens_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="max_tokens must be between 5 and 4096 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['max_tokens']: 2 is less than the minimum of 5")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): @@ -688,14 +691,14 @@ def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_zero_max_tokens_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="max_tokens must be between 5 and 4096 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['max_tokens']: 0 is less than the minimum of 5")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): @@ -705,14 +708,14 @@ def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_top_p_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt-3.5-turbo', 'top_p': 3.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="top_p must be between 0.0 and 1.0!"): + with pytest.raises(ValidationError, match=re.escape("['top_p']: 3.0 is greater than the maximum of 1.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_n_hyperparameter(self): @@ -722,14 +725,14 @@ def test_add_prompt_action_with_invalid_n_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_n_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 7, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="n must be between 1 and 5 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['n']: 7 is greater than the maximum of 5")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_zero_n_hyperparameter(self): @@ -739,14 +742,14 @@ def test_add_prompt_action_with_zero_n_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_zero_n_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 0, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="n must be between 1 and 5 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['n']: 0 is less than the minimum of 1")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): @@ -756,14 +759,14 @@ def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_logit_bias_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 2, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': 'a'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="logit_bias must be a dictionary!"): + with pytest.raises(ValidationError, match=re.escape('[\'logit_bias\']: "a" is not of type "object"')): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_faq_action_already_exist(self): @@ -822,7 +825,7 @@ def test_edit_prompt_action_faq_action(self): 'source': 'static', 'is_enabled': True}], "failure_message": "updated_failure_message", "use_query_prompt": True, "use_bot_responses": True, "query_prompt": "updated_query_prompt", - "num_bot_responses": 5, "hyperparameters": Utility.get_llm_hyperparameters(), + "num_bot_responses": 5, "hyperparameters": Utility.get_llm_hyperparameters('openai'), "set_slots": [{"name": "gpt_result", "value": "${data}", "evaluation_type": "expression"}, {"name": "gpt_result_type", "value": "${data.type}", "evaluation_type": "script"}], "dispatch_response": False @@ -830,12 +833,12 @@ def test_edit_prompt_action_faq_action(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -853,7 +856,8 @@ def test_edit_prompt_action_faq_action(self): 'is_enabled': True}], 'instructions': [], 'set_slots': [{'name': 'gpt_result', 'value': '${data}', 'evaluation_type': 'expression'}, {'name': 'gpt_result_type', 'value': '${data.type}', - 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}] + 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}], + ignore_order=True) request = {'name': 'test_edit_prompt_action_faq_action_again', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -862,10 +866,10 @@ def test_edit_prompt_action_faq_action(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_faq_action_again', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action_again', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -873,7 +877,7 @@ def test_edit_prompt_action_faq_action(self): {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], - 'dispatch_response': True, 'status': True}] + 'dispatch_response': True, 'status': True}], ignore_order=True) def test_edit_prompt_action_with_less_hyperparameters(self): processor = MongoProcessor() @@ -906,13 +910,13 @@ def test_edit_prompt_action_with_less_hyperparameters(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -928,7 +932,7 @@ def test_edit_prompt_action_with_less_hyperparameters(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': [], 'set_slots': [], 'dispatch_response': True, - 'status': True}] + 'status': True}], ignore_order=True) def test_get_prompt_action_does_not_exist(self): processor = MongoProcessor() @@ -941,13 +945,13 @@ def test_get_prompt_faq_action(self): bot = 'test_bot' action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -963,8 +967,7 @@ def test_get_prompt_faq_action(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': [], 'set_slots': [], 'dispatch_response': True, - 'status': True}] - + 'status': True}], ignore_order=True) def test_delete_prompt_action(self): processor = MongoProcessor() bot = 'test_bot' @@ -2629,7 +2632,7 @@ def test_start_training_fail(self): assert model_training.__len__() == 1 assert model_training.first().exception in str("Training data does not exists!") - @patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @patch.object(litellm, "aembedding", autospec=True) @patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) @patch("kairon.shared.account.processor.AccountProcessor.get_bot", autospec=True) @patch("kairon.train.train_model_for_bot", autospec=True) @@ -2650,8 +2653,8 @@ def test_start_training_with_llm_faq( settings = BotSettings.objects(bot=bot).get() settings.llm_settings = LLMSettings(enable_faq=True) settings.save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_openai.return_value = embedding + embedding = list(np.random.random(1532)) + mock_openai.return_value = {'data': [{'embedding': embedding}]} mock_bot.return_value = {"account": 1} mock_train.return_value = f"/models/{bot}" start_training(bot, user) diff --git a/tests/unit_test/events/events_test.py b/tests/unit_test/events/events_test.py index 98ae5ff10..4f7a63438 100644 --- a/tests/unit_test/events/events_test.py +++ b/tests/unit_test/events/events_test.py @@ -17,15 +17,15 @@ from rasa.shared.constants import DEFAULT_DOMAIN_PATH, DEFAULT_DATA_PATH, DEFAULT_CONFIG_PATH from rasa.shared.importers.rasa import RasaFileImporter from responses import matchers +from kairon.shared.utils import Utility + +Utility.load_system_metadata() from kairon.shared.channels.broadcast.whatsapp import WhatsappBroadcast from kairon.shared.chat.data_objects import ChannelLogs os.environ["system_file"] = "./tests/testing_data/system.yaml" -from kairon.events.definitions.message_broadcast import MessageBroadcastEvent - -from kairon.shared.chat.broadcast.processor import MessageBroadcastProcessor from kairon.events.definitions.data_importer import TrainingDataImporterEvent from kairon.events.definitions.faq_importer import FaqDataImporterEvent from kairon.events.definitions.history_delete import DeleteHistoryEvent @@ -42,7 +42,6 @@ from kairon.shared.data.processor import MongoProcessor from kairon.shared.importer.processor import DataImporterLogProcessor from kairon.shared.test.processor import ModelTestingLogProcessor -from kairon.shared.utils import Utility from kairon.test.test_models import ModelTester os.environ["system_file"] = "./tests/testing_data/system.yaml" @@ -2020,7 +2019,7 @@ def test_execute_message_broadcast_with_pyscript_failure(self, mock_is_exist, mo bot = 'test_execute_message_broadcast_with_pyscript_failure' user = 'test_user' script = """ - import time + import os """ script = textwrap.dedent(script) config = { @@ -2057,7 +2056,7 @@ def test_execute_message_broadcast_with_pyscript_failure(self, mock_is_exist, mo logs[0][0].pop("timestamp", None) assert logs[0][0] == {"event_id": event_id, 'reference_id': reference_id, 'log_type': 'common', 'bot': bot, 'status': 'Fail', - 'user': user, "exception": "Script execution error: import of 'time' is unauthorized"} + 'user': user, "exception": "Script execution error: import of 'os' is unauthorized"} with pytest.raises(AppException, match="Notification settings not found!"): MessageBroadcastProcessor.get_settings(event_id, bot) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 7494e1c0a..c8f727db3 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -7,16 +7,17 @@ import ujson as json from aiohttp import ClientConnectionError from mongoengine import connect +from kairon.shared.utils import Utility +Utility.load_system_metadata() from kairon.exceptions import AppException from kairon.shared.admin.constants import BotSecretType from kairon.shared.admin.data_objects import BotSecrets from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT -from kairon.shared.data.data_objects import LLMSettings -from kairon.shared.llm.factory import LLMFactory -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding, LLMBase -from kairon.shared.utils import Utility +from kairon.shared.llm.processor import LLMProcessor +import litellm +from deepdiff import DeepDiff class TestLLM: @@ -26,36 +27,9 @@ def init_connection(self): Utility.load_environment() connect(**Utility.mongoengine_connection(Utility.environment['database']["url"])) - def test_llm_base_train(self): - with pytest.raises(Exception): - base = LLMBase("Test") - base.train() - - def test_llm_base_predict(self): - with pytest.raises(Exception): - base = LLMBase('Test') - base.predict("Sample") - - def test_llm_factory_invalid_type(self): - with pytest.raises(Exception): - LLMFactory.get_instance("sample")("test", LLMSettings(provider="openai").to_mongo().to_dict()) - - def test_llm_factory_faq_type(self): - BotSecrets(secret_type=BotSecretType.gpt_key.value, value='value', bot='test', user='test').save() - inst = LLMFactory.get_instance("faq")("test", LLMSettings(provider="openai").to_mongo().to_dict()) - assert isinstance(inst, GPT3FAQEmbedding) - assert inst.db_url == Utility.environment['vector']['db'] - assert inst.headers == {} - - def test_llm_factory_faq_type_set_vector_key(self): - with mock.patch.dict(Utility.environment, {'vector': {"db": "http://test:6333", 'key': 'test'}}): - inst = LLMFactory.get_instance("faq")("test", LLMSettings(provider="openai").to_mongo().to_dict()) - assert isinstance(inst, GPT3FAQEmbedding) - assert inst.db_url == Utility.environment['vector']['db'] - assert inst.headers == {'api-key': Utility.environment['vector']['key']} - @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): bot = "test_embed_faq" user = "test" value = "nupurkhare" @@ -64,19 +38,11 @@ async def test_gpt3_faq_embedding_train(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} - + embedding = list(np.random.random(LLMProcessor.__embedding__)) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -100,23 +66,26 @@ async def test_gpt3_faq_embedding_train(self, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train() + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": test_content.data} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { 'points': [{'id': test_content.vector_id, 'vector': embedding, - 'payload': {"collection_name": f"{gpt3.bot}{gpt3.suffix}", 'content': test_content.data} + 'payload': {'content': test_content.data} }]} + expected = {"model": "text-embedding-3-small", + "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aioresponses): bot = "test_embed_faq_text" user = "test" value = "nupurkhare" @@ -149,18 +118,9 @@ async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]}, - repeat=True - ) - - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + embedding = list(np.random.random(LLMProcessor.__embedding__)) + mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]} + gpt3 = LLMProcessor(bot) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -205,39 +165,36 @@ async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train() + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 3 assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'name': f"{gpt3.bot}_country_details{gpt3.suffix}", 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": '{"country":"Spain","lang":"spanish"}'} - assert list(aioresponses.requests.values())[3][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][1].kwargs['json'] == {"model": "text-embedding-3-small", - 'input': '{"lang":"spanish","role":"ds"}'} - assert list(aioresponses.requests.values())[3][1].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][2].kwargs['json'] == {"model": "text-embedding-3-small", - "input": '{"name":"Nupur","city":"Pune"}'} - assert list(aioresponses.requests.values())[3][2].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[4][0].kwargs['json'] == {'points': [{'id': test_content_two.vector_id, + assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content_two.vector_id, 'vector': embedding, - 'payload': {'collection_name': f"{gpt3.bot}_country_details{gpt3.suffix}", - 'country': 'Spain'}}]} - assert list(aioresponses.requests.values())[4][1].kwargs['json'] == {'points': [{'id': test_content_three.vector_id, + 'payload': {'country': 'Spain'}}]} + assert list(aioresponses.requests.values())[3][1].kwargs['json'] == {'points': [{'id': test_content_three.vector_id, 'vector': embedding, - 'payload': {'collection_name': f"{gpt3.bot}_country_details{gpt3.suffix}", 'role': 'ds'}}]} + 'payload': {'role': 'ds'}}]} - assert list(aioresponses.requests.values())[5][0].kwargs['json'] == {'name': f"{gpt3.bot}_user_details{gpt3.suffix}", + assert list(aioresponses.requests.values())[4][0].kwargs['json'] == {'name': f"{gpt3.bot}_user_details{gpt3.suffix}", 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[6][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + assert list(aioresponses.requests.values())[5][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, 'vector': embedding, - 'payload': {'collection_name': f"{gpt3.bot}_user_details{gpt3.suffix}", - 'name': 'Nupur'}}]} + 'payload': {'name': 'Nupur'}}]} assert response['faq'] == 3 + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + print(mock_embedding.call_args) + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_payload_with_int(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, aioresponses): bot = "test_embed_faq_json" user = "test" value = "nupurkhare" @@ -254,17 +211,11 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} + embedding = list(np.random.random(LLMProcessor.__embedding__)) input = {"name": "Ram", "color": "red"} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), method="PUT", @@ -282,21 +233,25 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, aioresponses): ) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - response = await gpt3.train() + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_embed_faq_json_payload_with_int_faq_embd', 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": json.dumps(input)} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, 'vector': embedding, - 'payload': {'name': 'Ram', 'age': 23, 'color': 'red', "collection_name": "test_embed_faq_json_payload_with_int_faq_embd"} + 'payload': {'name': 'Ram', 'age': 23, 'color': 'red'} }]} + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_int(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): bot = "test_int" user = "test" value = "nupurkhare" @@ -313,18 +268,11 @@ async def test_gpt3_faq_embedding_train_int(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} + embedding = list(np.random.random(LLMProcessor.__embedding__)) input = {"name": "Ram", "color": "red"} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - + mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -354,28 +302,32 @@ async def test_gpt3_faq_embedding_train_int(self, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train() + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_int_embd_int_faq_embd', 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": json.dumps(input)} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header expected_payload = test_content.data - expected_payload['collection_name'] = 'test_int_embd_int_faq_embd' - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + #expected_payload['collection_name'] = 'test_int_embd_int_faq_embd' + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { 'points': [{'id': test_content.vector_id, 'vector': embedding, 'payload': expected_payload }]} + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + def test_gpt3_faq_embedding_train_failure(self): with pytest.raises(AppException, match=f"Bot secret '{BotSecretType.gpt_key.value}' not configured!"): - GPT3FAQEmbedding('test_failure', LLMSettings(provider="openai").to_mongo().to_dict()) + LLMProcessor('test_failure') @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_upsert_error(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aioresponses): bot = "test_embed_faq_not_exists" user = "test" value = "nupurk" @@ -384,19 +336,12 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - - request_header = {"Authorization": "Bearer nupurk"} + embedding = list(np.random.random(LLMProcessor.__embedding__)) - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -423,17 +368,21 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, aioresponses): ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train() + await gpt3.train(user=user, bot=bot) assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": test_content.data} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, 'payload': {'collection_name': f"{bot}{gpt3.suffix}",'content': test_content.data}}]} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + 'vector': embedding, 'payload': {'content': test_content.data}}]} + expected = {"model": "text-embedding-3-small", + "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_embedding, aioresponses): bot = "payload_upsert_error" user = "test" value = "nupurk" @@ -450,19 +399,11 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, aiorespo bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - - request_header = {"Authorization": "Bearer nupurk"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + embedding = list(np.random.random(LLMProcessor.__embedding__)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -489,21 +430,27 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, aiorespo ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train() + await gpt3.train(user=user, bot=bot) assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": json.dumps(test_content.data)} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header expected_payload = test_content.data - expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + #expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, 'vector': embedding, 'payload': expected_payload }]} + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict" user = "test" @@ -516,6 +463,7 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", @@ -523,8 +471,9 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}]} - hyperparameters = Utility.get_llm_hyperparameters() + 'collection': 'python'}], + "hyperparameters": hyperparameters + } mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, @@ -532,24 +481,10 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), @@ -558,24 +493,29 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response['content'] == generated_text - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = value + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_default_collection(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict_with_default_collection" user = "test" @@ -588,15 +528,17 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, aiorespo generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" - + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'default'}]} - hyperparameters = Utility.get_llm_hyperparameters() + 'collection': 'default'}], + 'hyperparameters': hyperparameters + } + mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, @@ -604,24 +546,10 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, aiorespo 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -631,40 +559,52 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, aiorespo {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) assert response['content'] == generated_text - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = value + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_embed_faq_predict", user="test").save() + collection='python', bot="test_gpt3_faq_embedding_predict_with_values", user="test").save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() + key = 'test' + user = "tests" + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}]} + 'collection': 'python'}], + "hyperparameters": hyperparameters + } - hyperparameters = Utility.get_llm_hyperparameters() mock_completion_request = {"messages": [ {"role": "system", "content": "You are a personal assistant. Answer the question according to the below context"}, @@ -672,24 +612,11 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), @@ -698,7 +625,7 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=gpt3.bot, **k_faq_action_config) assert response['content'] == generated_text assert gpt3.logs == [ {'messages': [{'role': 'system', @@ -714,25 +641,36 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': gpt3.bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "test_gpt3_faq_embedding_predict_with_values_with_instructions" + key = 'test' test_content = CognitionData( data="Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ", - collection='java', bot="test_embed_faq_predict", user="test").save() - + collection='java', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", @@ -740,9 +678,10 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, ai 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', "collection": "java"}], - 'instructions': ['Answer in a short way.', 'Keep it simple.']} + 'instructions': ['Answer in a short way.', 'Keep it simple.'], + "hyperparameters": hyperparameters + } - hyperparameters = Utility.get_llm_hyperparameters() mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, @@ -750,186 +689,210 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, ai 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url="https://api.openai.com/v1/chat/completions", + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + payload={'result': [ + {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) - - aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), - method="POST", - payload={'result': [ - {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} - ) - - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response['content'] == generated_text - assert gpt3.logs == [ - {'messages': [{'role': 'system', - 'content': 'You are a personal assistant. Answer the question according to the below context'}, - {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': { - 'choices': [{'message': {'content': 'Python is dynamically typed, garbage-collected, ' - 'high level, general purpose programming.', - 'role': 'assistant'}}]}, - 'type': 'answer_query', - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}}] - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) + assert response['content'] == generated_text + assert gpt3.logs == [ + {'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': { + 'choices': [{'message': {'content': 'Python is dynamically typed, garbage-collected, ' + 'high level, general purpose programming.', + 'role': 'assistant'}}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + embedding = list(np.random.random(LLMProcessor.__embedding__)) + bot = "test_gpt3_faq_embedding_predict_completion_connection_error" + user = 'test' + key = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_embed_faq_predict", user="test").save() + collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() + hyperparameters = Utility.get_default_llm_hyperparameters() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}]} + "collection": 'python'}], + "hyperparameters": hyperparameters + } def __mock_connection_error(*args, **kwargs): - import openai - - raise openai.error.APIConnectionError("Connection reset by peer!") + raise Exception("Connection reset by peer!") - mock_embedding.return_value = embedding + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.side_effect = __mock_connection_error - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) - - aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), - method="POST", - payload={'result': [ - {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} - ) + gpt3 = LLMProcessor(test_content.bot) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - print(mock_completion.call_args.args[3]) + aioresponses.add( + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + method="POST", + payload={'result': [ + {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} + ) - assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} - assert mock_embedding.call_args.args[1] == query + assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == 'You are a personal assistant. Answer the question according to the below context' - assert mock_completion.call_args.args[3] == """Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n""" - assert mock_completion.call_args.kwargs == {'similarity_prompt': [ - {'top_results': 10, 'similarity_threshold': 0.7, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', 'collection': 'python'}]} - assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error'}, + 'api_key': 'test', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio @mock.patch("kairon.shared.rest_client.AioRestClient._AioRestClient__trigger", autospec=True) - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock_llm_request): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user ="test" + bot = "test_gpt3_faq_embedding_predict_exact_match" + key = 'test' test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_embed_faq_predict", user="test").save() + collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}]} + "collection": 'python'}], + "hyperparameters": hyperparameters + } - mock_embedding.return_value = embedding + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_llm_request.side_effect = ClientConnectionError() - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} + response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_exact_match", **k_faq_action_config) + assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} - assert mock_embedding.call_args.args[1] == query - assert gpt3.logs == [] - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_embedding): - import openai - - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "test_gpt3_faq_embedding_predict_embedding_connection_error" + key = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - bot="test_embed_faq_predict", user="test").save() + bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() + hyperparameters = Utility.get_default_llm_hyperparameters() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", - "context_prompt": "Based on below context answer question, if answer not in context check previous logs."} + "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", + "hyperparameters": hyperparameters + } + mock_embedding.side_effect = [Exception("Connection reset by peer!"), {'data': [{'embedding': embedding}]}] - mock_embedding.side_effect = [openai.error.APIConnectionError("Connection reset by peer!"), embedding] + gpt3 = LLMProcessor(test_content.bot) + mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_embedding_connection_error", **k_faq_action_config) + assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} + assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - assert mock_embedding.call_args.args[1] == query - assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) - bot = "test_embed_faq_predict" + bot = "test_gpt3_faq_embedding_predict_with_previous_bot_responses" user = "test" + key = "test" + hyperparameters = Utility.get_default_llm_hyperparameters() test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" @@ -940,9 +903,10 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior ], "similarity_prompt": [{'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}] + "collection": 'python'}], + "hyperparameters": hyperparameters } - hyperparameters = Utility.get_llm_hyperparameters() + mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below'}, {'role': 'user', 'content': 'hello'}, @@ -951,23 +915,10 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior 'content': "Answer question based on the context below, if answer is not in the context go check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), @@ -976,44 +927,55 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - print(list(aioresponses.requests.values())[2][0].kwargs['json']) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) assert response['content'] == generated_text - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) - bot = "test_embed_faq_predict" + bot = "test_gpt3_faq_embedding_predict_with_query_prompt" user = "test" + key = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" rephrased_query = "Explain python is called high level programming language in laymen terms?" + hyperparameters = Utility.get_default_llm_hyperparameters() + k_faq_action_config = {"query_prompt": { "query_prompt": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", "use_query_prompt": True}, - "similarity_prompt": [ - {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}] - } - hyperparameters = Utility.get_llm_hyperparameters() + "similarity_prompt": [ + {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], + "hyperparameters": hyperparameters + } + mock_rephrase_request = {"messages": [ {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, @@ -1029,31 +991,11 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): ]} mock_rephrase_request.update(hyperparameters) mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]} - ) - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, - repeat=True - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), @@ -1062,18 +1004,21 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - print(list(aioresponses.requests.values())[2][1].kwargs['json']) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) assert response['content'] == generated_text - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_rephrase_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[2][1].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][1].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/unit_test/utility_test.py b/tests/unit_test/utility_test.py index 5910f13bb..8d4c784c5 100644 --- a/tests/unit_test/utility_test.py +++ b/tests/unit_test/utility_test.py @@ -8,6 +8,9 @@ from io import BytesIO from unittest.mock import patch, MagicMock from urllib.parse import urlencode +from kairon.shared.utils import Utility, MailUtility + +Utility.load_system_metadata() import numpy as np import pandas as pd @@ -36,12 +39,7 @@ from kairon.shared.data.data_objects import EventConfig, Slots, LLMSettings from kairon.shared.data.processor import MongoProcessor from kairon.shared.data.utils import DataUtility -from kairon.shared.llm.clients.azure import AzureGPT3Resources -from kairon.shared.llm.clients.factory import LLMClientFactory -from kairon.shared.llm.clients.gpt3 import GPT3Resources -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from kairon.shared.models import TemplateType -from kairon.shared.utils import Utility, MailUtility from kairon.shared.verification.email import QuickEmailVerification @@ -2111,7 +2109,7 @@ def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, user=user, ws_url="http://localhost:5000/event_url" @@ -2123,14 +2121,14 @@ def publish_auditlog(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user, action="save" ).count() - assert count == 2 + assert count == 1 def test_save_and_publish_auditlog_action_save_another(self, monkeypatch): def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2146,14 +2144,14 @@ def publish_auditlog(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user, action="save" ).count() - assert count == 3 + assert count == 2 def test_save_and_publish_auditlog_action_update(self, monkeypatch): def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2168,14 +2166,14 @@ def publish_auditlog(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user, action="update" ).count() - assert count == 2 + assert count == 1 def test_save_and_publish_auditlog_total_count(self, monkeypatch): def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2197,7 +2195,7 @@ def execute_http_request(*args, **kwargs): return None monkeypatch.setattr(Utility, "execute_http_request", execute_http_request) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2212,11 +2210,11 @@ def execute_http_request(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user ).count() - assert count >= 3 + assert count >= 2 @responses.activate def test_publish_auditlog(self): - bot = "secret" + bot = "publish_auditlog" user = "secret_user" config = { "bot_user_oAuth_token": "xoxb-801939352912-801478018484-v3zq6MYNu62oSs8vammWOY8K", @@ -2252,7 +2250,7 @@ def test_publish_auditlog(self): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user ).count() - assert count == 4 + assert count == 1 @pytest.mark.asyncio async def test_messageConverter_messenger_button_one(self): @@ -2945,7 +2943,7 @@ def test_verify_email_enable_valid_email(self): Utility.verify_email(email) def test_get_llm_hyperparameters(self): - hyperparameters = Utility.get_llm_hyperparameters() + hyperparameters = Utility.get_llm_hyperparameters("openai") assert hyperparameters == { "temperature": 0.0, "max_tokens": 300, @@ -2960,508 +2958,10 @@ def test_get_llm_hyperparameters(self): } def test_get_llm_hyperparameters_not_found(self, monkeypatch): - monkeypatch.setitem(Utility.environment["llm"], "faq", None) - with pytest.raises( - AppException, match="Could not find any hyperparameters for configured LLM." - ): - Utility.get_llm_hyperparameters() - - @pytest.mark.asyncio - async def test_trigger_gpt3_client_completion_with_generated_text( - self, aioresponses - ): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - messages = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = messages - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - - resp = await GPT3Resources("test").invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert resp[0] == generated_text - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gpt3_client_completion_with_response(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - - formatted_response, raw_response = await GPT3Resources("test").invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == { - "choices": [{"message": {"content": generated_text, "role": "assistant"}}] - } - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_completion(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - formatted_response, raw_response = await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == { - "choices": [{"message": {"content": generated_text, "role": "assistant"}}] - } - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_completion_failure(self, aioresponses): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=504, - payload={"error": {"message": "Server unavailable!", "id": 876543456789}}, - ) - with pytest.raises( - AppException, match="Failed to connect to service: api.openai.com" - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=201, - body="openai".encode(), - repeat=True, - ) - with pytest.raises(AppException): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_embedding(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": f"Bearer {api_key}"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={"data": [{"embedding": embedding}]}, - ) - formatted_response, raw_response = await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - assert formatted_response == embedding - assert raw_response == {"data": [{"embedding": embedding}]} - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_embedding_failure(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - request_header = {"Authorization": f"Bearer {api_key}"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", method="POST", status=504 - ) - - with pytest.raises( - AppException, match="Failed to connect to service: api.openai.com" - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=204, - payload={"error": {"message": "Server unavailable!", "id": 876543456789}}, - repeat=True, - ) - - with pytest.raises( - AppException, match="Server unavailable!. Request id: 876543456789" - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - assert list(aioresponses.requests.values())[0][1].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][1].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_streaming_completion(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - content = """data: {"choices": [{"delta": {"role": "assistant"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": "Python"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " is"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " dynamically"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " typed"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " garbage-collected"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " high"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " level"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " general"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " purpose"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " programming"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": "."}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {}, "index": 0, "finish_reason": "stop"}]}\n\n -data: [DONE]\n\n""" - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - body=content.encode(), - content_type="text/event-stream", - ) - - formatted_response, raw_response = await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == [ - b'data: {"choices": [{"delta": {"role": "assistant"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": "Python"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " is"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " dynamically"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " typed"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " garbage-collected"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " high"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " level"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " general"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " purpose"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " programming"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": "."}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {}, "index": 0, "finish_reason": "stop"}]}\n', - b"\n", - b"\n", - b"data: [DONE]\n", - b"\n", - ] - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_streaming_connection_error(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=401, - ) - - with pytest.raises( - AppException, - match=re.escape( - "Failed to execute the url: 401, message='Unauthorized', url=URL('https://api.openai.com/v1/chat/completions')" - ), - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_streaming_completion_failure(self, aioresponses): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - content = "data: {'choices': [{'delta': {'role': 'assistant'}}]}\n\n" - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - body=content.encode(), - content_type="text/event-stream", - ) - with pytest.raises( - AppException, - match=re.escape( - "Failed to parse streaming response: b\"data: {'choices': [{'delta': {'role': 'assistant'}}]}\\n\"" - ), - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_completion_failure_invalid_json( - self, aioresponses - ): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - content = "data: {'choices': [{'delta': {'role': 'assistant'}}]}\n\n" - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=504, - body=content.encode(), - ) with pytest.raises( - AppException, match="Failed to connect to service: api.openai.com" + AppException, match="Could not find any hyperparameters for claude LLM." ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) + Utility.get_llm_hyperparameters("claude") def test_get_client_ip_with_request_client(self): request = MagicMock() @@ -3470,217 +2970,6 @@ def test_get_client_ip_with_request_client(self): ip = Utility.get_client_ip(request) assert "58.0.127.89" == ip - def test_llm_resource_provider_factory(self): - client = LLMClientFactory.get_resource_provider(LLMResourceProvider.azure.value) - assert isinstance(client("test"), AzureGPT3Resources) - - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.openai.value - ) - assert isinstance(client("test"), GPT3Resources) - - def test_llm_resource_provider_not_implemented(self): - with pytest.raises(AppException, match="aws client not supported"): - LLMClientFactory.get_resource_provider("aws") - - @pytest.mark.asyncio - async def test_trigger_azure_client_completion(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"api-key": api_key} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['chat_completion_model_id']}/{GPT3ResourceTypes.chat_completion.value}?api-version={llm_settings['api_version']}", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - formatted_response, raw_response = await client.invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == { - "choices": [{"message": {"content": generated_text, "role": "assistant"}}] - } - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_azure_client_embedding(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"api-key": api_key} - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['embeddings_model_id']}/{GPT3ResourceTypes.embeddings.value}?api-version={llm_settings['api_version']}", - method="POST", - status=200, - payload={"data": [{"embedding": embedding}]}, - ) - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - formatted_response, raw_response = await client.invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - assert formatted_response == embedding - assert raw_response == {"data": [{"embedding": embedding}]} - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_azure_client_embedding_failure(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - request_header = {"api-key": api_key} - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['embeddings_model_id']}/{GPT3ResourceTypes.embeddings.value}?api-version={llm_settings['api_version']}", - method="POST", - status=504, - ) - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - - with pytest.raises( - AppException, match="Failed to connect to service: kairon.openai.azure.com" - ): - await client.invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_azure_client_completion_failure(self, aioresponses): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"api-key": api_key} - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['chat_completion_model_id']}/{GPT3ResourceTypes.chat_completion.value}?api-version={llm_settings['api_version']}", - method="POST", - status=504, - payload={"error": {"message": "Server unavailable!", "id": 876543456789}}, - ) - - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - with pytest.raises( - AppException, match="Failed to connect to service: kairon.openai.azure.com" - ): - await client.invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) @pytest.mark.asyncio async def test_messageConverter_whatsapp_dropdown(self): diff --git a/tests/unit_test/validator/training_data_validator_test.py b/tests/unit_test/validator/training_data_validator_test.py index 2633c01f1..f3aa8dc66 100644 --- a/tests/unit_test/validator/training_data_validator_test.py +++ b/tests/unit_test/validator/training_data_validator_test.py @@ -3,6 +3,8 @@ import pytest import yaml from mongoengine import connect +from kairon.shared.utils import Utility +Utility.load_system_metadata() from kairon.exceptions import AppException from kairon.importer.validator.file_validator import TrainingDataValidator @@ -797,55 +799,34 @@ def test_validate_custom_actions_with_errors(self): assert len(error_summary['google_search_actions']) == 2 assert len(error_summary['zendesk_actions']) == 2 assert len(error_summary['pipedrive_leads_actions']) == 3 - assert len(error_summary['prompt_actions']) == 49 + assert len(error_summary['prompt_actions']) == 36 assert len(error_summary['razorpay_actions']) == 3 assert len(error_summary['pyscript_actions']) == 3 assert len(error_summary['database_actions']) == 6 - required_fields_error = error_summary["prompt_actions"][21] - assert re.match(r"Required fields .* not found in action: prompt_action_with_no_llm_prompts", required_fields_error) - del error_summary["prompt_actions"][21] - print(error_summary['prompt_actions']) - assert error_summary['prompt_actions'] == ['top_results should not be greater than 30 and of type int!', - 'similarity_threshold should be within 0.3 and 1.0 and of type int or float!', - 'Collection is required for bot content prompts!', - 'System prompt is required', 'Query prompt must have static source', - 'Name cannot be empty', 'System prompt is required', - 'num_bot_responses should not be greater than 5 and of type int: prompt_action_invalid_num_bot_responses', - 'Collection is required for bot content prompts!', - 'data field in prompts should of type string.', - 'data is required for static prompts', - 'Temperature must be between 0.0 and 2.0!', - 'max_tokens must be between 5 and 4096!', - 'top_p must be between 0.0 and 1.0!', 'n must be between 1 and 5!', - 'presence_penality must be between -2.0 and 2.0!', - 'frequency_penalty must be between -2.0 and 2.0!', - 'logit_bias must be a dictionary!', - 'System prompt must have static source', - 'Collection is required for bot content prompts!', - 'Collection is required for bot content prompts!', - 'Duplicate action found: test_add_prompt_action_one', - 'Invalid action configuration format. Dictionary expected.', - 'Temperature must be between 0.0 and 2.0!', - 'max_tokens must be between 5 and 4096!', - 'top_p must be between 0.0 and 1.0!', 'n must be between 1 and 5!', - 'Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers.', - 'presence_penality must be between -2.0 and 2.0!', - 'frequency_penalty must be between -2.0 and 2.0!', - 'logit_bias must be a dictionary!', - 'Only one system prompt can be present', 'Invalid prompt type', - 'Invalid prompt source', 'Only one system prompt can be present', - 'Invalid prompt type', 'Invalid prompt source', - 'type in LLM Prompts should be of type string.', - 'source in LLM Prompts should be of type string.', - 'Instructions in LLM Prompts should be of type string.', - 'Only one system prompt can be present', - 'Data must contain action name', - 'Only one system prompt can be present', - 'Data must contain slot name', - 'Only one system prompt can be present', - 'Only one system prompt can be present', - 'Only one system prompt can be present', - 'Only one history source can be present'] + expected_errors = ['top_results should not be greater than 30 and of type int!', + 'similarity_threshold should be within 0.3 and 1.0 and of type int or float!', + 'Collection is required for bot content prompts!', 'System prompt is required', + 'Query prompt must have static source', 'Name cannot be empty', 'System prompt is required', + 'num_bot_responses should not be greater than 5 and of type int: prompt_action_invalid_num_bot_responses', + 'Collection is required for bot content prompts!', + 'data field in prompts should of type string.', 'data is required for static prompts', + "['frequency_penalty']: 5 is greater than the maximum of 2.0", + 'System prompt must have static source', 'Collection is required for bot content prompts!', + 'Collection is required for bot content prompts!', + "Required fields ['llm_prompts', 'name'] not found in action: prompt_action_with_no_llm_prompts", + 'Duplicate action found: test_add_prompt_action_one', + 'Invalid action configuration format. Dictionary expected.', + "['frequency_penalty']: 5 is greater than the maximum of 2.0", + 'Only one system prompt can be present', 'Invalid prompt type', + 'Only one system prompt can be present', 'Invalid prompt type', 'Invalid prompt source', + 'type in LLM Prompts should be of type string.', + 'source in LLM Prompts should be of type string.', + 'Instructions in LLM Prompts should be of type string.', + 'Only one system prompt can be present', 'Data must contain action name', + 'Only one system prompt can be present', 'Data must contain slot name', + 'Only one system prompt can be present', 'Only one system prompt can be present', + 'Only one system prompt can be present', 'Only one history source can be present'] + assert not DeepDiff(error_summary['prompt_actions'], expected_errors, ignore_order=True) assert component_count == {'http_actions': 7, 'slot_set_actions': 10, 'form_validation_actions': 9, 'email_actions': 5, 'google_search_actions': 5, 'jira_actions': 6, 'zendesk_actions': 4, 'pipedrive_leads_actions': 5, 'prompt_actions': 8, diff --git a/training_data/ReadMe.md b/training_data/ReadMe.md deleted file mode 100644 index f827d7c61..000000000 --- a/training_data/ReadMe.md +++ /dev/null @@ -1 +0,0 @@ -Trained Data Directory \ No newline at end of file From a4b01c051c4a16379005a95d6da8a98830735285 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Mon, 24 Jun 2024 19:00:46 +0530 Subject: [PATCH 14/57] 1. added missing test cases 2. removed stream from hyperparameters from prompt action --- kairon/actions/definitions/database.py | 2 +- kairon/actions/definitions/prompt.py | 1 - kairon/api/models.py | 10 +- kairon/shared/actions/utils.py | 3 +- kairon/shared/llm/base.py | 4 +- kairon/shared/llm/processor.py | 41 +- kairon/shared/rest_client.py | 2 +- kairon/shared/utils.py | 4 +- kairon/shared/vector_embeddings/db/base.py | 8 +- kairon/shared/vector_embeddings/db/qdrant.py | 18 +- kairon/train.py | 2 +- metadata/integrations.yml | 4 - tests/conftest.py | 1 - tests/integration_test/action_service_test.py | 35 +- tests/integration_test/services_test.py | 243 +++++++++-- tests/unit_test/action/action_test.py | 4 +- .../data_processor/data_processor_test.py | 44 +- tests/unit_test/llm_test.py | 390 ++++++++++++------ .../vector_embeddings/qdrant_test.py | 31 +- 19 files changed, 584 insertions(+), 263 deletions(-) diff --git a/kairon/actions/definitions/database.py b/kairon/actions/definitions/database.py index ebdf83510..0d54abd49 100644 --- a/kairon/actions/definitions/database.py +++ b/kairon/actions/definitions/database.py @@ -83,7 +83,7 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma request_body = ActionUtility.get_payload(payload, tracker) msg_logger.append(request_body) tracker_data = ActionUtility.build_context(tracker, True) - response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id, bot=self.bot) + response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id) logger.info("response: " + str(response)) response_context = self.__add_user_context_to_http_response(response, tracker_data) bot_response, bot_resp_log, _ = ActionUtility.compose_response(vector_action_config['response'], response_context) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 381e6f543..4c7bf6bc4 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -71,7 +71,6 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma llm_processor = LLMProcessor(self.bot) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, user=tracker.sender_id, - bot=self.bot, **llm_params) status = "FAILURE" if llm_response.get("is_failure", False) is True else status exception = llm_response.get("exception") diff --git a/kairon/api/models.py b/kairon/api/models.py index 435566f34..789873fff 100644 --- a/kairon/api/models.py +++ b/kairon/api/models.py @@ -15,8 +15,7 @@ ACTIVITY_STATUS, INTEGRATION_STATUS, FALLBACK_MESSAGE, - DEFAULT_NLU_FALLBACK_RESPONSE, - DEFAULT_LLM + DEFAULT_NLU_FALLBACK_RESPONSE ) from ..shared.actions.models import ( ActionParameterType, @@ -1037,8 +1036,8 @@ class PromptActionConfigRequest(BaseModel): num_bot_responses: int = 5 failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE user_question: UserQuestionModel = UserQuestionModel() - llm_type: str = DEFAULT_LLM - hyperparameters: dict = None + llm_type: str + hyperparameters: dict llm_prompts: List[LlmPromptRequest] instructions: List[str] = [] set_slots: List[SetSlotsUsingActionResponse] = [] @@ -1067,7 +1066,8 @@ def validate_llm_type(cls, v, values, **kwargs): @validator("hyperparameters") def validate_llm_hyperparameters(cls, v, values, **kwargs): - Utility.validate_llm_hyperparameters(v, kwargs['llm_type'], ValueError) + if values.get('llm_type'): + Utility.validate_llm_hyperparameters(v, values['llm_type'], ValueError) @root_validator def check(cls, values): diff --git a/kairon/shared/actions/utils.py b/kairon/shared/actions/utils.py index 00911f55c..d0c97d72f 100644 --- a/kairon/shared/actions/utils.py +++ b/kairon/shared/actions/utils.py @@ -5,6 +5,8 @@ import re from datetime import datetime from typing import Any, List, Text, Dict +from ..utils import Utility +Utility.load_system_metadata() import requests from aiohttp import ContentTypeError @@ -26,7 +28,6 @@ from ..data.data_objects import Slots, KeyVault from ..plugins.factory import PluginFactory from ..rest_client import AioRestClient -from ..utils import Utility from ...exceptions import AppException diff --git a/kairon/shared/llm/base.py b/kairon/shared/llm/base.py index f07eceda0..006e38a3d 100644 --- a/kairon/shared/llm/base.py +++ b/kairon/shared/llm/base.py @@ -8,9 +8,9 @@ def __init__(self, bot: Text): self.bot = bot @abstractmethod - async def train(self, user, bot, *args, **kwargs) -> Dict: + async def train(self, user, *args, **kwargs) -> Dict: pass @abstractmethod - async def predict(self, query, user, bot, *args, **kwargs) -> Dict: + async def predict(self, query, user, *args, **kwargs) -> Dict: pass diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index ffc48e2eb..c6e7fa8af 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -1,4 +1,4 @@ -import random +from secrets import randbelow, choice import time from typing import Text, Dict, List, Tuple from urllib.parse import urljoin @@ -39,7 +39,7 @@ def __init__(self, bot: Text): self.EMBEDDING_CTX_LENGTH = 8191 self.__logs = [] - async def train(self, user, bot, *args, **kwargs) -> Dict: + async def train(self, user, *args, **kwargs) -> Dict: await self.__delete_collections() count = 0 processor = CognitionDataProcessor() @@ -59,26 +59,25 @@ async def train(self, user, bot, *args, **kwargs) -> Dict: content['data'], metadata) else: search_payload, embedding_payload = {'content': content["data"]}, content["data"] - #search_payload['collection_name'] = collection - embeddings = await self.get_embedding(embedding_payload, user, bot) + embeddings = await self.get_embedding(embedding_payload, user) points = [{'id': content['vector_id'], 'vector': embeddings, 'payload': search_payload}] await self.__collection_upsert__(collection, {'points': points}, err_msg="Unable to train FAQ! Contact support") count += 1 return {"faq": count} - async def predict(self, query: Text, user, bot, *args, **kwargs) -> Tuple: + async def predict(self, query: Text, user, *args, **kwargs) -> Tuple: start_time = time.time() embeddings_created = False try: - query_embedding = await self.get_embedding(query, user, bot) + query_embedding = await self.get_embedding(query, user) embeddings_created = True system_prompt = kwargs.pop('system_prompt', DEFAULT_SYSTEM_PROMPT) context_prompt = kwargs.pop('context_prompt', DEFAULT_CONTEXT_PROMPT) context = await self.__attach_similarity_prompt_if_enabled(query_embedding, context_prompt, **kwargs) - answer = await self.__get_answer(query, system_prompt, context, user, bot, **kwargs) + answer = await self.__get_answer(query, system_prompt, context, user, **kwargs) response = {"content": answer} except Exception as e: logging.exception(e) @@ -100,11 +99,11 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def get_embedding(self, text: Text, user, bot) -> List[float]: + async def get_embedding(self, text: Text, user) -> List[float]: truncated_text = self.truncate_text(text) result = await litellm.aembedding(model="text-embedding-3-small", input=[truncated_text], - metadata={'user': user, 'bot': bot}, + metadata={'user': user, 'bot': self.bot}, api_key=self.api_key, num_retries=3) return result["data"][0]["embedding"] @@ -112,24 +111,25 @@ async def get_embedding(self, text: Text, user, bot) -> List[float]: async def __parse_completion_response(self, response, **kwargs): if kwargs.get("stream"): formatted_response = '' - msg_choice = random.randint(0, kwargs.get("n", 1) - 1) + msg_choice = randbelow(kwargs.get("n", 1)) if response["choices"][0].get("index") == msg_choice and response["choices"][0]['delta'].get('content'): formatted_response = f"{response['choices'][0]['delta']['content']}" else: - msg_choice = random.choice(response['choices']) + msg_choice = choice(response['choices']) formatted_response = msg_choice['message']['content'] return formatted_response - async def __get_completion(self, messages, hyperparameters, user, bot, **kwargs): + async def __get_completion(self, messages, hyperparameters, user, **kwargs): response = await litellm.acompletion(messages=messages, - metadata={'user': user, 'bot': bot}, + metadata={'user': user, 'bot': self.bot}, api_key=self.api_key, num_retries=3, **hyperparameters) - formatted_response = await self.__parse_completion_response(response, **kwargs) + formatted_response = await self.__parse_completion_response(response, + **hyperparameters) return formatted_response, response - async def __get_answer(self, query, system_prompt: Text, context: Text, user, bot, **kwargs): + async def __get_answer(self, query, system_prompt: Text, context: Text, user, **kwargs): use_query_prompt = False query_prompt = '' if kwargs.get('query_prompt', {}): @@ -144,8 +144,7 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, bo if use_query_prompt and query_prompt: query = await self.__rephrase_query(query, system_prompt, query_prompt, hyperparameters=hyperparameters, - user=user, - bot=bot) + user=user) messages = [ {"role": "system", "content": system_prompt}, ] @@ -156,13 +155,12 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, bo completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user, - bot=bot) + user=user) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'answer_query', 'hyperparameters': hyperparameters}) return completion - async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, bot, **kwargs): + async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, **kwargs): messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": f"{query_prompt}\n\n Q: {query}\n A:"} @@ -171,8 +169,7 @@ async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user, - bot=bot) + user=user) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'rephrase_query', 'hyperparameters': hyperparameters}) return completion diff --git a/kairon/shared/rest_client.py b/kairon/shared/rest_client.py index a76faad82..301e11323 100644 --- a/kairon/shared/rest_client.py +++ b/kairon/shared/rest_client.py @@ -60,7 +60,7 @@ async def request(self, request_method: str, http_url: str, request_body: Union[ headers: dict = None, return_json: bool = True, **kwargs): max_retries = kwargs.get("max_retries", 1) - status_forcelist = kwargs.get("status_forcelist", [104, 502, 503, 504]) + status_forcelist = set(kwargs.get("status_forcelist", [104, 502, 503, 504])) timeout = ClientTimeout(total=kwargs['timeout']) if kwargs.get('timeout') else None is_streaming_resp = kwargs.pop("is_streaming_resp", False) content_type = kwargs.pop("content_type", HttpRequestContentType.json.value) diff --git a/kairon/shared/utils.py b/kairon/shared/utils.py index 5cb61ce02..2899923b3 100644 --- a/kairon/shared/utils.py +++ b/kairon/shared/utils.py @@ -2069,12 +2069,12 @@ def get_llm_hyperparameters(llm_type): @staticmethod def validate_llm_hyperparameters(hyperparameters: dict, llm_type: str, exception_class): - from jsonschema_rs import JSONSchema, ValidationError + from jsonschema_rs import JSONSchema, ValidationError as JValidationError schema = Utility.system_metadata["llm"][llm_type] try: validator = JSONSchema(schema) validator.validate(hyperparameters) - except ValidationError as e: + except JValidationError as e: message = f"{e.instance_path}: {e.message}" raise exception_class(message) diff --git a/kairon/shared/vector_embeddings/db/base.py b/kairon/shared/vector_embeddings/db/base.py index d1c2a1e97..887be41bb 100644 --- a/kairon/shared/vector_embeddings/db/base.py +++ b/kairon/shared/vector_embeddings/db/base.py @@ -8,16 +8,16 @@ class VectorEmbeddingsDbBase(ABC): @abstractmethod - async def embedding_search(self, request_body: Dict, **kwargs): + async def embedding_search(self, request_body: Dict, user: str, **kwargs): raise NotImplementedError("Provider not implemented") @abstractmethod - async def payload_search(self, request_body: Dict, **kwargs): + async def payload_search(self, request_body: Dict, user: str, **kwargs): raise NotImplementedError("Provider not implemented") - async def perform_operation(self, op_type: Text, request_body: Dict, **kwargs): + async def perform_operation(self, op_type: Text, request_body: Dict, user: str, **kwargs): supported_ops = {DbActionOperationType.payload_search.value: self.payload_search, DbActionOperationType.embedding_search.value: self.embedding_search} if op_type not in supported_ops.keys(): raise AppException("Operation type not supported") - return await supported_ops[op_type](request_body, **kwargs) + return await supported_ops[op_type](request_body, user, **kwargs) diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index 893a310ad..12b5268e2 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -29,23 +29,15 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 - def truncate_text(self, text: Text) -> Text: - """ - Truncate text to 8191 tokens for openai - """ - tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] - return self.tokenizer.decode(tokens) + async def __get_embedding(self, text: Text, user: str, **kwargs) -> List[float]: + return await self.llm.get_embedding(text, user=user) - async def __get_embedding(self, text: Text, **kwargs) -> List[float]: - result, _ = await self.llm.get_embedding(text, user=kwargs.get('user'), bot=kwargs.get('bot')) - return result - - async def embedding_search(self, request_body: Dict, **kwargs): + async def embedding_search(self, request_body: Dict, user: str, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points") if request_body.get("text"): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/search") user_msg = request_body.get("text") - vector = await self.__get_embedding(user_msg, **kwargs) + vector = await self.__get_embedding(user_msg, user, **kwargs) request_body = {'vector': vector, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} embedding_search_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', @@ -53,7 +45,7 @@ async def embedding_search(self, request_body: Dict, **kwargs): request_body=request_body) return embedding_search_result - async def payload_search(self, request_body: Dict, **kwargs): + async def payload_search(self, request_body: Dict, user, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/scroll") payload_filter_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', diff --git a/kairon/train.py b/kairon/train.py index 0276f7bc5..3ddcf9eb0 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -102,7 +102,7 @@ def start_training(bot: str, user: str, token: str = None): settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: llm_processor = LLMProcessor(bot) - faqs = asyncio.run(llm_processor.train(user=user, bot=bot)) + faqs = asyncio.run(llm_processor.train(user=user)) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) agent_url = Utility.environment['model']['agent'].get('url') diff --git a/metadata/integrations.yml b/metadata/integrations.yml index 227b4c413..6ba78b15f 100644 --- a/metadata/integrations.yml +++ b/metadata/integrations.yml @@ -129,10 +129,6 @@ llm: minimum: 1 maximum: 5 description: "The n hyperparameter controls the number of different response options that are generated by the model." - stream: - type: boolean - default: false - description: "the stream hyperparameter controls whether the model generates a continuous stream of text or a single response." stop: anyOf: - type: "string" diff --git a/tests/conftest.py b/tests/conftest.py index c613d74a5..10bd6434c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ from kairon.shared.concurrency.actors.factory import ActorFactory import pytest -from mock import patch import os diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index 1941a8fd6..e54daa27e 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -10497,7 +10497,7 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10564,7 +10564,7 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10634,7 +10634,7 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10859,7 +10859,7 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion.return_value = {'choices': [{'delta': {'role': 'assistant', 'content': generated_text}, 'finish_reason': None, 'index': 0}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10875,6 +10875,7 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m response = client.post("/webhook", json=request_object) response_json = response.json() + print(response_json['events']) assert response_json['events'] == [ {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': generated_text}] assert response_json['responses'] == [ @@ -11161,14 +11162,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11245,7 +11246,7 @@ def mock_completion_for_answer(*args, **kwargs): 'content': 'Kanban is a workflow management tool which visualizes both the process (the workflow) and the actual work passing through that process.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) @@ -11253,7 +11254,7 @@ def mock_completion_for_answer(*args, **kwargs): 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11370,14 +11371,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11477,14 +11478,14 @@ def mock_completion_for_answer(*args, **kwargs): 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11575,14 +11576,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11646,7 +11647,7 @@ def __mock_fetch_similar(*args, **kwargs): 'content': "\nInstructions on how to use Similarity Prompt:\n['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Kanban And Scrum Together? \nA:"}], 'metadata': {'user': user, 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11710,7 +11711,7 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], 'metadata': {'user': 'udit.pandey', 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11777,5 +11778,5 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], 'metadata': {'user': user, 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 1b4219e76..8d99a6c06 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -47,6 +47,7 @@ KAIRON_TWO_STAGE_FALLBACK, FeatureMappings, DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_LLM ) from kairon.shared.data.data_objects import ( Stories, @@ -1573,7 +1574,6 @@ def test_get_live_agent_with_no_live_agent(): def test_enable_live_agent(): - bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.live_agent_enabled = True bot_settings.save() @@ -1596,7 +1596,6 @@ def test_enable_live_agent(): assert actual["success"] - def test_get_live_agent_after_enabled_no_bot_settings_enabled(): bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.live_agent_enabled = False @@ -1613,6 +1612,7 @@ def test_get_live_agent_after_enabled_no_bot_settings_enabled(): assert not actual["message"] assert actual["success"] + def test_get_live_agent_after_enabled(): bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.live_agent_enabled = True @@ -2349,7 +2349,9 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters()} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -3086,6 +3088,8 @@ def _mock_get_bot_settings(*args, **kwargs): ], "num_bot_responses": 5, "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3143,6 +3147,8 @@ def _mock_get_bot_settings(*args, **kwargs): ], "num_bot_responses": 5, "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3187,6 +3193,8 @@ def test_add_prompt_action_with_invalid_query_prompt(): ], "num_bot_responses": 5, "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3244,6 +3252,8 @@ def test_add_prompt_action_with_invalid_num_bot_responses(): ], "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 10, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3300,7 +3310,9 @@ def test_add_prompt_action_with_invalid_system_prompt_source(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3365,7 +3377,9 @@ def test_add_prompt_action_with_multiple_system_prompt(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3422,7 +3436,9 @@ def test_add_prompt_action_with_empty_llm_prompt_name(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3479,7 +3495,9 @@ def test_add_prompt_action_with_empty_data_for_static_prompt(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3540,7 +3558,9 @@ def test_add_prompt_action_with_multiple_history_source_prompts(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3600,6 +3620,8 @@ def test_add_prompt_action_with_gpt_feature_disabled(): "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3616,6 +3638,138 @@ def test_add_prompt_action_with_gpt_feature_disabled(): assert actual["error_code"] == 422 +def test_add_prompt_action_with_invalid_llm_type(monkeypatch): + def _mock_get_bot_settings(*args, **kwargs): + return BotSettings( + bot=pytest.bot, + user="integration@demo.ai", + llm_settings=LLMSettings(enable_faq=True), + ) + + monkeypatch.setattr(MongoProcessor, "get_bot_settings", _mock_get_bot_settings) + action = { + "name": "test_add_prompt_action_with_invalid_llm_type", 'user_question': {'type': 'from_user_message'}, + "llm_prompts": [ + { + "name": "System Prompt", + "data": "You are a personal assistant.", + "type": "system", + "source": "static", + "is_enabled": True, + }, + { + "name": "Similarity Prompt", + "data": "Bot_collection", + "instructions": "Answer question based on the context above, if answer is not in the context go check previous logs.", + "type": "user", + "source": "bot_content", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "If there is no specific query, assume that user is aking about java programming.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + ], + "instructions": ["Answer in a short manner.", "Keep it simple."], + "num_bot_responses": 5, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": "test", + "hyperparameters": Utility.get_default_llm_hyperparameters() + } + response = client.post( + f"/api/bot/{pytest.bot}/action/prompt", + json=action, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + assert not DeepDiff(actual["message"], + [{'loc': ['body', 'llm_type'], 'msg': 'Invalid llm type', 'type': 'value_error'}], + ignore_order=True) + assert not actual["success"] + assert not actual["data"] + assert actual["error_code"] == 422 + + +def test_add_prompt_action_with_invalid_hyperameters(monkeypatch): + temp = Utility.get_default_llm_hyperparameters() + temp['temperature'] = 3.0 + + def _mock_get_bot_settings(*args, **kwargs): + return BotSettings( + bot=pytest.bot, + user="integration@demo.ai", + llm_settings=LLMSettings(enable_faq=True), + ) + + monkeypatch.setattr(MongoProcessor, "get_bot_settings", _mock_get_bot_settings) + action = { + "name": "test_add_prompt_action_with_invalid_hyperameters", 'user_question': {'type': 'from_user_message'}, + "llm_prompts": [ + { + "name": "System Prompt", + "data": "You are a personal assistant.", + "type": "system", + "source": "static", + "is_enabled": True, + }, + { + "name": "Similarity Prompt", + "data": "Bot_collection", + "instructions": "Answer question based on the context above, if answer is not in the context go check previous logs.", + "type": "user", + "source": "bot_content", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "If there is no specific query, assume that user is aking about java programming.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + ], + "instructions": ["Answer in a short manner.", "Keep it simple."], + "num_bot_responses": 5, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": temp + } + response = client.post( + f"/api/bot/{pytest.bot}/action/prompt", + json=action, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + assert not DeepDiff(actual["message"], + [{'loc': ['body', 'hyperparameters'], + 'msg': "['temperature']: 3.0 is greater than the maximum of 2.0", 'type': 'value_error'}], + ignore_order=True) + assert not actual["success"] + assert not actual["data"] + assert actual["error_code"] == 422 + + def test_add_prompt_action(monkeypatch): def _mock_get_bot_settings(*args, **kwargs): return BotSettings( @@ -3662,7 +3816,9 @@ def _mock_get_bot_settings(*args, **kwargs): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3723,7 +3879,9 @@ def _mock_get_bot_settings(*args, **kwargs): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3774,7 +3932,9 @@ def test_update_prompt_action_does_not_exist(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/61512cc2c6219f0aae7bba3d", @@ -3826,7 +3986,9 @@ def test_update_prompt_action_with_invalid_similarity_threshold(): }, ], "num_bot_responses": 5, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -3871,7 +4033,9 @@ def test_update_prompt_action_with_invalid_top_results(): }, ], "num_bot_responses": 5, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -3915,7 +4079,9 @@ def test_update_prompt_action_with_invalid_num_bot_responses(): }, ], "num_bot_responses": 50, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -3964,6 +4130,8 @@ def test_update_prompt_action_with_invalid_query_prompt(): "num_bot_responses": 5, "use_query_prompt": True, "query_prompt": "", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4021,6 +4189,8 @@ def test_update_prompt_action_with_query_prompt_with_false(): }, ], "dispatch_response": False, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4077,7 +4247,9 @@ def test_update_prompt_action(): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4107,7 +4279,7 @@ def test_get_prompt_action(): 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity_analytical Prompt', 'data': 'Bot_collection', @@ -4171,7 +4343,9 @@ def _mock_get_bot_settings(*args, **kwargs): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -4200,7 +4374,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'user_question': {'type': 'from_user_message'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -4269,7 +4443,10 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() + } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -4289,7 +4466,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'user_question': {'type': 'from_user_message'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -4361,7 +4538,10 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() + } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -4381,7 +4561,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'user_question': {'type': 'from_user_message'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -5036,7 +5216,8 @@ def test_get_data_importer_logs(): assert actual['data']["logs"][3]['event_status'] == EVENT_STATUS.COMPLETED.value assert actual['data']["logs"][3]['status'] == 'Failure' assert set(actual['data']["logs"][3]['files_received']) == {'rules', 'stories', 'nlu', 'domain', 'config', - 'actions', 'chat_client_config', 'multiflow_stories', 'bot_content'} + 'actions', 'chat_client_config', 'multiflow_stories', + 'bot_content'} assert actual['data']["logs"][3]['is_data_uploaded'] assert actual['data']["logs"][3]['start_timestamp'] assert actual['data']["logs"][3]['end_timestamp'] @@ -5068,7 +5249,8 @@ def test_get_data_importer_logs(): ] assert actual['data']["logs"][3]['is_data_uploaded'] assert set(actual['data']["logs"][3]['files_received']) == {'rules', 'stories', 'nlu', 'config', 'domain', - 'actions', 'chat_client_config', 'multiflow_stories','bot_content'} + 'actions', 'chat_client_config', 'multiflow_stories', + 'bot_content'} @responses.activate @@ -6197,6 +6379,7 @@ def test_add_story_lone_intent(): } ] + def test_add_story_consecutive_intents(): response = client.post( f"/api/bot/{pytest.bot}/stories", @@ -9850,6 +10033,7 @@ def test_login_for_verified(): pytest.access_token = actual["data"]["access_token"] pytest.token_type = actual["data"]["token_type"] + def test_list_bots_for_different_user(): response = client.get( "/api/account/bot", @@ -18696,7 +18880,7 @@ def test_set_templates_with_sysadmin_as_user(): intents = Intents.objects(bot=pytest.bot) intents = [{k: v for k, v in intent.to_mongo().to_dict().items() if k not in ['_id', 'bot', 'timestamp']} for - intent in intents] + intent in intents] assert intents == [ {'name': 'greet', 'user': 'sysadmin', 'status': True, 'is_integration': False, 'use_entities': False}, @@ -18772,7 +18956,6 @@ def test_add_channel_config(monkeypatch): def test_add_bot_with_template_with_sysadmin_as_user(monkeypatch): - def mock_reload_model(*args, **kwargs): mock_reload_model.called_with = (args, kwargs) return None @@ -18811,7 +18994,7 @@ def mock_reload_model(*args, **kwargs): rules = Rules.objects(bot=bot_id) rules = [{k: v for k, v in rule.to_mongo().to_dict().items() if k not in ['_id', 'bot', 'timestamp']} for - rule in rules] + rule in rules] assert rules == [ {'block_name': 'ask the user to rephrase whenever they send a message with low nlu confidence', @@ -18825,7 +19008,7 @@ def mock_reload_model(*args, **kwargs): utterances = Utterances.objects(bot=bot_id) utterances = [{k: v for k, v in utterance.to_mongo().to_dict().items() if k not in ['_id', 'bot', 'timestamp']} for - utterance in utterances] + utterance in utterances] assert utterances == [ {'name': 'utter_please_rephrase', 'user': 'sysadmin', 'status': True}, @@ -19940,7 +20123,7 @@ def test_get_bot_settings(): 'whatsapp': 'meta', 'cognition_collections_limit': 3, 'cognition_columns_per_collection_limit': 5, - 'integrations_per_user_limit':3 } + 'integrations_per_user_limit': 3} def test_update_analytics_settings_with_empty_value(): @@ -20018,7 +20201,7 @@ def test_update_analytics_settings(): 'live_agent_enabled': False, 'cognition_collections_limit': 3, 'cognition_columns_per_collection_limit': 5, - 'integrations_per_user_limit':3 } + 'integrations_per_user_limit': 3} def test_delete_channels_config(): diff --git a/tests/unit_test/action/action_test.py b/tests/unit_test/action/action_test.py index 715fecc12..079c17a64 100644 --- a/tests/unit_test/action/action_test.py +++ b/tests/unit_test/action/action_test.py @@ -2657,7 +2657,7 @@ def test_get_prompt_action_config(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'bot': 'test_action_server', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', @@ -3949,7 +3949,7 @@ def test_get_prompt_action_config_2(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'bot': 'test_bot_action_test', 'user': 'test_user_action_test', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'dispatch_response': True, 'set_slots': [], 'llm_type': 'openai', diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 7d4a9ad6e..75eaaccbe 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -172,7 +172,7 @@ def test_add_prompt_action_with_invalid_slots(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', @@ -200,7 +200,7 @@ def test_add_prompt_action_with_invalid_http_action(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', @@ -229,7 +229,7 @@ def test_add_prompt_action_with_invalid_similarity_threshold(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -259,7 +259,7 @@ def test_add_prompt_action_with_invalid_top_results(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -311,7 +311,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', @@ -356,7 +356,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', @@ -556,7 +556,7 @@ def test_add_prompt_action_with_empty_llm_prompts(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': []} with pytest.raises(ValidationError, match="llm_prompts are required!"): @@ -583,7 +583,7 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -604,7 +604,7 @@ def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -621,7 +621,7 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': ["\n", ".", "?", "!", ";"], + 'n': 1, 'stop': ["\n", ".", "?", "!", ";"], 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -641,7 +641,7 @@ def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': -3.0, + 'n': 1, 'stop': '?', 'presence_penalty': -3.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -659,7 +659,7 @@ def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 3.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -676,7 +676,7 @@ def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -693,7 +693,7 @@ def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -710,7 +710,7 @@ def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt-3.5-turbo', 'top_p': 3.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -727,7 +727,7 @@ def test_add_prompt_action_with_invalid_n_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 7, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 7, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -744,7 +744,7 @@ def test_add_prompt_action_with_zero_n_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 0, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 0, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -761,7 +761,7 @@ def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 2, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 2, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': 'a'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -836,7 +836,7 @@ def test_edit_prompt_action_faq_action(self): assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -871,7 +871,7 @@ def test_edit_prompt_action_faq_action(self): 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -914,7 +914,7 @@ def test_edit_prompt_action_with_less_hyperparameters(self): 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -949,7 +949,7 @@ def test_get_prompt_faq_action(self): 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index c8f727db3..7738dfeb0 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -8,6 +8,7 @@ from aiohttp import ClientConnectionError from mongoengine import connect from kairon.shared.utils import Utility + Utility.load_system_metadata() from kairon.exceptions import AppException @@ -66,7 +67,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, @@ -119,14 +120,16 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() embedding = list(np.random.random(LLMProcessor.__embedding__)) - mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]} + mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, { + 'data': [{'embedding': embedding}]} gpt3 = LLMProcessor(bot) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), method="GET", - payload={"time": 0, "status": "ok", "result": {"collections": [{"name": "test_embed_faq_text_swift_faq_embd"}, - {"name": "example_bot_swift_faq_embd"}]}} + payload={"time": 0, "status": "ok", + "result": {"collections": [{"name": "test_embed_faq_text_swift_faq_embd"}, + {"name": "example_bot_swift_faq_embd"}]}} ) aioresponses.add( @@ -141,19 +144,22 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_user_details{gpt3.suffix}/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_user_details{gpt3.suffix}/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_country_details{gpt3.suffix}"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_country_details{gpt3.suffix}"), method="PUT", status=200 ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_country_details{gpt3.suffix}/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_country_details{gpt3.suffix}/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) @@ -165,24 +171,29 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 3 - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'name': f"{gpt3.bot}_country_details{gpt3.suffix}", - 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'name': f"{gpt3.bot}_country_details{gpt3.suffix}", + 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content_two.vector_id, - 'vector': embedding, - 'payload': {'country': 'Spain'}}]} - assert list(aioresponses.requests.values())[3][1].kwargs['json'] == {'points': [{'id': test_content_three.vector_id, - 'vector': embedding, - 'payload': {'role': 'ds'}}]} + assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + 'points': [{'id': test_content_two.vector_id, + 'vector': embedding, + 'payload': {'country': 'Spain'}}]} + assert list(aioresponses.requests.values())[3][1].kwargs['json'] == { + 'points': [{'id': test_content_three.vector_id, + 'vector': embedding, + 'payload': {'role': 'ds'}}]} - assert list(aioresponses.requests.values())[4][0].kwargs['json'] == {'name': f"{gpt3.bot}_user_details{gpt3.suffix}", - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[5][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, - 'payload': {'name': 'Nupur'}}]} + assert list(aioresponses.requests.values())[4][0].kwargs['json'] == { + 'name': f"{gpt3.bot}_user_details{gpt3.suffix}", + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[5][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, + 'payload': {'name': 'Nupur'}}]} assert response['faq'] == 3 expected = {"model": "text-embedding-3-small", @@ -217,7 +228,8 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a gpt3 = LLMProcessor(bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), method="PUT", status=200 ) @@ -227,18 +239,21 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a payload={"time": 0, "status": "ok", "result": {"collections": []}}) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/test_embed_faq_json_payload_with_int_faq_embd/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 1 - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_embed_faq_json_payload_with_int_faq_embd', - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == { + 'name': 'test_embed_faq_json_payload_with_int_faq_embd', + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, 'vector': embedding, 'payload': {'name': 'Ram', 'age': 23, 'color': 'red'} }]} @@ -302,7 +317,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_int_embd_int_faq_embd', @@ -363,16 +378,18 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aiore url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}{gpt3.suffix}/points"), method="PUT", payload={"result": None, - 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, - "time": 0.003612634} + 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, + "time": 0.003612634} ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train(user=user, bot=bot) + await gpt3.train(user=user) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, 'payload': {'content': test_content.data}}]} + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, 'payload': {'content': test_content.data}}]} expected = {"model": "text-embedding-3-small", "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, @@ -412,33 +429,38 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb aioresponses.add( method="DELETE", - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd"), ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd"), method="PUT", status=200 ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd/points"), method="PUT", payload={"result": None, - 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, - "time": 0.003612634} + 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, + "time": 0.003612634} ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train(user=user, bot=bot) + await gpt3.train(user=user) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == { + 'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} expected_payload = test_content.data #expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, - 'payload': expected_payload - }]} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, + 'payload': expected_payload + }]} expected = {"model": "text-embedding-3-small", "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, @@ -449,7 +471,7 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict" @@ -469,9 +491,9 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], "hyperparameters": hyperparameters } mock_completion_request = {"messages": [ @@ -487,13 +509,14 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, @@ -514,7 +537,8 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict_with_default_collection" @@ -559,7 +583,7 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_emb {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, @@ -599,11 +623,11 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], "hyperparameters": hyperparameters - } + } mock_completion_request = {"messages": [ {"role": "system", @@ -615,17 +639,18 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + with mock.patch.dict(Utility.environment, {'vector': {"key": "test", 'db': "http://localhost:6333"}}): gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=gpt3.bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert gpt3.logs == [ {'messages': [{'role': 'system', @@ -638,10 +663,12 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 @@ -659,25 +686,133 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) - user = "test" - bot = "test_gpt3_faq_embedding_predict_with_values_with_instructions" - key = 'test' + test_content = CognitionData( - data="Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ", - collection='java', bot=bot, user=user).save() - BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() + data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", + collection='python', bot="test_gpt3_faq_embedding_predict_with_values_and_stream", user="test").save() + generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" hyperparameters = Utility.get_default_llm_hyperparameters() + hyperparameters['stream'] = True + key = 'test' + user = "tests" + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": "java"}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], + "hyperparameters": hyperparameters + } + + mock_completion_request = {"messages": [ + {"role": "system", + "content": "You are a personal assistant. Answer the question according to the below context"}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} + ]} + mock_completion_request.update(hyperparameters) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.side_effect = [{'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[29:60]}, + 'finish_reason': None, 'index': 0}]}, + {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[60:]}, + 'finish_reason': 'stop', 'index': 0}]} + ] + + with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + gpt3 = LLMProcessor(test_content.bot) + + aioresponses.add( + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + method="POST", + payload={'result': [ + {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} + ) + + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + assert response['content'] == "Python is dynamically typed, " + assert gpt3.logs == [ + {'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': True, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} + + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': gpt3.bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + @pytest.mark.asyncio + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, + mock_embedding, + mock_completion, + aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "payload_with_instruction" + key = 'test' + CognitionSchema( + metadata=[{"column_name": "name", "data_type": "str", "enable_search": True, "create_embeddings": True}, + {"column_name": "city", "data_type": "str", "enable_search": True, "create_embeddings": True}], + collection_name="User_details", + bot=bot, user=user + ).save() + test_content1 = CognitionData( + data={"name": "Nupur", "city": "Pune"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + test_content2 = CognitionData( + data={"name": "Fahad", "city": "Mumbai"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + test_content3 = CognitionData( + data={"name": "Hitesh", "city": "Mumbai"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=bot, user=user).save() + + generated_text = "Hitesh and Fahad lives in mumbai city." + query = "List all the user lives in mumbai city" + hyperparameters = Utility.get_default_llm_hyperparameters() + k_faq_action_config = { + "system_prompt": "You are a personal assistant. Answer the question according to the below context", + "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", + "similarity_prompt": [{"top_results": 10, + "similarity_threshold": 0.70, + 'use_similarity_prompt': True, + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": "user_details"}], 'instructions': ['Answer in a short way.', 'Keep it simple.'], "hyperparameters": hyperparameters } @@ -686,39 +821,39 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mo {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"} + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n[{'name': 'Fahad', 'city': 'Mumbai'}, {'name': 'Hitesh', 'city': 'Mumbai'}]\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: List all the user lives in mumbai city \nA:"} ]} mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot) - + gpt3 = LLMProcessor(bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content1.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ - {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} + {'id': test_content2.vector_id, 'score': 0.80, "payload": test_content2.data}, + {'id': test_content3.vector_id, 'score': 0.80, "payload": test_content3.data} + ]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text - assert gpt3.logs == [ - {'messages': [{'role': 'system', - 'content': 'You are a personal assistant. Answer the question according to the below context'}, - {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': { - 'choices': [{'message': {'content': 'Python is dynamically typed, garbage-collected, ' - 'high level, general purpose programming.', - 'role': 'assistant'}}]}, - 'type': 'answer_query', - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}}] - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert gpt3.logs == [{'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n[{'name': 'Fahad', 'city': 'Mumbai'}, {'name': 'Hitesh', 'city': 'Mumbai'}]\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: List all the user lives in mumbai city \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Hitesh and Fahad lives in mumbai city.', 'role': 'assistant'}}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', + 'top_p': 0.0, 'n': 1, 'stop': None, + 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 @@ -736,7 +871,8 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mo @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_gpt3_faq_embedding_predict_completion_connection_error" user = 'test' @@ -755,11 +891,11 @@ async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters - } + } def __mock_connection_error(*args, **kwargs): raise Exception("Connection reset by peer!") @@ -770,18 +906,21 @@ def __mock_connection_error(*args, **kwargs): gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", @@ -795,7 +934,7 @@ def __mock_connection_error(*args, **kwargs): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error'}, 'api_key': 'test', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, - 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -804,7 +943,7 @@ def __mock_connection_error(*args, **kwargs): @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock_llm_request): embedding = list(np.random.random(LLMProcessor.__embedding__)) - user ="test" + user = "test" bot = "test_gpt3_faq_embedding_predict_exact_match" key = 'test' test_content = CognitionData( @@ -818,9 +957,9 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters } @@ -829,16 +968,17 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock gpt3 = LLMProcessor(test_content.bot) - response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_exact_match", **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} - assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] + assert gpt3.logs == [ + {'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, - "api_key": key, - "num_retries": 3} + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio @@ -867,7 +1007,7 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ gpt3 = LLMProcessor(test_content.bot) mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] - response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_embedding_connection_error", **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] @@ -882,7 +1022,8 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_gpt3_faq_embedding_predict_with_previous_bot_responses" @@ -921,13 +1062,14 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, @@ -970,11 +1112,11 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding "query_prompt": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", "use_query_prompt": True}, "similarity_prompt": [ - {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters - } + } mock_rephrase_request = {"messages": [ {"role": "system", @@ -993,18 +1135,20 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, { + 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, diff --git a/tests/unit_test/vector_embeddings/qdrant_test.py b/tests/unit_test/vector_embeddings/qdrant_test.py index 7bf166116..667285715 100644 --- a/tests/unit_test/vector_embeddings/qdrant_test.py +++ b/tests/unit_test/vector_embeddings/qdrant_test.py @@ -14,7 +14,9 @@ from kairon.shared.data.data_objects import LLMSettings from kairon.shared.vector_embeddings.db.factory import VectorEmbeddingsDbFactory from kairon.shared.vector_embeddings.db.qdrant import Qdrant - +import litellm +from kairon.shared.llm.processor import LLMProcessor +import numpy as np class TestQdrant: @@ -25,16 +27,21 @@ def init_connection(self): connect(**Utility.mongoengine_connection(Utility.environment['database']["url"])) @pytest.mark.asyncio + @mock.patch.dict(Utility.environment, {'vector': {"key": "TEST", 'db': 'http://localhost:6333'}}) + @mock.patch.object(litellm, "aembedding", autospec=True) @mock.patch.object(ActionUtility, "execute_http_request", autospec=True) - async def test_embedding_search_valid_request_body(self, mock_http_request): + async def test_embedding_search_valid_request_body(self, mock_http_request, mock_embedding): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" Utility.load_environment() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value", bot="5f50fd0a56v098ca10d75d2g", user="user").save() qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) - request_body = {"ids": [0], "with_payload": True, "with_vector": True} + request_body = {"ids": [0], "with_payload": True, "with_vector": True, 'text': "Hi"} mock_http_request.return_value = 'expected_result' - result = await qdrant.embedding_search(request_body) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + result = await qdrant.embedding_search(request_body, user=user) assert result == 'expected_result' @pytest.mark.asyncio @@ -46,29 +53,31 @@ async def test_payload_search_valid_request_body(self, mock_http_request): request_body = {"filter": {"should": [{"key": "city", "match": {"value": "London"}}, {"key": "color", "match": {"value": "red"}}]}} mock_http_request.return_value = 'expected_result' - result = await qdrant.payload_search(request_body) + result = await qdrant.payload_search(request_body, user="test") assert result == 'expected_result' @pytest.mark.asyncio @mock.patch.object(ActionUtility, "execute_http_request", autospec=True) async def test_perform_operation_valid_op_type_and_request_body(self, mock_http_request): Utility.load_environment() + user = "test" qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) request_body = {} mock_http_request.return_value = 'expected_result' - result_embedding = await qdrant.perform_operation('embedding_search', request_body) + result_embedding = await qdrant.perform_operation('embedding_search', request_body, user=user) assert result_embedding == 'expected_result' - result_payload = await qdrant.perform_operation('payload_search', request_body) + result_payload = await qdrant.perform_operation('payload_search', request_body, user=user) assert result_payload == 'expected_result' @pytest.mark.asyncio async def test_embedding_search_empty_request_body(self): Utility.load_environment() + user = "test" qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) with pytest.raises(ActionFailure): - await qdrant.embedding_search({}) + await qdrant.embedding_search({}, user=user) @pytest.mark.asyncio async def test_payload_search_empty_request_body(self): @@ -76,7 +85,7 @@ async def test_payload_search_empty_request_body(self): qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) with pytest.raises(ActionFailure): - await qdrant.payload_search({}) + await qdrant.payload_search({}, user="test") @pytest.mark.asyncio async def test_perform_operation_invalid_op_type(self): @@ -85,7 +94,7 @@ async def test_perform_operation_invalid_op_type(self): LLMSettings(provider="openai").to_mongo().to_dict()) request_body = {} with pytest.raises(AppException, match="Operation type not supported"): - await qdrant.perform_operation("vector_search", request_body) + await qdrant.perform_operation("vector_search", request_body, user="test") def test_get_instance_raises_exception_when_db_not_implemented(self): with pytest.raises(AppException, match="Database not yet implemented!"): @@ -99,7 +108,7 @@ async def test_embedding_search_valid_request_body_payload(self, mock_http_reque LLMSettings(provider="openai").to_mongo().to_dict()) request_body = {'ids': [0], 'with_payload': True, 'with_vector': True} mock_http_request.return_value = 'expected_result' - result = await qdrant.embedding_search(request_body) + result = await qdrant.embedding_search(request_body, user="test") assert result == 'expected_result' mock_http_request.assert_called_once() From 8cc9c97534312978ec435322689136a2176a4cf3 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 10:40:04 +0530 Subject: [PATCH 15/57] test cased fixed --- tests/unit_test/data_processor/data_processor_test.py | 6 ++++-- tests/unit_test/utility_test.py | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 75eaaccbe..7f3d138de 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -60,7 +60,7 @@ from kairon.shared.data.constant import UTTERANCE_TYPE, EVENT_STATUS, STORY_EVENT, ALLOWED_DOMAIN_FORMATS, \ ALLOWED_CONFIG_FORMATS, ALLOWED_NLU_FORMATS, ALLOWED_STORIES_FORMATS, ALLOWED_RULES_FORMATS, REQUIREMENTS, \ DEFAULT_NLU_FALLBACK_RULE, SLOT_TYPE, KAIRON_TWO_STAGE_FALLBACK, AuditlogActions, TOKEN_TYPE, GPT_LLM_FAQ, \ - DEFAULT_CONTEXT_PROMPT, DEFAULT_NLU_FALLBACK_RESPONSE, DEFAULT_SYSTEM_PROMPT + DEFAULT_CONTEXT_PROMPT, DEFAULT_NLU_FALLBACK_RESPONSE, DEFAULT_SYSTEM_PROMPT, DEFAULT_LLM from kairon.shared.data.data_objects import (TrainingExamples, Slots, Entities, EntitySynonyms, RegexFeatures, @@ -8507,7 +8507,9 @@ def test_delete_action_with_attached_http_action(self): 'data': 'tester_action', 'instructions': 'Answer according to the context', 'type': 'user', 'source': 'action', - 'is_enabled': True}] + 'is_enabled': True}], + llm_type=DEFAULT_LLM, + hyperparameters=Utility.get_default_llm_hyperparameters() ) processor.add_http_action_config(http_action_config.dict(), user, bot) processor.add_prompt_action(prompt_action_config.dict(), bot, user) diff --git a/tests/unit_test/utility_test.py b/tests/unit_test/utility_test.py index 8d4c784c5..429dd3cf2 100644 --- a/tests/unit_test/utility_test.py +++ b/tests/unit_test/utility_test.py @@ -2950,7 +2950,6 @@ def test_get_llm_hyperparameters(self): "model": "gpt-3.5-turbo", "top_p": 0.0, "n": 1, - "stream": False, "stop": None, "presence_penalty": 0.0, "frequency_penalty": 0.0, From e3bf9aabece2f6c98302c0e12023a23bb3cb876e Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 15:48:54 +0530 Subject: [PATCH 16/57] 1. added missing test case 2. updated litellm --- kairon/shared/llm/logger.py | 27 ++++++++++++--------------- requirements/prod.txt | 2 +- tests/unit_test/llm_test.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py index 06b720b48..dbb93e469 100644 --- a/kairon/shared/llm/logger.py +++ b/kairon/shared/llm/logger.py @@ -1,14 +1,10 @@ from litellm.integrations.custom_logger import CustomLogger from .data_objects import LLMLogs import ujson as json +from loguru import logger class LiteLLMLogger(CustomLogger): - def log_pre_api_call(self, model, messages, kwargs): - pass - - def log_post_api_call(self, kwargs, response_obj, start_time, end_time): - pass def log_stream_event(self, kwargs, response_obj, start_time, end_time): self.__logs_litellm(**kwargs) @@ -29,15 +25,16 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti self.__logs_litellm(**kwargs) def __logs_litellm(self, **kwargs): - litellm_params = kwargs['litellm_params'] - self.__save_logs(**{'response': json.loads(kwargs['original_response']), - 'start_time': kwargs['start_time'], - 'end_time': kwargs['end_time'], - 'cost': kwargs["response_cost"], - 'llm_call_id': litellm_params['litellm_call_id'], - 'llm_provider': litellm_params['custom_llm_provider'], - 'model_params': kwargs["additional_args"]["complete_input_dict"], - 'metadata': litellm_params['metadata']}) + logger.info("logging llms call") + litellm_params = kwargs.get('litellm_params') + self.__save_logs(**{'response': json.loads(kwargs.get('original_response')) if kwargs.get('original_response') else None, + 'start_time': kwargs.get('start_time'), + 'end_time': kwargs.get('end_time'), + 'cost': kwargs.get("response_cost"), + 'llm_call_id': litellm_params.get('litellm_call_id'), + 'llm_provider': litellm_params.get('custom_llm_provider'), + 'model_params': kwargs.get("additional_args", {}).get("complete_input_dict"), + 'metadata': litellm_params.get('metadata')}) def __save_logs(self, **kwargs): - LLMLogs(**kwargs).save() + print(LLMLogs(**kwargs).save().to_mongo().to_dict()) diff --git a/requirements/prod.txt b/requirements/prod.txt index e00d448a9..13fcc40e7 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -64,6 +64,6 @@ opentelemetry-instrumentation-requests==0.46b0 opentelemetry-instrumentation-sklearn==0.46b0 pykwalify==1.8.0 gunicorn==22.0.0 -litellm==1.38.11 +litellm==1.39.5 jsonschema_rs==0.18.0 mongoengine-jsonschema==0.1.3 \ No newline at end of file diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 7738dfeb0..bb446e802 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1,4 +1,5 @@ import os +import time from urllib.parse import urljoin import mock @@ -17,6 +18,7 @@ from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT from kairon.shared.llm.processor import LLMProcessor +from kairon.shared.llm.data_objects import LLMLogs import litellm from deepdiff import DeepDiff @@ -1166,3 +1168,33 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + @pytest.mark.asyncio + async def test_llm_logging(self): + from kairon.shared.llm.logger import LiteLLMLogger + bot = "test_llm_logging" + user = "test" + litellm.callbacks = [LiteLLMLogger()] + + result = await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + metadata={'user': user, 'bot': bot}) + assert result + + result = litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + metadata={'user': user, 'bot': bot}) + assert result + + result = litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + stream=True, + metadata={'user': user, 'bot': bot}) + for chunk in result: + print(chunk["choices"][0]["delta"]["content"]) + assert chunk["choices"][0]["delta"]["content"] + + assert list(LLMLogs.objects(metadata__bot=bot)) From ab991a50e311442422b4232174017664e61e3eef Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 17:28:09 +0530 Subject: [PATCH 17/57] 1. added missing test case --- tests/unit_test/llm_test.py | 56 +++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index bb446e802..09eea5168 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -688,7 +688,8 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) test_content = CognitionData( @@ -720,7 +721,8 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe ]} mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.side_effect = [{'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + mock_completion.side_effect = [{'choices': [ + {'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[29:60]}, 'finish_reason': None, 'index': 0}]}, {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[60:]}, @@ -745,7 +747,9 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe 'content': 'You are a personal assistant. Answer the question according to the below context'}, {'role': 'user', 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + 'raw_completion_response': {'choices': [ + {'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, + 'index': 0}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': True, 'stop': None, 'presence_penalty': 0.0, @@ -1177,17 +1181,17 @@ async def test_llm_logging(self): litellm.callbacks = [LiteLLMLogger()] result = await litellm.acompletion(messages=["Hi"], - model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", - metadata={'user': user, 'bot': bot}) - assert result - - result = litellm.completion(messages=["Hi"], model="gpt-3.5-turbo", mock_response="Hi, How may i help you?", metadata={'user': user, 'bot': bot}) assert result + result = litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + metadata={'user': user, 'bot': bot}) + assert result + result = litellm.completion(messages=["Hi"], model="gpt-3.5-turbo", mock_response="Hi, How may i help you?", @@ -1197,4 +1201,38 @@ async def test_llm_logging(self): print(chunk["choices"][0]["delta"]["content"]) assert chunk["choices"][0]["delta"]["content"] + result = await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + stream=True, + metadata={'user': user, 'bot': bot}) + async for chunk in result: + print(chunk["choices"][0]["delta"]["content"]) + assert chunk["choices"][0]["delta"]["content"] + assert list(LLMLogs.objects(metadata__bot=bot)) + + with pytest.raises(Exception) as e: + await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + metadata={'user': user, 'bot': bot}) + + assert str(e) == "Authentication error" + + with pytest.raises(Exception) as e: + litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + metadata={'user': user, 'bot': bot}) + + assert str(e) == "Authentication error" + + with pytest.raises(Exception) as e: + await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + stream=True, + metadata={'user': user, 'bot': bot}) + + assert str(e) == "Authentication error" From 2da9cc68819f694cdf2a224486d126989982489e Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 18:50:36 +0530 Subject: [PATCH 18/57] 1. added missing test case --- tests/unit_test/llm_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 09eea5168..bcc2ed661 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1210,8 +1210,6 @@ async def test_llm_logging(self): print(chunk["choices"][0]["delta"]["content"]) assert chunk["choices"][0]["delta"]["content"] - assert list(LLMLogs.objects(metadata__bot=bot)) - with pytest.raises(Exception) as e: await litellm.acompletion(messages=["Hi"], model="gpt-3.5-turbo", From 16104bd111928d9da62a0dc094a829fe304e01d5 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 11:10:39 +0530 Subject: [PATCH 19/57] 1. added missing test case 2. removed deprecated api --- kairon/shared/rest_client.py | 29 +++---------------- tests/integration_test/action_service_test.py | 2 +- tests/integration_test/chat_service_test.py | 2 +- tests/integration_test/event_service_test.py | 2 +- .../integration_test/history_services_test.py | 2 +- tests/integration_test/services_test.py | 4 +-- tests/unit_test/action/action_test.py | 1 - tests/unit_test/chat/chat_test.py | 27 +++++++++-------- tests/unit_test/cli_test.py | 17 +++++------ .../data_processor/agent_processor_test.py | 2 +- .../data_processor/data_processor_test.py | 2 +- .../unit_test/data_processor/history_test.py | 2 +- tests/unit_test/events/definitions_test.py | 4 +-- tests/unit_test/events/events_test.py | 2 +- tests/unit_test/events/scheduler_test.py | 2 +- tests/unit_test/idp/test_idp_helper.py | 1 - tests/unit_test/llm_test.py | 7 ++--- tests/unit_test/plugins_test.py | 3 +- tests/unit_test/rest_client_test.py | 17 +++++++++++ tests/unit_test/verification_test.py | 2 +- 20 files changed, 59 insertions(+), 71 deletions(-) diff --git a/kairon/shared/rest_client.py b/kairon/shared/rest_client.py index 301e11323..5c144a0df 100644 --- a/kairon/shared/rest_client.py +++ b/kairon/shared/rest_client.py @@ -32,14 +32,6 @@ def __init__(self, close_session_with_rqst_completion=True): self._time_elapsed = None self._status_code = None - @property - def streaming_response(self): - return self._streaming_response - - @streaming_response.setter - def streaming_response(self, resp): - self._streaming_response = resp - @property def time_elapsed(self): return self._time_elapsed @@ -124,12 +116,9 @@ async def __trigger(self, client, *args, **kwargs) -> ClientResponse: logger.debug(f"Content-type: {response.headers['content-type']}") logger.debug(f"Status code: {str(response.status)}") self.status_code = response.status - if is_streaming_resp: - streaming_resp = await AioRestClient.parse_streaming_response(response) - self.streaming_response = streaming_resp - logger.debug(f"Raw streaming response: {streaming_resp}") - text = await response.text() - logger.debug(f"Raw response: {text}") + if not is_streaming_resp: + text = await response.text() + logger.debug(f"Raw response: {text}") return response def __validate_response(self, response: ClientResponse, **kwargs): @@ -149,14 +138,4 @@ async def cleanup(self): Close underlying connector to release all acquired resources. """ if not self.session.closed: - await self.session.close() - - @staticmethod - async def parse_streaming_response(response): - chunks = [] - async for chunk in response.content: - if not chunk: - break - chunks.append(chunk) - - return chunks + await self.session.close() \ No newline at end of file diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index e54daa27e..cd5e2c63c 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -3,7 +3,7 @@ from urllib.parse import urlencode, urljoin import litellm -import mock +from unittest import mock import numpy as np import pytest import responses diff --git a/tests/integration_test/chat_service_test.py b/tests/integration_test/chat_service_test.py index 77b828a71..04d65276a 100644 --- a/tests/integration_test/chat_service_test.py +++ b/tests/integration_test/chat_service_test.py @@ -19,7 +19,7 @@ import pytest import responses -from mock import patch +from unittest.mock import patch from mongoengine import connect from slack_sdk.web.slack_response import SlackResponse from starlette.exceptions import HTTPException diff --git a/tests/integration_test/event_service_test.py b/tests/integration_test/event_service_test.py index 445a37949..f03f65817 100644 --- a/tests/integration_test/event_service_test.py +++ b/tests/integration_test/event_service_test.py @@ -3,7 +3,7 @@ from dramatiq.brokers.stub import StubBroker from loguru import logger -from mock import patch +from unittest.mock import patch from starlette.testclient import TestClient from kairon.shared.constants import EventClass, EventExecutor diff --git a/tests/integration_test/history_services_test.py b/tests/integration_test/history_services_test.py index 2d8cc807a..46e683b43 100644 --- a/tests/integration_test/history_services_test.py +++ b/tests/integration_test/history_services_test.py @@ -8,7 +8,7 @@ from mongomock import MongoClient from kairon.history.processor import HistoryProcessor from pymongo.collection import Collection -import mock +from unittest import mock from urllib.parse import urlencode diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 8d99a6c06..5efd5c6ca 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -6,11 +6,11 @@ import tempfile from datetime import datetime, timedelta from io import BytesIO -from mock import patch +from unittest.mock import patch from urllib.parse import urljoin from zipfile import ZipFile -import mock +from unittest import mock import pytest import responses from botocore.exceptions import ClientError diff --git a/tests/unit_test/action/action_test.py b/tests/unit_test/action/action_test.py index 079c17a64..beb360969 100644 --- a/tests/unit_test/action/action_test.py +++ b/tests/unit_test/action/action_test.py @@ -3,7 +3,6 @@ import re from unittest import mock -import mock from googleapiclient.http import HttpRequest from pipedrive.exceptions import UnauthorizedError, BadRequestError from kairon.shared.utils import Utility diff --git a/tests/unit_test/chat/chat_test.py b/tests/unit_test/chat/chat_test.py index 9abbcab6b..6336152bb 100644 --- a/tests/unit_test/chat/chat_test.py +++ b/tests/unit_test/chat/chat_test.py @@ -3,7 +3,7 @@ import ujson as json import os from re import escape -from unittest.mock import patch +from unittest import mock from urllib.parse import urlencode, quote_plus import mongomock @@ -21,7 +21,6 @@ from kairon.shared.data.constant import ACCESS_ROLES, TOKEN_TYPE from kairon.shared.data.utils import DataUtility from kairon.shared.utils import Utility -import mock from pymongo.errors import ServerSelectionTimeoutError @@ -49,7 +48,7 @@ def test_save_channel_config_invalid(self): "test", "test" ) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -93,7 +92,7 @@ def test_save_channel_config_invalid(self): "test" ) - @patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) + @mock.patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) def test_save_channel_config_slack_team_id_error(self, mock_slack_info): mock_slack_info.side_effect = AppException("The request to the Slack API failed. ") with pytest.raises(AppException, match="The request to the Slack API failed.*"): @@ -108,7 +107,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -133,7 +132,7 @@ def __mock_get_bot(*args, **kwargs): "client_secret": "a23456789sfdghhtyutryuivcbn", "is_primary": True}}, "test", "test" ) - @patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) + @mock.patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) def test_save_channel_config_slack_secondary_app_team_id_error(self, mock_slack_info ): mock_slack_info.side_effect = AppException("The request to the Slack API failed. ") with pytest.raises(AppException, match="The request to the Slack API failed.*"): @@ -149,7 +148,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -252,7 +251,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -317,7 +316,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -384,7 +383,7 @@ def test_save_channel_config_telegram(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/telegram/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): ChatDataProcessor.save_channel_config({"connector_type": "telegram", "config": { "access_token": access_token, @@ -405,7 +404,7 @@ def test_save_channel_config_telegram_invalid(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/telegram/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): ChatDataProcessor.save_channel_config({"connector_type": "telegram", "config": { "access_token": access_token, @@ -487,7 +486,7 @@ def test_save_channel_config_business_messages_with_invalid_private_key(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/business_messages/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): channel_endpoint = ChatDataProcessor.save_channel_config( { "connector_type": "business_messages", @@ -515,7 +514,7 @@ def test_save_channel_config_business_messages(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/business_messages/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): channel_endpoint = ChatDataProcessor.save_channel_config( { "connector_type": "business_messages", @@ -603,7 +602,7 @@ def test_get_channel_end_point_whatsapp(self, monkeypatch): def _mock_generate_integration_token(*arge, **kwargs): return "testtoken", "ignore" - with patch.object(Authentication, "generate_integration_token", _mock_generate_integration_token): + with mock.patch.object(Authentication, "generate_integration_token", _mock_generate_integration_token): channel_url = ChatDataProcessor.save_channel_config({ "connector_type": "whatsapp", "config": { "app_secret": "app123", diff --git a/tests/unit_test/cli_test.py b/tests/unit_test/cli_test.py index 80f1e6bf2..1f28f1fd2 100644 --- a/tests/unit_test/cli_test.py +++ b/tests/unit_test/cli_test.py @@ -1,8 +1,10 @@ +import argparse +import os from datetime import datetime -from unittest.mock import patch +from unittest import mock import pytest -import os +from mongoengine import connect from kairon import cli from kairon.cli.conversations_deletion import initiate_history_deletion_archival @@ -10,22 +12,17 @@ from kairon.cli.delete_logs import delete_logs from kairon.cli.importer import validate_and_import from kairon.cli.message_broadcast import send_notifications -from kairon.cli.training import train from kairon.cli.testing import run_tests_on_model +from kairon.cli.training import train from kairon.cli.translator import translate_multilingual_bot from kairon.events.definitions.data_generator import DataGenerationEvent from kairon.events.definitions.data_importer import TrainingDataImporterEvent from kairon.events.definitions.history_delete import DeleteHistoryEvent -from kairon.events.definitions.message_broadcast import MessageBroadcastEvent from kairon.events.definitions.model_testing import ModelTestingEvent from kairon.events.definitions.multilingual import MultilingualEvent from kairon.shared.concurrency.actors.factory import ActorFactory -from kairon.shared.utils import Utility -from mongoengine import connect -import mock -import argparse - from kairon.shared.constants import EventClass +from kairon.shared.utils import Utility class TestTrainingCli: @@ -395,7 +392,7 @@ def test_message_broadcast_no_event_id(self, monkeypatch): return_value=argparse.Namespace(func=send_notifications, bot="test_cli", user="testUser", event_id="65432123456789876543")) def test_message_broadcast_all_arguments(self, mock_namespace): - with patch('kairon.events.definitions.message_broadcast.MessageBroadcastEvent.execute', autospec=True): + with mock.patch('kairon.events.definitions.message_broadcast.MessageBroadcastEvent.execute', autospec=True): cli() for proxy in ActorFactory._ActorFactory__actors.values(): diff --git a/tests/unit_test/data_processor/agent_processor_test.py b/tests/unit_test/data_processor/agent_processor_test.py index ec62c2dec..8d37b1541 100644 --- a/tests/unit_test/data_processor/agent_processor_test.py +++ b/tests/unit_test/data_processor/agent_processor_test.py @@ -16,7 +16,7 @@ from kairon.shared.data.constant import EVENT_STATUS from kairon.shared.data.model_processor import ModelProcessor -from mock import patch +from unittest.mock import patch from mongoengine import connect diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 7f3d138de..735bf069c 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -14,7 +14,7 @@ Utility.load_system_metadata() -from mock import patch +from unittest.mock import patch import numpy as np import pandas as pd import pytest diff --git a/tests/unit_test/data_processor/history_test.py b/tests/unit_test/data_processor/history_test.py index 3428736b9..8d7532bc2 100644 --- a/tests/unit_test/data_processor/history_test.py +++ b/tests/unit_test/data_processor/history_test.py @@ -2,7 +2,7 @@ import os from datetime import datetime -import mock +from unittest import mock import mongomock import pytest diff --git a/tests/unit_test/events/definitions_test.py b/tests/unit_test/events/definitions_test.py index aff9455f7..988b14072 100644 --- a/tests/unit_test/events/definitions_test.py +++ b/tests/unit_test/events/definitions_test.py @@ -4,11 +4,11 @@ from io import BytesIO from urllib.parse import urljoin -import mock +from unittest import mock import pytest import responses from fastapi import UploadFile -from mock.mock import patch +from unittest.mock import patch from mongoengine import connect from augmentation.utils import WebsiteParser diff --git a/tests/unit_test/events/events_test.py b/tests/unit_test/events/events_test.py index 4f7a63438..7f1448a9d 100644 --- a/tests/unit_test/events/events_test.py +++ b/tests/unit_test/events/events_test.py @@ -8,7 +8,7 @@ from unittest.mock import patch from urllib.parse import urljoin -import mock +from unittest import mock import mongomock import pytest import responses diff --git a/tests/unit_test/events/scheduler_test.py b/tests/unit_test/events/scheduler_test.py index ec6426415..678c11d2e 100644 --- a/tests/unit_test/events/scheduler_test.py +++ b/tests/unit_test/events/scheduler_test.py @@ -1,7 +1,7 @@ import os import re -from mock import patch +from unittest.mock import patch import pytest from apscheduler.jobstores.mongodb import MongoDBJobStore diff --git a/tests/unit_test/idp/test_idp_helper.py b/tests/unit_test/idp/test_idp_helper.py index db6ff9609..0d74cc842 100644 --- a/tests/unit_test/idp/test_idp_helper.py +++ b/tests/unit_test/idp/test_idp_helper.py @@ -17,7 +17,6 @@ from kairon.shared.organization.processor import OrgProcessor from kairon.shared.utils import Utility from stress_test.data_objects import User -from mock import patch def get_user(): diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index bcc2ed661..16daaec64 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1,13 +1,13 @@ import os -import time +from unittest import mock from urllib.parse import urljoin -import mock import numpy as np import pytest import ujson as json from aiohttp import ClientConnectionError from mongoengine import connect + from kairon.shared.utils import Utility Utility.load_system_metadata() @@ -18,7 +18,6 @@ from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT from kairon.shared.llm.processor import LLMProcessor -from kairon.shared.llm.data_objects import LLMLogs import litellm from deepdiff import DeepDiff @@ -1233,4 +1232,4 @@ async def test_llm_logging(self): stream=True, metadata={'user': user, 'bot': bot}) - assert str(e) == "Authentication error" + assert str(e) == "Authentication error" \ No newline at end of file diff --git a/tests/unit_test/plugins_test.py b/tests/unit_test/plugins_test.py index cc24f8f04..bb9de7443 100644 --- a/tests/unit_test/plugins_test.py +++ b/tests/unit_test/plugins_test.py @@ -1,7 +1,7 @@ import os import re -import mock +from unittest import mock import pytest import requests import responses @@ -11,7 +11,6 @@ from kairon.shared.constants import PluginTypes from kairon.shared.plugins.factory import PluginFactory from kairon.shared.utils import Utility -from mongomock import MongoClient class TestUtility: diff --git a/tests/unit_test/rest_client_test.py b/tests/unit_test/rest_client_test.py index 0d6afa160..6daa1e5e4 100644 --- a/tests/unit_test/rest_client_test.py +++ b/tests/unit_test/rest_client_test.py @@ -1,4 +1,5 @@ import asyncio +import ujson as json from unittest import mock import pytest @@ -101,3 +102,19 @@ async def test_aio_rest_client_timeout_error(self, aioresponses): with pytest.raises(AppException, match="Request timed out: Request timed out"): await AioRestClient().request("get", url, request_body={"name": "udit.pandey", "loc": "blr"}, headers={"Authorization": "Bearer sasdfghjkytrtyui"}, max_retries=3) + + @pytest.mark.asyncio + async def test_aio_rest_client_post_request_stream(self, aioresponses): + url = 'http://kairon.com' + aioresponses.post("http://kairon.com", status=200, body=json.dumps({'data': 'hi!'})) + resp = await AioRestClient().request("post", url, request_body={"name": "udit.pandey", "loc": "blr"}, + headers={"Authorization": "Bearer sasdfghjkytrtyui"}, is_streaming_resp=True) + response = '' + async for content in resp.content: + response += content.decode() + + assert json.loads(response) == {"data": "hi!"} + assert list(aioresponses.requests.values())[0][0].kwargs == {'allow_redirects': True, 'headers': { + 'Authorization': 'Bearer sasdfghjkytrtyui'}, 'json': {'loc': 'blr', 'name': 'udit.pandey'}, 'timeout': None, + 'data': None, + 'trace_request_ctx': {'current_attempt': 1}} \ No newline at end of file diff --git a/tests/unit_test/verification_test.py b/tests/unit_test/verification_test.py index 4e6c2b360..6462ae579 100644 --- a/tests/unit_test/verification_test.py +++ b/tests/unit_test/verification_test.py @@ -2,7 +2,7 @@ import responses from kairon.shared.verification.email import QuickEmailVerification from urllib.parse import urlencode -import mock +from unittest import mock from kairon.shared.utils import Utility import os From 72463135cb5affd2a5870f6d934fe1405567b3e9 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 13:38:29 +0530 Subject: [PATCH 20/57] test cases fixed --- kairon/shared/llm/logger.py | 3 ++- tests/unit_test/llm_test.py | 46 +++++++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py index dbb93e469..3af7a7870 100644 --- a/kairon/shared/llm/logger.py +++ b/kairon/shared/llm/logger.py @@ -37,4 +37,5 @@ def __logs_litellm(self, **kwargs): 'metadata': litellm_params.get('metadata')}) def __save_logs(self, **kwargs): - print(LLMLogs(**kwargs).save().to_mongo().to_dict()) + logs = LLMLogs(**kwargs).save().to_mongo().to_dict() + logger.info(f"llm logs: {logs}") diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 16daaec64..1042be91d 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1179,38 +1179,50 @@ async def test_llm_logging(self): user = "test" litellm.callbacks = [LiteLLMLogger()] - result = await litellm.acompletion(messages=["Hi"], + messages = [{"role":"user", "content":"Hi"}] + expected = "Hi, How may i help you?" + + result = await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, metadata={'user': user, 'bot': bot}) - assert result + assert result['choices'][0]['message']['content'] == expected - result = litellm.completion(messages=["Hi"], + result = litellm.completion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, metadata={'user': user, 'bot': bot}) - assert result + assert result['choices'][0]['message']['content'] == expected - result = litellm.completion(messages=["Hi"], + result = litellm.completion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, stream=True, metadata={'user': user, 'bot': bot}) + response = '' for chunk in result: - print(chunk["choices"][0]["delta"]["content"]) - assert chunk["choices"][0]["delta"]["content"] + content = chunk["choices"][0]["delta"]["content"] + if content: + response = response + content + + assert response == expected - result = await litellm.acompletion(messages=["Hi"], + result = await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, stream=True, metadata={'user': user, 'bot': bot}) + response = '' async for chunk in result: - print(chunk["choices"][0]["delta"]["content"]) - assert chunk["choices"][0]["delta"]["content"] + content = chunk["choices"][0]["delta"]["content"] + print(chunk) + if content: + response += content + + assert response.__contains__(expected) with pytest.raises(Exception) as e: - await litellm.acompletion(messages=["Hi"], + await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), metadata={'user': user, 'bot': bot}) @@ -1218,7 +1230,7 @@ async def test_llm_logging(self): assert str(e) == "Authentication error" with pytest.raises(Exception) as e: - litellm.completion(messages=["Hi"], + litellm.completion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), metadata={'user': user, 'bot': bot}) @@ -1226,7 +1238,7 @@ async def test_llm_logging(self): assert str(e) == "Authentication error" with pytest.raises(Exception) as e: - await litellm.acompletion(messages=["Hi"], + await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), stream=True, From ab31dd254d85b3b0db1e4eb872b0eac230f25376 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 14:53:59 +0530 Subject: [PATCH 21/57] removed unused variable --- kairon/actions/definitions/prompt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 4c7bf6bc4..6323589b9 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -66,7 +66,6 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma k_faq_action_config, bot_settings = self.retrieve_config() user_question = k_faq_action_config.get('user_question') user_msg = self.__get_user_msg(tracker, user_question) - llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) llm_processor = LLMProcessor(self.bot) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, From 6090cf4e3dbb0e0e4c2cdbd480940a91dfd80f5b Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 15:15:10 +0530 Subject: [PATCH 22/57] fixed unused variable --- kairon/actions/definitions/prompt.py | 3 +- kairon/shared/llm/processor.py | 3 +- kairon/shared/vector_embeddings/db/qdrant.py | 6 ++-- kairon/train.py | 4 +-- tests/unit_test/llm_test.py | 36 ++++++++++---------- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 6323589b9..6441d607f 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -66,8 +66,9 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma k_faq_action_config, bot_settings = self.retrieve_config() user_question = k_faq_action_config.get('user_question') user_msg = self.__get_user_msg(tracker, user_question) + llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) - llm_processor = LLMProcessor(self.bot) + llm_processor = LLMProcessor(self.bot, llm_type) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, user=tracker.sender_id, **llm_params) diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index c6e7fa8af..adbb039ae 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -26,13 +26,14 @@ class LLMProcessor(LLMBase): __embedding__ = 1536 - def __init__(self, bot: Text): + def __init__(self, bot: Text, llm_type: str): super().__init__(bot) self.db_url = Utility.environment['vector']['db'] self.headers = {} if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.suffix = "_faq_embd" + self.llm_type = llm_type self.vector_config = {'size': self.__embedding__, 'distance': 'Cosine'} self.api_key = Sysadmin.get_bot_secret(bot, BotSecretType.gpt_key.value, raise_err=True) self.tokenizer = get_encoding("cl100k_base") diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index 12b5268e2..454eeb8fe 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -5,10 +5,8 @@ from kairon import Utility from kairon.shared.actions.utils import ActionUtility -from kairon.shared.admin.constants import BotSecretType -from kairon.shared.admin.processor import Sysadmin -from kairon.shared.constants import GPT3ResourceTypes from kairon.shared.llm.processor import LLMProcessor +from kairon.shared.data.constant import DEFAULT_LLM from kairon.shared.vector_embeddings.db.base import VectorEmbeddingsDbBase @@ -25,7 +23,7 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.llm_settings = llm_settings - self.llm = LLMProcessor(self.bot) + self.llm = LLMProcessor(self.bot, DEFAULT_LLM) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 diff --git a/kairon/train.py b/kairon/train.py index 3ddcf9eb0..d8360ce67 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -6,7 +6,7 @@ from rasa.api import train from rasa.model import DEFAULT_MODELS_PATH from rasa.shared.constants import DEFAULT_CONFIG_PATH, DEFAULT_DATA_PATH, DEFAULT_DOMAIN_PATH - +from kairon.shared.data.constant import DEFAULT_LLM from kairon.chat.agent.agent import KaironAgent from kairon.exceptions import AppException from kairon.shared.account.processor import AccountProcessor @@ -101,7 +101,7 @@ def start_training(bot: str, user: str, token: str = None): settings = processor.get_bot_settings(bot, user) settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: - llm_processor = LLMProcessor(bot) + llm_processor = LLMProcessor(bot, DEFAULT_LLM) faqs = asyncio.run(llm_processor.train(user=user)) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 1042be91d..a385106cd 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -16,7 +16,7 @@ from kairon.shared.admin.constants import BotSecretType from kairon.shared.admin.data_objects import BotSecrets from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema -from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT +from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_LLM from kairon.shared.llm.processor import LLMProcessor import litellm from deepdiff import DeepDiff @@ -44,7 +44,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM)[[]] aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -123,7 +123,7 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore embedding = list(np.random.random(LLMProcessor.__embedding__)) mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, { 'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -227,7 +227,7 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a input = {"name": "Ram", "color": "red"} mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), @@ -288,7 +288,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): input = {"name": "Ram", "color": "red"} mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -339,7 +339,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): def test_gpt3_faq_embedding_train_failure(self): with pytest.raises(AppException, match=f"Bot secret '{BotSecretType.gpt_key.value}' not configured!"): - LLMProcessor('test_failure') + LLMProcessor('test_failure', DEFAULT_LLM) @pytest.mark.asyncio @mock.patch.object(litellm, "aembedding", autospec=True) @@ -357,7 +357,7 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aiore mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -421,7 +421,7 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -507,7 +507,7 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -574,7 +574,7 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_emb mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -641,7 +641,7 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'vector': {"key": "test", 'db': "http://localhost:6333"}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -729,7 +729,7 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe ] with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -832,7 +832,7 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content1.collection}{gpt3.suffix}/points/search"), @@ -908,7 +908,7 @@ def __mock_connection_error(*args, **kwargs): mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.side_effect = __mock_connection_error - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -971,7 +971,7 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_llm_request.side_effect = ClientConnectionError() - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} @@ -1009,7 +1009,7 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ } mock_embedding.side_effect = [Exception("Connection reset by peer!"), {'data': [{'embedding': embedding}]}] - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) @@ -1064,7 +1064,7 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -1143,7 +1143,7 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, { 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], From cdc9dfab43f6de8c810fd8543ba845b362e84b83 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 16:28:11 +0530 Subject: [PATCH 23/57] fixed unused variable --- tests/unit_test/llm_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index a385106cd..53c31bace 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -44,7 +44,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM)[[]] + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), From e04f8d5c4ce0c549ae18331929fd1672dcee1bd2 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 28 Jun 2024 12:56:08 +0530 Subject: [PATCH 24/57] added test cased for fetching logs --- kairon/api/app/routers/bot/bot.py | 23 +++++++++++-- kairon/shared/llm/processor.py | 24 +++++++++++++ tests/integration_test/services_test.py | 45 ++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/kairon/api/app/routers/bot/bot.py b/kairon/api/app/routers/bot/bot.py index 6dfeda5c5..276404377 100644 --- a/kairon/api/app/routers/bot/bot.py +++ b/kairon/api/app/routers/bot/bot.py @@ -26,7 +26,7 @@ from kairon.shared.actions.data_objects import ActionServerLogs from kairon.shared.auth import Authentication from kairon.shared.constants import TESTER_ACCESS, DESIGNER_ACCESS, CHAT_ACCESS, UserActivityType, ADMIN_ACCESS, \ - VIEW_ACCESS, EventClass, AGENT_ACCESS + EventClass, AGENT_ACCESS from kairon.shared.data.assets_processor import AssetsProcessor from kairon.shared.data.audit.processor import AuditDataProcessor from kairon.shared.data.constant import EVENT_STATUS, ENDPOINT_TYPE, TOKEN_TYPE, ModelTestType, \ @@ -38,10 +38,12 @@ from kairon.shared.data.utils import DataUtility from kairon.shared.importer.data_objects import ValidationLogs from kairon.shared.importer.processor import DataImporterLogProcessor +from kairon.shared.live_agent.live_agent import LiveAgentHandler from kairon.shared.models import User, TemplateType from kairon.shared.test.processor import ModelTestingLogProcessor from kairon.shared.utils import Utility -from kairon.shared.live_agent.live_agent import LiveAgentHandler +from kairon.shared.llm.processor import LLMProcessor +from kairon.shared.llm.data_objects import LLMLogs router = APIRouter() @@ -1668,3 +1670,20 @@ async def get_live_agent_token(current_user: User = Security(Authentication.get_ data = await LiveAgentHandler.authenticate_agent(current_user.get_user(), current_user.get_bot()) return Response(data=data) + +@router.get("/llm/logs", response_model=Response) +async def get_llm_logs( + start_idx: int = 0, page_size: int = 10, + current_user: User = Security(Authentication.get_current_user_and_bot, scopes=TESTER_ACCESS) +): + """ + Get data llm event logs. + """ + logs = list(LLMProcessor.get_logs(current_user.get_bot(), start_idx, page_size)) + row_cnt = LLMProcessor.get_row_count(current_user.get_bot()) + data = { + "logs": logs, + "total": row_cnt + } + return Response(data=data) + diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index adbb039ae..7365e0aa7 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -16,6 +16,7 @@ from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_CONTEXT_PROMPT from kairon.shared.llm.base import LLMBase from kairon.shared.llm.logger import LiteLLMLogger +from kairon.shared.llm.data_objects import LLMLogs from kairon.shared.models import CognitionDataType from kairon.shared.rest_client import AioRestClient from kairon.shared.utils import Utility @@ -260,3 +261,26 @@ async def __attach_similarity_prompt_if_enabled(self, query_embedding, context_p similarity_context = f"Instructions on how to use {similarity_prompt_name}:\n{extracted_values}\n{similarity_prompt_instructions}\n" context_prompt = f"{context_prompt}\n{similarity_context}" return context_prompt + + @staticmethod + def get_logs(bot: str, start_idx: int = 0, page_size: int = 10): + """ + Get all logs for data importer event. + @param bot: bot id. + @param start_idx: start index + @param page_size: page size + @return: list of logs. + """ + for log in LLMLogs.objects(metadata__bot=bot).order_by("-start_time").skip(start_idx).limit(page_size): + llm_log = log.to_mongo().to_dict() + llm_log.pop('_id') + yield llm_log + + @staticmethod + def get_row_count(bot: str): + """ + Gets the count of rows in a LLMLogs for a particular bot. + :param bot: bot id + :return: Count of rows + """ + return LLMLogs.objects(metadata__bot=bot).count() diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 5efd5c6ca..62fb261b3 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -1,3 +1,5 @@ +import time + import ujson as json import os import re @@ -23352,6 +23354,47 @@ def test_trigger_widget(): assert actual["error_code"] == 0 assert len(actual["data"]) == 2 assert not actual["message"] + + +def test_get_llm_logs(): + from kairon.shared.llm.logger import LiteLLMLogger + import litellm + import asyncio + + loop = asyncio.new_event_loop() + user = "test" + litellm.callbacks = [LiteLLMLogger()] + + messages = [{"role": "user", "content": "Hi"}] + expected = "Hi, How may i help you?" + + result = loop.run_until_complete(litellm.acompletion(messages=messages, + model="gpt-3.5-turbo", + mock_response=expected, + metadata={'user': user, 'bot': pytest.bot})) + assert result['choices'][0]['message']['content'] == expected + + time.sleep(2) + + response = client.get( + f"/api/bot/{pytest.bot}/llm/logs?start_idx=0&page_size=10", + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + print(actual) + assert actual["success"] + assert actual["error_code"] == 0 + assert len(actual["data"]["logs"]) == 1 + assert actual["data"]["total"] == 1 + assert actual["data"]["logs"][0]['start_time'] + assert actual["data"]["logs"][0]['end_time'] + assert actual["data"]["logs"][0]['cost'] + assert actual["data"]["logs"][0]['llm_call_id'] + assert actual["data"]["logs"][0]["llm_provider"] == "openai" + assert not actual["data"]["logs"][0].get("model") + assert actual["data"]["logs"][0]["model_params"] == {} + assert actual["data"]["logs"][0]["metadata"]['bot'] == pytest.bot + assert actual["data"]["logs"][0]["metadata"]['user'] == "test" def test_add_custom_widget_invalid_config(): @@ -24251,4 +24294,4 @@ def test_list_system_metadata(): actual = response.json() assert actual["error_code"] == 0 assert actual["success"] - assert len(actual["data"]) == 17 + assert len(actual["data"]) == 17 \ No newline at end of file From 866384652625f68c0ca05fd86a67e786c8a33fbc Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 28 Jun 2024 15:26:15 +0530 Subject: [PATCH 25/57] added test cased for fetching logs --- tests/unit_test/data_processor/data_processor_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 735bf069c..fd3f19b54 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -81,7 +81,6 @@ from kairon.shared.data.utils import DataUtility from kairon.shared.importer.processor import DataImporterLogProcessor from kairon.shared.live_agent.live_agent import LiveAgentHandler -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from kairon.shared.metering.constants import MetricType from kairon.shared.metering.data_object import Metering from kairon.shared.models import StoryEventType, HttpContentType, CognitionDataType From bcd7a5ee4a642cf3e171db7cbc4bf310d3feb990 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 28 Jun 2024 15:45:20 +0530 Subject: [PATCH 26/57] removed unused import --- kairon/api/app/routers/bot/bot.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kairon/api/app/routers/bot/bot.py b/kairon/api/app/routers/bot/bot.py index 276404377..839ae0fe8 100644 --- a/kairon/api/app/routers/bot/bot.py +++ b/kairon/api/app/routers/bot/bot.py @@ -39,12 +39,10 @@ from kairon.shared.importer.data_objects import ValidationLogs from kairon.shared.importer.processor import DataImporterLogProcessor from kairon.shared.live_agent.live_agent import LiveAgentHandler +from kairon.shared.llm.processor import LLMProcessor from kairon.shared.models import User, TemplateType from kairon.shared.test.processor import ModelTestingLogProcessor from kairon.shared.utils import Utility -from kairon.shared.llm.processor import LLMProcessor -from kairon.shared.llm.data_objects import LLMLogs - router = APIRouter() v2 = APIRouter() From f8545c61ff803d5929c096f25839f8ba9b6135db Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 28 Jun 2024 20:07:49 +0530 Subject: [PATCH 27/57] added invocation in metadata for litellm --- kairon/actions/definitions/prompt.py | 1 + kairon/shared/llm/processor.py | 25 +++++--- kairon/shared/vector_embeddings/db/qdrant.py | 2 +- kairon/train.py | 2 +- tests/integration_test/action_service_test.py | 24 +++---- tests/unit_test/llm_test.py | 62 +++++++++---------- 6 files changed, 62 insertions(+), 54 deletions(-) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 6441d607f..2d3b5257b 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -71,6 +71,7 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma llm_processor = LLMProcessor(self.bot, llm_type) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, user=tracker.sender_id, + invocation='prompt_action', **llm_params) status = "FAILURE" if llm_response.get("is_failure", False) is True else status exception = llm_response.get("exception") diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index 7365e0aa7..5b6831fe3 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -42,6 +42,7 @@ def __init__(self, bot: Text, llm_type: str): self.__logs = [] async def train(self, user, *args, **kwargs) -> Dict: + invocation = kwargs.pop('invocation', None) await self.__delete_collections() count = 0 processor = CognitionDataProcessor() @@ -61,7 +62,7 @@ async def train(self, user, *args, **kwargs) -> Dict: content['data'], metadata) else: search_payload, embedding_payload = {'content': content["data"]}, content["data"] - embeddings = await self.get_embedding(embedding_payload, user) + embeddings = await self.get_embedding(embedding_payload, user, invocation=invocation) points = [{'id': content['vector_id'], 'vector': embeddings, 'payload': search_payload}] await self.__collection_upsert__(collection, {'points': points}, err_msg="Unable to train FAQ! Contact support") @@ -71,15 +72,16 @@ async def train(self, user, *args, **kwargs) -> Dict: async def predict(self, query: Text, user, *args, **kwargs) -> Tuple: start_time = time.time() embeddings_created = False + invocation = kwargs.pop('invocation', None) try: - query_embedding = await self.get_embedding(query, user) + query_embedding = await self.get_embedding(query, user, invocation=invocation) embeddings_created = True system_prompt = kwargs.pop('system_prompt', DEFAULT_SYSTEM_PROMPT) context_prompt = kwargs.pop('context_prompt', DEFAULT_CONTEXT_PROMPT) context = await self.__attach_similarity_prompt_if_enabled(query_embedding, context_prompt, **kwargs) - answer = await self.__get_answer(query, system_prompt, context, user, **kwargs) + answer = await self.__get_answer(query, system_prompt, context, user, invocation=invocation,**kwargs) response = {"content": answer} except Exception as e: logging.exception(e) @@ -101,11 +103,11 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def get_embedding(self, text: Text, user) -> List[float]: + async def get_embedding(self, text: Text, user, **kwargs) -> List[float]: truncated_text = self.truncate_text(text) result = await litellm.aembedding(model="text-embedding-3-small", input=[truncated_text], - metadata={'user': user, 'bot': self.bot}, + metadata={'user': user, 'bot': self.bot, 'invocation': kwargs.get("invocation")}, api_key=self.api_key, num_retries=3) return result["data"][0]["embedding"] @@ -123,7 +125,7 @@ async def __parse_completion_response(self, response, **kwargs): async def __get_completion(self, messages, hyperparameters, user, **kwargs): response = await litellm.acompletion(messages=messages, - metadata={'user': user, 'bot': self.bot}, + metadata={'user': user, 'bot': self.bot, 'invocation': kwargs.get("invocation")}, api_key=self.api_key, num_retries=3, **hyperparameters) @@ -134,6 +136,7 @@ async def __get_completion(self, messages, hyperparameters, user, **kwargs): async def __get_answer(self, query, system_prompt: Text, context: Text, user, **kwargs): use_query_prompt = False query_prompt = '' + invocation = kwargs.pop('invocation') if kwargs.get('query_prompt', {}): query_prompt_dict = kwargs.pop('query_prompt') query_prompt = query_prompt_dict.get('query_prompt', '') @@ -146,7 +149,8 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, ** if use_query_prompt and query_prompt: query = await self.__rephrase_query(query, system_prompt, query_prompt, hyperparameters=hyperparameters, - user=user) + user=user, + invocation=f"{invocation}_rephrase") messages = [ {"role": "system", "content": system_prompt}, ] @@ -157,12 +161,14 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, ** completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user) + user=user, + invocation=invocation) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'answer_query', 'hyperparameters': hyperparameters}) return completion async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, **kwargs): + invocation = kwargs.pop('invocation') messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": f"{query_prompt}\n\n Q: {query}\n A:"} @@ -171,7 +177,8 @@ async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user) + user=user, + invocation=invocation) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'rephrase_query', 'hyperparameters': hyperparameters}) return completion diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index 454eeb8fe..400e3103c 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -28,7 +28,7 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: self.EMBEDDING_CTX_LENGTH = 8191 async def __get_embedding(self, text: Text, user: str, **kwargs) -> List[float]: - return await self.llm.get_embedding(text, user=user) + return await self.llm.get_embedding(text, user=user, invocation='db_action_qdrant') async def embedding_search(self, request_body: Dict, user: str, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points") diff --git a/kairon/train.py b/kairon/train.py index d8360ce67..aa19943cb 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -102,7 +102,7 @@ def start_training(bot: str, user: str, token: str = None): settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: llm_processor = LLMProcessor(bot, DEFAULT_LLM) - faqs = asyncio.run(llm_processor.train(user=user)) + faqs = asyncio.run(llm_processor.train(user=user, invocation='model_training')) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) agent_url = Utility.environment['model']['agent'].get('url') diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index cd5e2c63c..43f5e93ca 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -10495,7 +10495,7 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], - 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10562,7 +10562,7 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], - 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10632,7 +10632,7 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], - 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10884,7 +10884,7 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m ] expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above.\n \nQ: What kind of language is python? \nA:"}], - 'metadata': {'user': 'udit.pandeyy', 'bot': '5f50k90a56b698ca10d35d2e'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandeyy', 'bot': '5f50k90a56b698ca10d35d2e', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': True, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args.kwargs, expected, ignore_order=True) @@ -11167,7 +11167,7 @@ def __mock_fetch_similar(*args, **kwargs): assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], - 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11252,7 +11252,7 @@ def mock_completion_for_answer(*args, **kwargs): assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], - 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11376,7 +11376,7 @@ def __mock_fetch_similar(*args, **kwargs): assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], - 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11483,7 +11483,7 @@ def mock_completion_for_answer(*args, **kwargs): assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], - 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11581,7 +11581,7 @@ def __mock_fetch_similar(*args, **kwargs): assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], - 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11645,7 +11645,7 @@ def __mock_fetch_similar(*args, **kwargs): ] expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "\nInstructions on how to use Similarity Prompt:\n['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Kanban And Scrum Together? \nA:"}], - 'metadata': {'user': user, 'bot': bot}, 'api_key': value, + 'metadata': {'user': user, 'bot': bot, 'invocation': 'prompt_action'}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11709,7 +11709,7 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], - 'metadata': {'user': 'udit.pandey', 'bot': bot}, 'api_key': value, + 'metadata': {'user': 'udit.pandey', 'bot': bot, 'invocation': 'prompt_action'}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11776,7 +11776,7 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], - 'metadata': {'user': user, 'bot': bot}, 'api_key': value, + 'metadata': {'user': user, 'bot': bot, 'invocation': 'prompt_action'}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 53c31bace..bf6b54d50 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -80,7 +80,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): }]} expected = {"model": "text-embedding-3-small", - "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, + "input": [test_content.data], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -198,7 +198,7 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore assert response['faq'] == 3 expected = {"model": "text-embedding-3-small", - "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, + "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} print(mock_embedding.call_args) @@ -260,7 +260,7 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a }]} expected = {"model": "text-embedding-3-small", - "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot}, + "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -332,7 +332,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): }]} expected = {"model": "text-embedding-3-small", - "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot}, + "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -393,7 +393,7 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aiore 'vector': embedding, 'payload': {'content': test_content.data}}]} expected = {"model": "text-embedding-3-small", - "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, + "input": [test_content.data], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -464,7 +464,7 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb }]} expected = {"model": "text-embedding-3-small", - "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, + "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -525,12 +525,12 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': bot} + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} expected['api_key'] = value expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -594,12 +594,12 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_emb assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': bot} + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} expected['api_key'] = value expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -674,12 +674,12 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot}, + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': gpt3.bot} + expected['metadata'] = {'user': user, 'bot': gpt3.bot, 'invocation': None} expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -761,12 +761,12 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot}, + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': gpt3.bot} + expected['metadata'] = {'user': user, 'bot': gpt3.bot, 'invocation': None} expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -863,12 +863,12 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': bot} + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -929,7 +929,7 @@ def __mock_connection_error(*args, **kwargs): assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -937,7 +937,7 @@ def __mock_connection_error(*args, **kwargs): 'content': 'You are a personal assistant. Answer the question according to the below context'}, {'role': 'user', 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], - 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error'}, + 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error', 'invocation': None}, 'api_key': 'test', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} @@ -981,7 +981,7 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -1019,7 +1019,7 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -1084,12 +1084,12 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': bot} + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -1162,12 +1162,12 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': bot} + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -1185,20 +1185,20 @@ async def test_llm_logging(self): result = await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", mock_response=expected, - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) assert result['choices'][0]['message']['content'] == expected result = litellm.completion(messages=messages, model="gpt-3.5-turbo", mock_response=expected, - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) assert result['choices'][0]['message']['content'] == expected result = litellm.completion(messages=messages, model="gpt-3.5-turbo", mock_response=expected, stream=True, - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) response = '' for chunk in result: content = chunk["choices"][0]["delta"]["content"] @@ -1211,7 +1211,7 @@ async def test_llm_logging(self): model="gpt-3.5-turbo", mock_response=expected, stream=True, - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) response = '' async for chunk in result: content = chunk["choices"][0]["delta"]["content"] @@ -1225,7 +1225,7 @@ async def test_llm_logging(self): await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) assert str(e) == "Authentication error" @@ -1233,7 +1233,7 @@ async def test_llm_logging(self): litellm.completion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) assert str(e) == "Authentication error" @@ -1242,6 +1242,6 @@ async def test_llm_logging(self): model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), stream=True, - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) assert str(e) == "Authentication error" \ No newline at end of file From 2875c2ff637a6edafd1574079934cb720a8bd0e3 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Mon, 1 Jul 2024 17:19:25 +0530 Subject: [PATCH 28/57] 1. changed rasa rule policy to allow max history 2. changed rasa domain.yml schemas to allow unicode Alphabets for slots and form name --- docker/Dockerfile | 2 + kairon/shared/schemas/domain.yml | 142 +++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 kairon/shared/schemas/domain.yml diff --git a/docker/Dockerfile b/docker/Dockerfile index 58f48e6ba..4b9cadd30 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,6 +38,8 @@ COPY . . RUN rm -rf ${TEMPLATE_DIR_DEFAULT}/models/* && \ rasa train --data ${TEMPLATE_DIR_DEFAULT}/data --config ${TEMPLATE_DIR_DEFAULT}/config.yml --domain ${TEMPLATE_DIR_DEFAULT}/domain.yml --out ${TEMPLATE_DIR_DEFAULT}/models +RUN cp kairon/shared/rule_policy.py /usr/local/lib/python3.10/site-packages/rasa/core/policies/rule_policy.py +RUN cp kairon/shared/schemas/domain.yml /usr/local/lib/python3.10/site-packages/rasa/shared/utils/schemas/domain.yml ENV HF_HOME="/home/cache" SENTENCE_TRANSFORMERS_HOME="/home/cache" diff --git a/kairon/shared/schemas/domain.yml b/kairon/shared/schemas/domain.yml new file mode 100644 index 000000000..b5ceed4c3 --- /dev/null +++ b/kairon/shared/schemas/domain.yml @@ -0,0 +1,142 @@ +allowempty: True +mapping: + version: + type: "str" + required: False + allowempty: False + intents: + type: "seq" + sequence: + - type: "map" + mapping: + use_entities: + type: "any" + ignore_entities: + type: "any" + allowempty: True + - type: "str" + entities: + type: "seq" + matching: "any" + sequence: + - type: "map" + mapping: + roles: + type: "seq" + sequence: + - type: "str" + groups: + type: "seq" + sequence: + - type: "str" + allowempty: True + - type: "str" + actions: + type: seq + matching: "any" + seq: + - type: str + - type: map + mapping: + regex;([A-Za-z]+): + type: map + mapping: + send_domain: + type: "bool" + responses: + # see shared/nlu/training_data/schemas/responses.yml + include: responses + + slots: + type: "map" + allowempty: True + mapping: + regex;([A-Za-z\u00C0-\u017F\u0400-\u04FF\u0370-\u03FF\u0530-\u058F\u0590-\u05FF\u0600-\u06FF\u0980-\u09FF\u0A80-\u0AFF\u0B80-\u0BFF\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF]+): + type: "map" + allowempty: True + mapping: + influence_conversation: + type: "bool" + required: False + type: + type: "any" + required: True + values: + type: "seq" + sequence: + - type: "any" + required: False + min_value: + type: "number" + required: False + max_value: + type: "number" + required: False + initial_value: + type: "any" + required: False + mappings: + type: "seq" + required: True + allowempty: False + sequence: + - type: "map" + allowempty: True + mapping: + type: + type: "str" + intent: + type: "any" + not_intent: + type: "any" + entity: + type: "str" + role: + type: "str" + group: + type: "str" + value: + type: "any" + action: + type: "str" + conditions: + type: "seq" + sequence: + - type: "map" + mapping: + active_loop: + type: "str" + nullable: True + requested_slot: + type: "str" + forms: + type: "map" + required: False + mapping: + regex;([A-Za-z\u00C0-\u017F\u0400-\u04FF\u0370-\u03FF\u0530-\u058F\u0590-\u05FF\u0600-\u06FF\u0980-\u09FF\u0A80-\u0AFF\u0B80-\u0BFF\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF]+): + type: "map" + mapping: + required_slots: + type: "seq" + sequence: + - type: str + required: False + allowempty: True + ignored_intents: + type: any + config: + type: "map" + allowempty: True + mapping: + store_entities_as_slots: + type: "bool" + session_config: + type: "map" + allowempty: True + mapping: + session_expiration_time: + type: "number" + range: + min: 0 + carry_over_slots_to_new_session: + type: "bool" From a710ddd3622d3f991e02940d7796687117276fd2 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 21 Jun 2024 20:21:05 +0530 Subject: [PATCH 29/57] litellm base version --- augmentation/paraphrase/gpt3/gpt.py | 7 +- custom/__init__.py | 0 custom/fallback.py | 58 - custom/ner.py | 169 --- kairon/actions/definitions/database.py | 2 +- kairon/actions/definitions/prompt.py | 31 +- kairon/api/models.py | 24 +- kairon/chat/agent/message_processor.py | 9 - kairon/importer/validator/file_validator.py | 31 +- kairon/shared/actions/data_objects.py | 8 +- .../concurrency/actors/pyscript_runner.py | 10 +- kairon/shared/data/constant.py | 1 + kairon/shared/data/processor.py | 4 +- kairon/shared/llm/base.py | 4 +- kairon/shared/llm/clients/__init__.py | 0 kairon/shared/llm/clients/azure.py | 25 - kairon/shared/llm/clients/base.py | 8 - kairon/shared/llm/clients/factory.py | 18 - kairon/shared/llm/clients/gpt3.py | 92 -- kairon/shared/llm/data_objects.py | 13 + kairon/shared/llm/factory.py | 17 - kairon/shared/llm/logger.py | 43 + kairon/shared/llm/{gpt3.py => processor.py} | 106 +- kairon/shared/utils.py | 85 +- kairon/shared/vector_embeddings/db/base.py | 8 +- kairon/shared/vector_embeddings/db/qdrant.py | 18 +- kairon/train.py | 7 +- metadata/integrations.yml | 127 +- requirements/dev.txt | 13 +- requirements/prod.txt | 68 +- tests/integration_test/action_service_test.py | 1066 ++++++++++------- tests/integration_test/chat_service_test.py | 10 +- tests/integration_test/services_test.py | 24 +- tests/unit_test/action/action_test.py | 6 +- tests/unit_test/api/api_processor_test.py | 9 +- .../augmentation/gpt_augmentation_test.py | 10 +- .../data_processor/data_processor_test.py | 111 +- tests/unit_test/events/events_test.py | 11 +- tests/unit_test/llm_test.py | 791 ++++++------ tests/unit_test/utility_test.py | 745 +----------- .../validator/training_data_validator_test.py | 73 +- training_data/ReadMe.md | 1 - 42 files changed, 1481 insertions(+), 2382 deletions(-) delete mode 100644 custom/__init__.py delete mode 100644 custom/fallback.py delete mode 100644 custom/ner.py delete mode 100644 kairon/shared/llm/clients/__init__.py delete mode 100644 kairon/shared/llm/clients/azure.py delete mode 100644 kairon/shared/llm/clients/base.py delete mode 100644 kairon/shared/llm/clients/factory.py delete mode 100644 kairon/shared/llm/clients/gpt3.py create mode 100644 kairon/shared/llm/data_objects.py delete mode 100644 kairon/shared/llm/factory.py create mode 100644 kairon/shared/llm/logger.py rename kairon/shared/llm/{gpt3.py => processor.py} (73%) delete mode 100644 training_data/ReadMe.md diff --git a/augmentation/paraphrase/gpt3/gpt.py b/augmentation/paraphrase/gpt3/gpt.py index 340c220c0..b11a8eac5 100644 --- a/augmentation/paraphrase/gpt3/gpt.py +++ b/augmentation/paraphrase/gpt3/gpt.py @@ -1,7 +1,7 @@ """Creates the Example and GPT classes for a user to interface with the OpenAI API.""" -import openai +from openai import OpenAI import uuid @@ -95,8 +95,9 @@ def submit_request(self, prompt, num_responses, api_key): """Calls the OpenAI API with the specified parameters.""" if num_responses < 1: num_responses = 1 - response = openai.Completion.create(api_key=api_key, - engine=self.get_engine(), + client = OpenAI(api_key=api_key) + response = client.completions.create( + model=self.get_engine(), prompt=self.craft_query(prompt), max_tokens=self.get_max_tokens(), temperature=self.get_temperature(), diff --git a/custom/__init__.py b/custom/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/custom/fallback.py b/custom/fallback.py deleted file mode 100644 index 4c396c039..000000000 --- a/custom/fallback.py +++ /dev/null @@ -1,58 +0,0 @@ -''' -Custom component to get fallback action intent -Reference: https://forum.rasa.com/t/fallback-intents-for-context-sensitive-fallbacks/963 -''' - -from rasa.nlu.classifiers.classifier import IntentClassifier - -class FallbackIntentFilter(IntentClassifier): - - # Name of the component to be used when integrating it in a - # pipeline. E.g. ``[ComponentA, ComponentB]`` - # will be a proper pipeline definition where ``ComponentA`` - # is the name of the first component of the pipeline. - name = "FallbackIntentFilter" - - # Defines what attributes the pipeline component will - # provide when called. The listed attributes - # should be set by the component on the message object - # during test and train, e.g. - # ```message.set("entities", [...])``` - provides = [] - - # Which attributes on a message are required by this - # component. e.g. if requires contains "tokens", than a - # previous component in the pipeline needs to have "tokens" - # within the above described `provides` property. - requires = [] - - # Defines the default configuration parameters of a component - # these values can be overwritten in the pipeline configuration - # of the model. The component should choose sensible defaults - # and should be able to create reasonable results with the defaults. - defaults = {} - - # Defines what language(s) this component can handle. - # This attribute is designed for instance method: `can_handle_language`. - # Default value is None which means it can handle all languages. - # This is an important feature for backwards compatibility of components. - language_list = None - - def __init__(self, component_config=None, low_threshold=0.3, high_threshold=0.4, fallback_intent="fallback", - out_of_scope_intent="out_of_scope"): - super().__init__(component_config) - self.fb_low_threshold = low_threshold - self.fb_high_threshold = high_threshold - self.fallback_intent = fallback_intent - self.out_of_scope_intent = out_of_scope_intent - - def process(self, message, **kwargs): - message_confidence = message.data['intent']['confidence'] - new_intent = None - if message_confidence <= self.fb_low_threshold: - new_intent = {'name': self.out_of_scope_intent, 'confidence': message_confidence} - elif message_confidence <= self.fb_high_threshold: - new_intent = {'name': self.fallback_intent, 'confidence': message_confidence} - if new_intent is not None: - message.data['intent'] = new_intent - message.data['intent_ranking'].insert(0, new_intent) diff --git a/custom/ner.py b/custom/ner.py deleted file mode 100644 index 1a38897fe..000000000 --- a/custom/ner.py +++ /dev/null @@ -1,169 +0,0 @@ -from rasa.nlu.components import Component -from typing import Any, Optional, Text, Dict, TYPE_CHECKING -import os -import spacy -import pickle -from spacy.matcher import Matcher -from rasa.nlu.extractors.extractor import EntityExtractor - - -if TYPE_CHECKING: - from rasa.nlu.model import Metadata - -PATTERN_NER_FILE = 'pattern_ner.pkl' -class SpacyPatternNER(EntityExtractor): - """A new component""" - name = "pattern_ner_spacy" - # Defines what attributes the pipeline component will - # provide when called. The listed attributes - # should be set by the component on the message object - # during test and train, e.g. - # ```message.set("entities", [...])``` - provides = ["entities"] - - # Which attributes on a message are required by this - # component. e.g. if requires contains "tokens", than a - # previous component in the pipeline needs to have "tokens" - # within the above described `provides` property. - requires = ["tokens"] - - # Defines the default configuration parameters of a component - # these values can be overwritten in the pipeline configuration - # of the model. The component should choose sensible defaults - # and should be able to create reasonable results with the defaults. - defaults = {} - - # Defines what language(s) this component can handle. - # This attribute is designed for instance method: `can_handle_language`. - # Default value is None which means it can handle all languages. - # This is an important feature for backwards compatibility of components. - language_list = None - - def __init__(self, component_config=None, matcher=None): - super(SpacyPatternNER, self).__init__(component_config) - if matcher: - self.matcher = matcher - self.spacy_nlp = spacy.blank('en') - self.spacy_nlp.vocab = self.matcher.vocab - else: - self.spacy_nlp = spacy.blank('en') - self.matcher = Matcher(self.spacy_nlp.vocab) - - def train(self, training_data, cfg, **kwargs): - """Train this component. - - This is the components chance to train itself provided - with the training data. The component can rely on - any context attribute to be present, that gets created - by a call to :meth:`components.Component.pipeline_init` - of ANY component and - on any context attributes created by a call to - :meth:`components.Component.train` - of components previous to this one.""" - for lookup_table in training_data.lookup_tables: - key = lookup_table['name'] - pattern = [] - for element in lookup_table['elements']: - tokens = [{'LOWER': token.lower()} for token in str(element).split()] - pattern.append(tokens) - self.matcher.add(key, pattern) - - def process(self, message, **kwargs): - """Process an incoming message. - - This is the components chance to process an incoming - message. The component can rely on - any context attribute to be present, that gets created - by a call to :meth:`components.Component.pipeline_init` - of ANY component and - on any context attributes created by a call to - :meth:`components.Component.process` - of components previous to this one.""" - entities = [] - - # with plural forms - doc = self.spacy_nlp(message.data['text'].lower()) - matches = self.matcher(doc) - entities = self.getNewEntityObj(doc, matches, entities) - - # Without plural forms - doc = self.spacy_nlp(' '.join([token.lemma_ for token in doc])) - matches = self.matcher(doc) - entities = self.getNewEntityObj(doc, matches, entities) - - # Remove duplicates - seen = set() - new_entities = [] - - for entityObj in entities: - record = tuple(entityObj.items()) - if record not in seen: - seen.add(record) - new_entities.append(entityObj) - - message.set("entities", message.get("entities", []) + new_entities, add_to_output=True) - - - def getNewEntityObj(self, doc, matches, entities): - - for ent_id, start, end in matches: - new_entity_value = doc[start:end].text - new_entity_value_len = len(new_entity_value.split()) - is_add = True - - for old_entity in entities: - old_entity_value = old_entity["value"] - old_entity_value_len = len(old_entity_value.split()) - - if old_entity_value_len > new_entity_value_len and new_entity_value in old_entity_value: - is_add = False - elif old_entity_value_len < new_entity_value_len and old_entity_value in new_entity_value: - entities.remove(old_entity) - - if is_add: - entities.append({ - 'start': start, - 'end': end, - 'value': doc[start:end].text, - 'entity': self.matcher.vocab.strings[ent_id], - 'confidence': None, - 'extractor': self.name - }) - - return entities - - - def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: - """Persist this component to disk for future loading.""" - if self.matcher: - modelFile = os.path.join(model_dir, PATTERN_NER_FILE) - self.saveModel(modelFile) - return {"pattern_ner_file": PATTERN_NER_FILE} - - - @classmethod - def load( - cls, - meta: Dict[Text, Any], - model_dir: Optional[Text] = None, - model_metadata: Optional["Metadata"] = None, - cached_component: Optional["Component"] = None, - **kwargs: Any - ) -> "Component": - """Load this component from file.""" - - file_name = meta.get("pattern_ner_file", PATTERN_NER_FILE) - modelFile = os.path.join(model_dir, file_name) - if os.path.exists(modelFile): - modelLoad = open(modelFile, "rb") - matcher = pickle.load(modelLoad) - modelLoad.close() - return cls(meta, matcher) - else: - return cls(meta) - - - def saveModel(self, modelFile): - modelSave = open(modelFile, "wb") - pickle.dump(self.matcher, modelSave) - modelSave.close() \ No newline at end of file diff --git a/kairon/actions/definitions/database.py b/kairon/actions/definitions/database.py index 6f0e48271..ebdf83510 100644 --- a/kairon/actions/definitions/database.py +++ b/kairon/actions/definitions/database.py @@ -83,7 +83,7 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma request_body = ActionUtility.get_payload(payload, tracker) msg_logger.append(request_body) tracker_data = ActionUtility.build_context(tracker, True) - response = await vector_db.perform_operation(operation_type, request_body) + response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id, bot=self.bot) logger.info("response: " + str(response)) response_context = self.__add_user_context_to_http_response(response, tracker_data) bot_response, bot_resp_log, _ = ActionUtility.compose_response(vector_action_config['response'], response_context) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 3de92f413..381e6f543 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -4,7 +4,6 @@ from rasa_sdk import Tracker from rasa_sdk.executor import CollectingDispatcher -from kairon import Utility from kairon.actions.definitions.base import ActionsBase from kairon.shared.actions.data_objects import ActionServerLogs from kairon.shared.actions.exception import ActionFailure @@ -12,8 +11,8 @@ from kairon.shared.actions.utils import ActionUtility from kairon.shared.constants import FAQ_DISABLED_ERR, KaironSystemSlots, KAIRON_USER_MSG_ENTITY from kairon.shared.data.constant import DEFAULT_NLU_FALLBACK_RESPONSE -from kairon.shared.llm.factory import LLMFactory from kairon.shared.models import LlmPromptType, LlmPromptSource +from kairon.shared.llm.processor import LLMProcessor class ActionPrompt(ActionsBase): @@ -62,14 +61,18 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma time_taken_slots = 0 final_slots = {"type": "slots_to_fill"} llm_response_log = {"type": "llm_response"} - + llm_processor = None try: k_faq_action_config, bot_settings = self.retrieve_config() user_question = k_faq_action_config.get('user_question') user_msg = self.__get_user_msg(tracker, user_question) + llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) - llm = LLMFactory.get_instance("faq")(self.bot, bot_settings["llm_settings"]) - llm_response, time_taken_llm_response = await llm.predict(user_msg, **llm_params) + llm_processor = LLMProcessor(self.bot) + llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, + user=tracker.sender_id, + bot=self.bot, + **llm_params) status = "FAILURE" if llm_response.get("is_failure", False) is True else status exception = llm_response.get("exception") bot_response = llm_response['content'] @@ -93,8 +96,8 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma total_time_elapsed = time_taken_llm_response + time_taken_slots events_to_extend = [llm_response_log, final_slots] events.extend(events_to_extend) - if llm: - llm_logs = llm.logs + if llm_processor: + llm_logs = llm_processor.logs ActionServerLogs( type=ActionType.prompt_action.value, intent=tracker.get_intent_of_latest_message(skip_fallback_intent=False), @@ -119,16 +122,6 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma return slots_to_fill async def __get_llm_params(self, k_faq_action_config: dict, dispatcher: CollectingDispatcher, tracker: Tracker, domain: Dict[Text, Any]): - implementations = { - "GPT3_FAQ_EMBED": self.__get_gpt_params, - } - - llm_type = Utility.environment['llm']["faq"] - if not implementations.get(llm_type): - raise ActionFailure(f'{llm_type} type LLM is not supported') - return await implementations[Utility.environment['llm']["faq"]](k_faq_action_config, dispatcher, tracker, domain) - - async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: CollectingDispatcher, tracker: Tracker, domain: Dict[Text, Any]): from kairon.actions.definitions.factory import ActionFactory system_prompt = None @@ -147,7 +140,7 @@ async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: Collecti history_prompt = ActionUtility.prepare_bot_responses(tracker, num_bot_responses) elif prompt['source'] == LlmPromptSource.bot_content.value and prompt['is_enabled']: use_similarity_prompt = True - hyperparameters = prompt.get('hyperparameters', {}) + hyperparameters = prompt.get("hyperparameters", {}) similarity_prompt.append({'similarity_prompt_name': prompt['name'], 'similarity_prompt_instructions': prompt['instructions'], 'collection': prompt['data'], @@ -179,7 +172,7 @@ async def __get_gpt_params(self, k_faq_action_config: dict, dispatcher: Collecti is_query_prompt_enabled = True query_prompt_dict.update({'query_prompt': query_prompt, 'use_query_prompt': is_query_prompt_enabled}) - params["hyperparameters"] = k_faq_action_config.get('hyperparameters', Utility.get_llm_hyperparameters()) + params["hyperparameters"] = k_faq_action_config['hyperparameters'] params["system_prompt"] = system_prompt params["context_prompt"] = context_prompt params["query_prompt"] = query_prompt_dict diff --git a/kairon/api/models.py b/kairon/api/models.py index 1eaa034f2..53ce8d62c 100644 --- a/kairon/api/models.py +++ b/kairon/api/models.py @@ -16,6 +16,7 @@ INTEGRATION_STATUS, FALLBACK_MESSAGE, DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_LLM ) from ..shared.actions.models import ( ActionParameterType, @@ -37,6 +38,7 @@ CognitionDataType, CognitionMetadataType, ) +from kairon.shared.utils import Utility class RecaptchaVerifiedRequest(BaseModel): @@ -1057,6 +1059,7 @@ class PromptActionConfigRequest(BaseModel): num_bot_responses: int = 5 failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE user_question: UserQuestionModel = UserQuestionModel() + llm_type: str = DEFAULT_LLM hyperparameters: dict = None llm_prompts: List[LlmPromptRequest] instructions: List[str] = [] @@ -1078,16 +1081,27 @@ def validate_num_bot_responses(cls, v, values, **kwargs): raise ValueError("num_bot_responses should not be greater than 5") return v + @validator("llm_type") + def validate_llm_type(cls, v, values, **kwargs): + if v not in Utility.get_llms(): + raise ValueError("Invalid llm type") + return v + + @validator("hyperparameters") + def validate_llm_hyperparameters(cls, v, values, **kwargs): + Utility.validate_llm_hyperparameters(v, kwargs['llm_type'], ValueError) + @root_validator def check(cls, values): from kairon.shared.utils import Utility - if not values.get("hyperparameters"): - values["hyperparameters"] = {} + if values.get("llm_type"): + if not values.get("hyperparameters"): + values["hyperparameters"] = {} - for key, value in Utility.get_llm_hyperparameters().items(): - if key not in values["hyperparameters"]: - values["hyperparameters"][key] = value + for key, value in Utility.get_llm_hyperparameters(values.get("llm_type")).items(): + if key not in values["hyperparameters"]: + values["hyperparameters"][key] = value return values diff --git a/kairon/chat/agent/message_processor.py b/kairon/chat/agent/message_processor.py index 827404063..e4ae506d2 100644 --- a/kairon/chat/agent/message_processor.py +++ b/kairon/chat/agent/message_processor.py @@ -294,15 +294,6 @@ def predict_next_with_tracker_if_should( Raises: ActionLimitReached if the limit of actions to predict has been reached. """ - should_predict_another_action = self.should_predict_another_action( - tracker.latest_action_name - ) - - if self.is_action_limit_reached(tracker, should_predict_another_action): - raise ActionLimitReached( - "The limit of actions to predict has been reached." - ) - prediction = self._predict_next_with_tracker(tracker) action = self.action_for_index( diff --git a/kairon/importer/validator/file_validator.py b/kairon/importer/validator/file_validator.py index b55b3f0e6..ad2062c01 100644 --- a/kairon/importer/validator/file_validator.py +++ b/kairon/importer/validator/file_validator.py @@ -695,9 +695,9 @@ def __validate_prompt_actions(prompt_actions: list): data_error.append( f'num_bot_responses should not be greater than 5 and of type int: {action.get("name")}') llm_prompts_errors = TrainingDataValidator.__validate_llm_prompts(action['llm_prompts']) - if action.get('hyperparameters') is not None: - llm_hyperparameters_errors = TrainingDataValidator.__validate_llm_prompts_hyperparamters( - action.get('hyperparameters')) + if action.get('hyperparameters'): + llm_hyperparameters_errors = TrainingDataValidator.__validate_llm_prompts_hyperparameters( + action.get('hyperparameters'), action.get("llm_type", "openai")) data_error.extend(llm_hyperparameters_errors) data_error.extend(llm_prompts_errors) if action['name'] in actions_present: @@ -785,27 +785,12 @@ def __validate_llm_prompts(llm_prompts: dict): return error_list @staticmethod - def __validate_llm_prompts_hyperparamters(hyperparameters: dict): + def __validate_llm_prompts_hyperparameters(hyperparameters: dict, llm_type: str): error_list = [] - for key, value in hyperparameters.items(): - if key == 'temperature' and not 0.0 <= value <= 2.0: - error_list.append("Temperature must be between 0.0 and 2.0!") - elif key == 'presence_penalty' and not -2.0 <= value <= 2.0: - error_list.append("presence_penality must be between -2.0 and 2.0!") - elif key == 'frequency_penalty' and not -2.0 <= value <= 2.0: - error_list.append("frequency_penalty must be between -2.0 and 2.0!") - elif key == 'top_p' and not 0.0 <= value <= 1.0: - error_list.append("top_p must be between 0.0 and 1.0!") - elif key == 'n' and not 1 <= value <= 5: - error_list.append("n must be between 1 and 5!") - elif key == 'max_tokens' and not 5 <= value <= 4096: - error_list.append("max_tokens must be between 5 and 4096!") - elif key == 'logit_bias' and not isinstance(value, dict): - error_list.append("logit_bias must be a dictionary!") - elif key == 'stop': - if value and (not isinstance(value, (str, int, list)) or (isinstance(value, list) and len(value) > 4)): - error_list.append( - "Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers.") + try: + Utility.validate_llm_hyperparameters(hyperparameters, llm_type, AppException) + except AppException as e: + error_list.append(e.__str__()) return error_list @staticmethod diff --git a/kairon/shared/actions/data_objects.py b/kairon/shared/actions/data_objects.py index b43a16da2..d02e88b02 100644 --- a/kairon/shared/actions/data_objects.py +++ b/kairon/shared/actions/data_objects.py @@ -34,6 +34,7 @@ KAIRON_TWO_STAGE_FALLBACK, FALLBACK_MESSAGE, DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_LLM ) from kairon.shared.data.signals import push_notification, auditlogger from kairon.shared.models import LlmPromptType, LlmPromptSource @@ -784,7 +785,8 @@ class PromptAction(Auditlog): bot = StringField(required=True) user = StringField(required=True) timestamp = DateTimeField(default=datetime.utcnow) - hyperparameters = DictField(default=Utility.get_llm_hyperparameters) + llm_type = StringField(default=DEFAULT_LLM, choices=Utility.get_llms()) + hyperparameters = DictField(default=Utility.get_default_llm_hyperparameters) llm_prompts = EmbeddedDocumentListField(LlmPrompt, required=True) instructions = ListField(StringField()) set_slots = EmbeddedDocumentListField(SetSlotsFromResponse) @@ -794,7 +796,7 @@ class PromptAction(Auditlog): meta = {"indexes": [{"fields": ["bot", ("bot", "name", "status")]}]} def clean(self): - for key, value in Utility.get_llm_hyperparameters().items(): + for key, value in Utility.get_llm_hyperparameters(self.llm_type).items(): if key not in self.hyperparameters: self.hyperparameters.update({key: value}) @@ -812,7 +814,7 @@ def validate(self, clean=True): dict_data["llm_prompts"], ValidationError ) Utility.validate_llm_hyperparameters( - dict_data["hyperparameters"], ValidationError + dict_data["hyperparameters"], self.llm_type, ValidationError ) diff --git a/kairon/shared/concurrency/actors/pyscript_runner.py b/kairon/shared/concurrency/actors/pyscript_runner.py index a68e352c9..bd5286739 100644 --- a/kairon/shared/concurrency/actors/pyscript_runner.py +++ b/kairon/shared/concurrency/actors/pyscript_runner.py @@ -1,20 +1,26 @@ from types import ModuleType from typing import Text, Dict, Optional, Callable +import orjson as json from AccessControl.ZopeGuards import _safe_globals from RestrictedPython import compile_restricted from RestrictedPython.Guards import safer_getattr from loguru import logger from timeout_decorator import timeout_decorator -import orjson as json -from ..actors.base import BaseActor from kairon.exceptions import AppException +from ..actors.base import BaseActor +from AccessControl.SecurityInfo import allow_module + +allow_module("datetime") +allow_module("time") + global_safe = _safe_globals global_safe['_getattr_'] = safer_getattr global_safe['json'] = json + class PyScriptRunner(BaseActor): def execute(self, source_code: Text, predefined_objects: Optional[Dict] = None, **kwargs): diff --git a/kairon/shared/data/constant.py b/kairon/shared/data/constant.py index ad88ee625..afa40a619 100644 --- a/kairon/shared/data/constant.py +++ b/kairon/shared/data/constant.py @@ -215,6 +215,7 @@ class ModelTestType(str, Enum): DEFAULT_SYSTEM_PROMPT = ( "You are a personal assistant. Answer question based on the context below" ) +DEFAULT_LLM = "openai" class AuditlogActions(str, Enum): diff --git a/kairon/shared/data/processor.py b/kairon/shared/data/processor.py index d430021c4..dfb8511d5 100644 --- a/kairon/shared/data/processor.py +++ b/kairon/shared/data/processor.py @@ -7287,9 +7287,7 @@ def edit_prompt_action( action.failure_message = request_data.get("failure_message") action.user_question = UserQuestion(**request_data.get("user_question")) action.num_bot_responses = request_data.get("num_bot_responses", 5) - action.hyperparameters = request_data.get( - "hyperparameters", Utility.get_llm_hyperparameters() - ) + action.hyperparameters = request_data.get("hyperparameters") action.llm_prompts = [ LlmPrompt(**prompt) for prompt in request_data.get("llm_prompts", []) ] diff --git a/kairon/shared/llm/base.py b/kairon/shared/llm/base.py index 4babc6a23..f07eceda0 100644 --- a/kairon/shared/llm/base.py +++ b/kairon/shared/llm/base.py @@ -8,9 +8,9 @@ def __init__(self, bot: Text): self.bot = bot @abstractmethod - async def train(self, *args, **kwargs) -> Dict: + async def train(self, user, bot, *args, **kwargs) -> Dict: pass @abstractmethod - async def predict(self, query, *args, **kwargs) -> Dict: + async def predict(self, query, user, bot, *args, **kwargs) -> Dict: pass diff --git a/kairon/shared/llm/clients/__init__.py b/kairon/shared/llm/clients/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/kairon/shared/llm/clients/azure.py b/kairon/shared/llm/clients/azure.py deleted file mode 100644 index 9b980d0d6..000000000 --- a/kairon/shared/llm/clients/azure.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Text - -from kairon.shared.constants import GPT3ResourceTypes -from kairon.shared.llm.clients.gpt3 import GPT3Resources - - -class AzureGPT3Resources(GPT3Resources): - resource_url = "https://kairon.openai.azure.com/openai/deployments" - - def __init__(self, api_key: Text, **kwargs): - super().__init__(api_key) - self.api_key = api_key - self.api_version = kwargs.get("api_version") - self.model_id = { - GPT3ResourceTypes.embeddings.value: kwargs.get("embeddings_model_id"), - GPT3ResourceTypes.chat_completion.value: kwargs.get("chat_completion_model_id") - } - - def get_headers(self): - return {"api-key": self.api_key} - - def get_resource_url(self, resource: Text): - model_id = self.model_id[resource] - resource_url = f"{self.resource_url}/{model_id}/{resource}?api-version={self.api_version}" - return resource_url diff --git a/kairon/shared/llm/clients/base.py b/kairon/shared/llm/clients/base.py deleted file mode 100644 index 71ef7037e..000000000 --- a/kairon/shared/llm/clients/base.py +++ /dev/null @@ -1,8 +0,0 @@ -from abc import ABC -from typing import Text - - -class LLMResources(ABC): - - async def invoke(self, resource: Text, engine: Text, **kwargs): - raise NotImplementedError("Provider not implemented") diff --git a/kairon/shared/llm/clients/factory.py b/kairon/shared/llm/clients/factory.py deleted file mode 100644 index def8d09c3..000000000 --- a/kairon/shared/llm/clients/factory.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import Text -from kairon.exceptions import AppException -from kairon.shared.constants import LLMResourceProvider -from kairon.shared.llm.clients.azure import AzureGPT3Resources -from kairon.shared.llm.clients.gpt3 import GPT3Resources - - -class LLMClientFactory: - __implementations = { - LLMResourceProvider.openai.value: GPT3Resources, - LLMResourceProvider.azure.value: AzureGPT3Resources - } - - @staticmethod - def get_resource_provider(_type: Text): - if not LLMClientFactory.__implementations.get(_type): - raise AppException(f'{_type} client not supported') - return LLMClientFactory.__implementations[_type] diff --git a/kairon/shared/llm/clients/gpt3.py b/kairon/shared/llm/clients/gpt3.py deleted file mode 100644 index d6f2c5679..000000000 --- a/kairon/shared/llm/clients/gpt3.py +++ /dev/null @@ -1,92 +0,0 @@ -import ujson as json -import random -from json import JSONDecodeError -from ujson import JSONDecodeError as UJSONDecodeError -from typing import Text -from loguru import logger -from openai.api_requestor import parse_stream_helper -from kairon.exceptions import AppException -from kairon.shared.constants import GPT3ResourceTypes -from kairon.shared.llm.clients.base import LLMResources -from kairon.shared.rest_client import AioRestClient - - -class GPT3Resources(LLMResources): - resource_url = "https://api.openai.com/v1" - - def __init__(self, api_key: Text, **kwargs): - self.api_key = api_key - - def get_headers(self): - return {"Authorization": f"Bearer {self.api_key}"} - - def get_resource_url(self, resource: Text): - return f"{self.resource_url}/{resource}" - - async def invoke(self, resource: Text, model: Text, **kwargs): - client = None - http_url = self.get_resource_url(resource) - request_body = kwargs.copy() - request_body.update({"model": model}) - is_streaming_resp = kwargs.get("stream", False) - try: - client = AioRestClient(False) - resp = await client.request("POST", http_url, request_body, self.get_headers(), - return_json=False, is_streaming_resp=is_streaming_resp, max_retries=3) - if resp.status != 200: - try: - resp = await resp.json() - logger.debug(f"GPT response error: {resp}") - raise AppException(f"{resp['error'].get('message')}. Request id: {resp['error'].get('id')}") - except JSONDecodeError: - raise AppException(f"Received non 200 status code ({resp.status}): {resp.text}") - - if is_streaming_resp: - resp = client.streaming_response - - data = await self.__parse_response(resource, resp, **kwargs) - finally: - if client: - await client.cleanup() - return data - - async def __parse_response(self, resource: Text, response, **kwargs): - parsers = { - GPT3ResourceTypes.embeddings.value: self._parse_embeddings_response, - GPT3ResourceTypes.chat_completion.value: self.__parse_completion_response - } - return await parsers[resource](response, **kwargs) - - async def _parse_embeddings_response(self, response, **hyperparameters): - raw_response = await response.json() - formatted_response = raw_response["data"][0]["embedding"] - return formatted_response, raw_response - - async def __parse_completion_response(self, response, **kwargs): - if kwargs.get("stream"): - formatted_response = await self._parse_streaming_response(response, kwargs.get("n", 1)) - raw_response = response - else: - formatted_response, raw_response = await self._parse_api_response(response) - return formatted_response, raw_response - - async def _parse_api_response(self, response): - raw_response = await response.json() - msg_choice = random.choice(raw_response['choices']) - formatted_response = msg_choice['message']['content'] - return formatted_response, raw_response - - async def _parse_streaming_response(self, response, num_choices): - formatted_response = '' - msg_choice = random.randint(0, num_choices - 1) - try: - for chunk in response or []: - line = parse_stream_helper(chunk) - if line: - line = json.loads(line) - if line["choices"][0].get("index") == msg_choice and line["choices"][0]['delta'].get('content'): - formatted_response = f"{formatted_response}{line['choices'][0]['delta']['content']}" - except Exception as e: - logger.exception(e) - raise AppException(f"Failed to parse streaming response: {chunk}") - return formatted_response diff --git a/kairon/shared/llm/data_objects.py b/kairon/shared/llm/data_objects.py new file mode 100644 index 000000000..7713444ba --- /dev/null +++ b/kairon/shared/llm/data_objects.py @@ -0,0 +1,13 @@ +from mongoengine import Document, DynamicField, StringField, FloatField, DateTimeField, DictField + + +class LLMLogs(Document): + response = DynamicField() + start_time = DateTimeField() + end_time = DateTimeField() + cost = FloatField() + llm_call_id = StringField() + llm_provider = StringField() + model = StringField() + model_params = DictField() + metadata = DictField() \ No newline at end of file diff --git a/kairon/shared/llm/factory.py b/kairon/shared/llm/factory.py deleted file mode 100644 index 5424d1eea..000000000 --- a/kairon/shared/llm/factory.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Text -from kairon.exceptions import AppException -from kairon.shared.utils import Utility -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - - -class LLMFactory: - __implementations = { - "GPT3_FAQ_EMBED": GPT3FAQEmbedding - } - - @staticmethod - def get_instance(_type: Text): - llm_type = Utility.environment['llm'][_type] - if not LLMFactory.__implementations.get(llm_type): - raise AppException(f'{llm_type} type LLM is not supported') - return LLMFactory.__implementations[llm_type] diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py new file mode 100644 index 000000000..06b720b48 --- /dev/null +++ b/kairon/shared/llm/logger.py @@ -0,0 +1,43 @@ +from litellm.integrations.custom_logger import CustomLogger +from .data_objects import LLMLogs +import ujson as json + + +class LiteLLMLogger(CustomLogger): + def log_pre_api_call(self, model, messages, kwargs): + pass + + def log_post_api_call(self, kwargs, response_obj, start_time, end_time): + pass + + def log_stream_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + async def async_log_stream_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.__logs_litellm(**kwargs) + + def __logs_litellm(self, **kwargs): + litellm_params = kwargs['litellm_params'] + self.__save_logs(**{'response': json.loads(kwargs['original_response']), + 'start_time': kwargs['start_time'], + 'end_time': kwargs['end_time'], + 'cost': kwargs["response_cost"], + 'llm_call_id': litellm_params['litellm_call_id'], + 'llm_provider': litellm_params['custom_llm_provider'], + 'model_params': kwargs["additional_args"]["complete_input_dict"], + 'metadata': litellm_params['metadata']}) + + def __save_logs(self, **kwargs): + LLMLogs(**kwargs).save() diff --git a/kairon/shared/llm/gpt3.py b/kairon/shared/llm/processor.py similarity index 73% rename from kairon/shared/llm/gpt3.py rename to kairon/shared/llm/processor.py index 4e991ca7c..ffc48e2eb 100644 --- a/kairon/shared/llm/gpt3.py +++ b/kairon/shared/llm/processor.py @@ -1,9 +1,9 @@ +import random import time - from typing import Text, Dict, List, Tuple from urllib.parse import urljoin -import openai +import litellm from loguru import logger as logging from tiktoken import get_encoding from tqdm import tqdm @@ -13,35 +13,33 @@ from kairon.shared.admin.processor import Sysadmin from kairon.shared.cognition.data_objects import CognitionData from kairon.shared.cognition.processor import CognitionDataProcessor -from kairon.shared.constants import GPT3ResourceTypes from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_CONTEXT_PROMPT from kairon.shared.llm.base import LLMBase -from kairon.shared.llm.clients.factory import LLMClientFactory +from kairon.shared.llm.logger import LiteLLMLogger from kairon.shared.models import CognitionDataType from kairon.shared.rest_client import AioRestClient from kairon.shared.utils import Utility +litellm.callbacks = [LiteLLMLogger()] + -class GPT3FAQEmbedding(LLMBase): +class LLMProcessor(LLMBase): __embedding__ = 1536 - def __init__(self, bot: Text, llm_settings: dict): + def __init__(self, bot: Text): super().__init__(bot) self.db_url = Utility.environment['vector']['db'] self.headers = {} if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.suffix = "_faq_embd" - self.vector_config = {'size': 1536, 'distance': 'Cosine'} - self.llm_settings = llm_settings + self.vector_config = {'size': self.__embedding__, 'distance': 'Cosine'} self.api_key = Sysadmin.get_bot_secret(bot, BotSecretType.gpt_key.value, raise_err=True) - self.client = LLMClientFactory.get_resource_provider(llm_settings["provider"])(self.api_key, - **self.llm_settings) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 self.__logs = [] - async def train(self, *args, **kwargs) -> Dict: + async def train(self, user, bot, *args, **kwargs) -> Dict: await self.__delete_collections() count = 0 processor = CognitionDataProcessor() @@ -51,35 +49,38 @@ async def train(self, *args, **kwargs) -> Dict: {'$project': {'collection': "$_id", 'content': 1, '_id': 0}} ])) for collections in collection_groups: - collection = f"{self.bot}_{collections['collection']}{self.suffix}" if collections['collection'] else f"{self.bot}{self.suffix}" + collection = f"{self.bot}_{collections['collection']}{self.suffix}" if collections[ + 'collection'] else f"{self.bot}{self.suffix}" await self.__create_collection__(collection) for content in tqdm(collections['content'], desc="Training FAQ"): if content['content_type'] == CognitionDataType.json.value: metadata = processor.find_matching_metadata(self.bot, content['data'], content.get('collection')) - search_payload, embedding_payload = Utility.retrieve_search_payload_and_embedding_payload(content['data'], metadata) + search_payload, embedding_payload = Utility.retrieve_search_payload_and_embedding_payload( + content['data'], metadata) else: search_payload, embedding_payload = {'content': content["data"]}, content["data"] - search_payload['collection_name'] = collection - embeddings = await self.__get_embedding(embedding_payload) + #search_payload['collection_name'] = collection + embeddings = await self.get_embedding(embedding_payload, user, bot) points = [{'id': content['vector_id'], 'vector': embeddings, 'payload': search_payload}] - await self.__collection_upsert__(collection, {'points': points}, err_msg="Unable to train FAQ! Contact support") + await self.__collection_upsert__(collection, {'points': points}, + err_msg="Unable to train FAQ! Contact support") count += 1 return {"faq": count} - async def predict(self, query: Text, *args, **kwargs) -> Tuple: + async def predict(self, query: Text, user, bot, *args, **kwargs) -> Tuple: start_time = time.time() embeddings_created = False try: - query_embedding = await self.__get_embedding(query) + query_embedding = await self.get_embedding(query, user, bot) embeddings_created = True system_prompt = kwargs.pop('system_prompt', DEFAULT_SYSTEM_PROMPT) context_prompt = kwargs.pop('context_prompt', DEFAULT_CONTEXT_PROMPT) context = await self.__attach_similarity_prompt_if_enabled(query_embedding, context_prompt, **kwargs) - answer = await self.__get_answer(query, system_prompt, context, **kwargs) + answer = await self.__get_answer(query, system_prompt, context, user, bot, **kwargs) response = {"content": answer} - except openai.error.APIConnectionError as e: + except Exception as e: logging.exception(e) if embeddings_created: failure_stage = "Retrieving chat completion for the provided query." @@ -87,9 +88,6 @@ async def predict(self, query: Text, *args, **kwargs) -> Tuple: failure_stage = "Creating a new embedding for the provided query." self.__logs.append({'error': f"{failure_stage} {str(e)}"}) response = {"is_failure": True, "exception": str(e), "content": None} - except Exception as e: - logging.exception(e) - response = {"is_failure": True, "exception": str(e), "content": None} end_time = time.time() elapsed_time = end_time - start_time @@ -102,13 +100,36 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def __get_embedding(self, text: Text) -> List[float]: + async def get_embedding(self, text: Text, user, bot) -> List[float]: truncated_text = self.truncate_text(text) - result, _ = await self.client.invoke(GPT3ResourceTypes.embeddings.value, model="text-embedding-3-small", - input=truncated_text) - return result + result = await litellm.aembedding(model="text-embedding-3-small", + input=[truncated_text], + metadata={'user': user, 'bot': bot}, + api_key=self.api_key, + num_retries=3) + return result["data"][0]["embedding"] + + async def __parse_completion_response(self, response, **kwargs): + if kwargs.get("stream"): + formatted_response = '' + msg_choice = random.randint(0, kwargs.get("n", 1) - 1) + if response["choices"][0].get("index") == msg_choice and response["choices"][0]['delta'].get('content'): + formatted_response = f"{response['choices'][0]['delta']['content']}" + else: + msg_choice = random.choice(response['choices']) + formatted_response = msg_choice['message']['content'] + return formatted_response + + async def __get_completion(self, messages, hyperparameters, user, bot, **kwargs): + response = await litellm.acompletion(messages=messages, + metadata={'user': user, 'bot': bot}, + api_key=self.api_key, + num_retries=3, + **hyperparameters) + formatted_response = await self.__parse_completion_response(response, **kwargs) + return formatted_response, response - async def __get_answer(self, query, system_prompt: Text, context: Text, **kwargs): + async def __get_answer(self, query, system_prompt: Text, context: Text, user, bot, **kwargs): use_query_prompt = False query_prompt = '' if kwargs.get('query_prompt', {}): @@ -116,12 +137,15 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, **kwargs query_prompt = query_prompt_dict.get('query_prompt', '') use_query_prompt = query_prompt_dict.get('use_query_prompt') previous_bot_responses = kwargs.get('previous_bot_responses') - hyperparameters = kwargs.get('hyperparameters', Utility.get_llm_hyperparameters()) + hyperparameters = kwargs['hyperparameters'] instructions = kwargs.get('instructions', []) instructions = '\n'.join(instructions) if use_query_prompt and query_prompt: - query = await self.__rephrase_query(query, system_prompt, query_prompt, hyperparameters=hyperparameters) + query = await self.__rephrase_query(query, system_prompt, query_prompt, + hyperparameters=hyperparameters, + user=user, + bot=bot) messages = [ {"role": "system", "content": system_prompt}, ] @@ -130,21 +154,25 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, **kwargs messages.append({"role": "user", "content": f"{context} \n{instructions} \nQ: {query} \nA:"}) if instructions \ else messages.append({"role": "user", "content": f"{context} \nQ: {query} \nA:"}) - completion, raw_response = await self.client.invoke(GPT3ResourceTypes.chat_completion.value, messages=messages, - **hyperparameters) + completion, raw_response = await self.__get_completion(messages=messages, + hyperparameters=hyperparameters, + user=user, + bot=bot) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'answer_query', 'hyperparameters': hyperparameters}) return completion - async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, **kwargs): + async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, bot, **kwargs): messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": f"{query_prompt}\n\n Q: {query}\n A:"} ] - hyperparameters = kwargs.get('hyperparameters', Utility.get_llm_hyperparameters()) + hyperparameters = kwargs['hyperparameters'] - completion, raw_response = await self.client.invoke(GPT3ResourceTypes.chat_completion.value, messages=messages, - **hyperparameters) + completion, raw_response = await self.__get_completion(messages=messages, + hyperparameters=hyperparameters, + user=user, + bot=bot) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'rephrase_query', 'hyperparameters': hyperparameters}) return completion @@ -153,9 +181,9 @@ async def __delete_collections(self): client = AioRestClient(False) try: response = await client.request(http_url=urljoin(self.db_url, "/collections"), - request_method="GET", - headers=self.headers, - timeout=5) + request_method="GET", + headers=self.headers, + timeout=5) if response.get('result'): for collection in response['result'].get('collections') or []: if collection['name'].startswith(self.bot): diff --git a/kairon/shared/utils.py b/kairon/shared/utils.py index 93fe213c6..2db890379 100644 --- a/kairon/shared/utils.py +++ b/kairon/shared/utils.py @@ -78,6 +78,7 @@ TOKEN_TYPE, KAIRON_TWO_STAGE_FALLBACK, SLOT_TYPE, + DEFAULT_LLM ) from .data.dto import KaironStoryStep from .models import StoryStepType, LlmPromptType, LlmPromptSource @@ -2050,73 +2051,33 @@ def verify_email(email: Text): raise AppException("Invalid or disposable Email!") @staticmethod - def get_llm_hyperparameters(): + def get_llms(): + return Utility.system_metadata["llm"].keys() + + @staticmethod + def get_default_llm_hyperparameters(): + return Utility.get_llm_hyperparameters(DEFAULT_LLM) + + @staticmethod + def get_llm_hyperparameters(llm_type): hyperparameters = {} - if Utility.environment["llm"]["faq"] in {"GPT3_FAQ_EMBED"}: - for key, value in Utility.system_metadata["llm"]["gpt"].items(): + if llm_type in Utility.system_metadata["llm"].keys(): + for key, value in Utility.system_metadata["llm"][llm_type]['properties'].items(): hyperparameters[key] = value["default"] return hyperparameters - raise AppException("Could not find any hyperparameters for configured LLM.") + raise AppException(f"Could not find any hyperparameters for {llm_type} LLM.") @staticmethod - def validate_llm_hyperparameters(hyperparameters: dict, exception_class): - params = Utility.system_metadata["llm"]["gpt"] - for key, value in hyperparameters.items(): - if ( - key == "temperature" - and not params["temperature"]["min"] - <= value - <= params["temperature"]["max"] - ): - raise exception_class( - f"Temperature must be between {params['temperature']['min']} and {params['temperature']['max']}!" - ) - elif ( - key == "presence_penalty" - and not params["presence_penalty"]["min"] - <= value - <= params["presence_penalty"]["max"] - ): - raise exception_class( - f"Presence penalty must be between {params['presence_penalty']['min']} and {params['presence_penalty']['max']}!" - ) - elif ( - key == "frequency_penalty" - and not params["presence_penalty"]["min"] - <= value - <= params["presence_penalty"]["max"] - ): - raise exception_class( - f"Frequency penalty must be between {params['presence_penalty']['min']} and {params['presence_penalty']['max']}!" - ) - elif ( - key == "top_p" - and not params["top_p"]["min"] <= value <= params["top_p"]["max"] - ): - raise exception_class( - f"top_p must be between {params['top_p']['min']} and {params['top_p']['max']}!" - ) - elif key == "n" and not params["n"]["min"] <= value <= params["n"]["max"]: - raise exception_class( - f"n must be between {params['n']['min']} and {params['n']['max']} and should not be 0!" - ) - elif ( - key == "max_tokens" - and not params["max_tokens"]["min"] - <= value - <= params["max_tokens"]["max"] - ): - raise exception_class( - f"max_tokens must be between {params['max_tokens']['min']} and {params['max_tokens']['max']} and should not be 0!" - ) - elif key == "logit_bias" and not isinstance(value, dict): - raise exception_class("logit_bias must be a dictionary!") - elif key == "stop": - exc_msg = "Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers." - if value and not isinstance(value, (str, int, list)): - raise exception_class(exc_msg) - elif value and (isinstance(value, list) and len(value) > 4): - raise exception_class(exc_msg) + def validate_llm_hyperparameters(hyperparameters: dict, llm_type: str, exception_class): + from jsonschema_rs import JSONSchema, ValidationError + schema = Utility.system_metadata["llm"][llm_type] + try: + validator = JSONSchema(schema) + validator.validate(hyperparameters) + except ValidationError as e: + message = f"{e.instance_path}: {e.message}" + raise exception_class(message) + @staticmethod def create_uuid_from_string(val: str): diff --git a/kairon/shared/vector_embeddings/db/base.py b/kairon/shared/vector_embeddings/db/base.py index 178ee25de..d1c2a1e97 100644 --- a/kairon/shared/vector_embeddings/db/base.py +++ b/kairon/shared/vector_embeddings/db/base.py @@ -8,16 +8,16 @@ class VectorEmbeddingsDbBase(ABC): @abstractmethod - async def embedding_search(self, request_body: Dict): + async def embedding_search(self, request_body: Dict, **kwargs): raise NotImplementedError("Provider not implemented") @abstractmethod - async def payload_search(self, request_body: Dict): + async def payload_search(self, request_body: Dict, **kwargs): raise NotImplementedError("Provider not implemented") - async def perform_operation(self, op_type: Text, request_body: Dict): + async def perform_operation(self, op_type: Text, request_body: Dict, **kwargs): supported_ops = {DbActionOperationType.payload_search.value: self.payload_search, DbActionOperationType.embedding_search.value: self.embedding_search} if op_type not in supported_ops.keys(): raise AppException("Operation type not supported") - return await supported_ops[op_type](request_body) + return await supported_ops[op_type](request_body, **kwargs) diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index d2ff7e69c..893a310ad 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -8,7 +8,7 @@ from kairon.shared.admin.constants import BotSecretType from kairon.shared.admin.processor import Sysadmin from kairon.shared.constants import GPT3ResourceTypes -from kairon.shared.llm.clients.factory import LLMClientFactory +from kairon.shared.llm.processor import LLMProcessor from kairon.shared.vector_embeddings.db.base import VectorEmbeddingsDbBase @@ -25,9 +25,7 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.llm_settings = llm_settings - self.api_key = Sysadmin.get_bot_secret(self.bot, BotSecretType.gpt_key.value, raise_err=True) - self.client = LLMClientFactory.get_resource_provider(llm_settings["provider"])(self.api_key, - **self.llm_settings) + self.llm = LLMProcessor(self.bot) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 @@ -38,18 +36,16 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def __get_embedding(self, text: Text) -> List[float]: - truncated_text = self.truncate_text(text) - result, _ = await self.client.invoke(GPT3ResourceTypes.embeddings.value, model="text-embedding-3-small", - input=truncated_text) + async def __get_embedding(self, text: Text, **kwargs) -> List[float]: + result, _ = await self.llm.get_embedding(text, user=kwargs.get('user'), bot=kwargs.get('bot')) return result - async def embedding_search(self, request_body: Dict): + async def embedding_search(self, request_body: Dict, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points") if request_body.get("text"): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/search") user_msg = request_body.get("text") - vector = await self.__get_embedding(user_msg) + vector = await self.__get_embedding(user_msg, **kwargs) request_body = {'vector': vector, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} embedding_search_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', @@ -57,7 +53,7 @@ async def embedding_search(self, request_body: Dict): request_body=request_body) return embedding_search_result - async def payload_search(self, request_body: Dict): + async def payload_search(self, request_body: Dict, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/scroll") payload_filter_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', diff --git a/kairon/train.py b/kairon/train.py index 05e761dc5..0276f7bc5 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -13,10 +13,10 @@ from kairon.shared.data.constant import EVENT_STATUS from kairon.shared.data.model_processor import ModelProcessor from kairon.shared.data.processor import MongoProcessor -from kairon.shared.llm.factory import LLMFactory from kairon.shared.metering.constants import MetricType from kairon.shared.metering.metering_processor import MeteringProcessor from kairon.shared.utils import Utility +from kairon.shared.llm.processor import LLMProcessor def train_model_for_bot(bot: str): @@ -81,6 +81,7 @@ def train_model_for_bot(bot: str): raise AppException(e) return model + def start_training(bot: str, user: str, token: str = None): """ prevents training of the bot, @@ -100,8 +101,8 @@ def start_training(bot: str, user: str, token: str = None): settings = processor.get_bot_settings(bot, user) settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: - llm = LLMFactory.get_instance("faq")(bot, settings["llm_settings"]) - faqs = asyncio.run(llm.train()) + llm_processor = LLMProcessor(bot) + faqs = asyncio.run(llm_processor.train(user=user, bot=bot)) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) agent_url = Utility.environment['model']['agent'].get('url') diff --git a/metadata/integrations.yml b/metadata/integrations.yml index f65e67b57..227b4c413 100644 --- a/metadata/integrations.yml +++ b/metadata/integrations.yml @@ -95,59 +95,74 @@ live_agents: websocket_url: wss://app.chatwoot.com/cable llm: - gpt: - temperature: - type: float - default: 0.0 - min: 0.0 - max: 2.0 - description: "The temperature hyperparameter controls the creativity or randomness of the generated responses." - max_tokens: - type: int - default: 300 - min: 5 - max: 4096 - description: "The max_tokens hyperparameter limits the length of generated responses in chat completion using ChatGPT." - model: - type: str - default: "gpt-3.5-turbo" - description: "The model hyperparameter is the ID of the model to use such as gpt-2, gpt-3, or a custom model that you have trained or fine-tuned." - top_p: - type: float - default: 0.0 - min: 0.0 - max: 1.0 - description: "The top_p hyperparameter is a value that controls the diversity of the generated responses." - n: - type: int - default: 1 - min: 1 - max: 5 - description: "The n hyperparameter controls the number of different response options that are generated by the model." - stream: - type: bool - default: false - description: "the stream hyperparameter controls whether the model generates a continuous stream of text or a single response." - stop: - type: - - str - - array - - int - default: null - description: "The stop hyperparameter is used to specify a list of tokens that should be used to indicate the end of a generated response." - presence_penalty: - type: float - default: 0.0 - min: -2.0 - max: 2.0 - description: "The presence_penalty hyperparameter penalizes the model for generating words that are not present in the context or input prompt. " - frequency_penalty: - type: float - default: 0.0 - min: -2.0 - max: 2.0 - description: "The frequency_penalty hyperparameter penalizes the model for generating words that have already been generated in the current response." - logit_bias: - type: dict - default: {} - description: "The logit_bias hyperparameter helps prevent GPT-3 from generating unwanted tokens or even to encourage generation of tokens that you do want. " + openai: + $schema: "https://json-schema.org/draft/2020-12/schema" + type: object + description: "Open AI Models for Prompt" + properties: + temperature: + type: number + default: 0.0 + minimum: 0.0 + maximum: 2.0 + description: "The temperature hyperparameter controls the creativity or randomness of the generated responses." + max_tokens: + type: integer + default: 300 + minimum: 5 + maximum: 4096 + description: "The max_tokens hyperparameter limits the length of generated responses in chat completion using ChatGPT." + model: + type: string + default: "gpt-3.5-turbo" + enum: ["gpt-3.5-turbo", "gpt-3.5-turbo-instruct"] + description: "The model hyperparameter is the ID of the model to use such as gpt-2, gpt-3, or a custom model that you have trained or fine-tuned." + top_p: + type: number + default: 0.0 + minimum: 0.0 + maximum: 1.0 + description: "The top_p hyperparameter is a value that controls the diversity of the generated responses." + n: + type: integer + default: 1 + minimum: 1 + maximum: 5 + description: "The n hyperparameter controls the number of different response options that are generated by the model." + stream: + type: boolean + default: false + description: "the stream hyperparameter controls whether the model generates a continuous stream of text or a single response." + stop: + anyOf: + - type: "string" + - type: "array" + maxItems: 4 + items: + type: "string" + - type: "integer" + - type: "null" + + type: + - "string" + - "array" + - "integer" + - "null" + default: null + description: "The stop hyperparameter is used to specify a list of tokens that should be used to indicate the end of a generated response." + presence_penalty: + type: number + default: 0.0 + minimum: -2.0 + maximum: 2.0 + description: "The presence_penalty hyperparameter penalizes the model for generating words that are not present in the context or input prompt. " + frequency_penalty: + type: number + default: 0.0 + minimum: -2.0 + maximum: 2.0 + description: "The frequency_penalty hyperparameter penalizes the model for generating words that have already been generated in the current response." + logit_bias: + type: object + default: {} + description: "The logit_bias hyperparameter helps prevent GPT-3 from generating unwanted tokens or even to encourage generation of tokens that you do want. " diff --git a/requirements/dev.txt b/requirements/dev.txt index d411ab022..19268e061 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,16 +1,15 @@ -r prod.txt -pytest==8.1.1 +pytest==8.2.2 pytest_httpx==0.30.0 -pytest-asyncio==0.23.6 -responses==0.25.0 +pytest-asyncio==0.23.7 +responses==0.25.2 mock==5.1.0 -moto[all]==5.0.5 +moto[all]==5.0.9 mongomock==4.1.2 black==22.12.0 -locust==2.25.0 +locust==2.29.0 deepdiff==7.0.1 pytest-cov==5.0.0 pytest-html==4.1.1 pytest-aioresponses==0.2.0 -aioresponses==0.7.6 -pykwalify==1.8.0 \ No newline at end of file +aioresponses==0.7.6 \ No newline at end of file diff --git a/requirements/prod.txt b/requirements/prod.txt index 19c6817ff..e00d448a9 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1,65 +1,69 @@ rasa[full]==3.6.20 mongoengine==0.28.2 fastapi==0.110.2 -uvicorn[standard]==0.29.0 +uvicorn[standard]==0.30.1 smart-config==0.1.3 fastapi_sso==0.9.1 fastapi-keycloak==1.0.10 pykka==3.1.1 zenpy==2.0.42 -validators==0.28.0 +validators==0.28.3 secure==0.3.0 password-strength==0.0.3.post2 beautifulsoup4==4.12.3 uuid6==2024.01.12 passlib[bcrypt]==1.7.4 -openai==0.28.1 json2html==1.3.0 -google-api-python-client==2.110.0 -jira==3.5.2 +google-api-python-client==2.133.0 +jira==3.8.0 pipedrive-python-lib==1.2.3 -google-cloud-translate==3.13.0 -blinker==1.7.0 -pymupdf==1.23.7 -python-docx==1.1.0 +google-cloud-translate==3.15.3 +blinker==1.8.2 +pymupdf==1.24.5 +python-docx==1.1.2 python-multipart==0.0.9 pandas==2.2.2 -openpyxl==3.1.2 +openpyxl==3.1.4 sentencepiece==0.1.99 -dramatiq==1.15.0 +dramatiq==1.17.0 dramatiq-mongodb==0.8.3 nlpaug==1.1.11 keybert==0.8.4 -pyTelegramBotAPI==4.17.0 +pyTelegramBotAPI==4.19.1 APScheduler==3.9.1.post1 -croniter==2.0.3 +croniter==2.0.5 faiss-cpu==1.8.0 -tiktoken==0.6.0 +tiktoken==0.7.0 RestrictedPython==7.1 -AccessControl==6.3 +AccessControl==7.0 timeout-decorator==0.5.0 -googlesearch-python==1.2.3 +googlesearch-python==1.2.4 aiohttp-retry==2.8.3 pqdict==1.4.0 google-businessmessages==1.0.5 google-apitools==0.5.32 -orjson==3.10.1 -opentelemetry-distro[otlp]==0.45b0 +orjson==3.10.5 +opentelemetry-distro[otlp]==0.46b0 opentelemetry-sdk-extension-aws==2.0.1 opentelemetry-propagator-aws-xray==1.0.1 -opentelemetry-instrumentation-fastapi==0.45b0 -opentelemetry-instrumentation-aiohttp-client==0.45b0 -opentelemetry-instrumentation-asyncio==0.45b0 -opentelemetry-instrumentation-aws-lambda==0.45b0 -opentelemetry-instrumentation-boto==0.45b0 -opentelemetry-instrumentation-botocore==0.45b0 -opentelemetry-instrumentation-httpx==0.45b0 -opentelemetry-instrumentation-logging==0.45b0 -opentelemetry-instrumentation-pymongo==0.45b0 -opentelemetry-instrumentation-requests==0.45b0 -opentelemetry-instrumentation-system-metrics==0.45b0 -opentelemetry-instrumentation-grpc==0.45b0 -opentelemetry-instrumentation-sklearn==0.45b0 -opentelemetry-instrumentation-asgi==0.45b0 +opentelemetry-instrumentation-fastapi==0.46b0 +opentelemetry-instrumentation-aiohttp-client==0.46b0 +opentelemetry-instrumentation-asyncio==0.46b0 +opentelemetry-instrumentation-aws-lambda==0.46b0 +opentelemetry-instrumentation-boto==0.46b0 +opentelemetry-instrumentation-botocore==0.46b0 +opentelemetry-instrumentation-httpx==0.46b0 +opentelemetry-instrumentation-logging==0.46b0 +opentelemetry-instrumentation-pymongo==0.46b0 +opentelemetry-instrumentation-requests==0.46b0 +opentelemetry-instrumentation-system-metrics==0.46b0 +opentelemetry-instrumentation-grpc==0.46b0 +opentelemetry-instrumentation-sklearn==0.46b0 +opentelemetry-instrumentation-asgi==0.46b0 +opentelemetry-instrumentation-requests==0.46b0 +opentelemetry-instrumentation-sklearn==0.46b0 pykwalify==1.8.0 gunicorn==22.0.0 +litellm==1.38.11 +jsonschema_rs==0.18.0 +mongoengine-jsonschema==0.1.3 \ No newline at end of file diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index ac56df754..8d0219d28 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -2,6 +2,7 @@ import os from urllib.parse import urlencode, urljoin +import litellm import mock import numpy as np import pytest @@ -10,8 +11,11 @@ from deepdiff import DeepDiff from fastapi.testclient import TestClient from jira import JIRAError -from mock import patch -from mongoengine import connect, DoesNotExist +from mongoengine import connect + +from kairon.shared.utils import Utility + +Utility.load_system_metadata() from kairon.actions.definitions.live_agent import ActionLiveAgent from kairon.actions.definitions.set_slot import ActionSetSlot @@ -33,15 +37,13 @@ DEFAULT_NLU_FALLBACK_RESPONSE from kairon.shared.data.data_objects import Slots, KeyVault, BotSettings, LLMSettings from kairon.shared.data.processor import MongoProcessor -from kairon.shared.llm.clients.gpt3 import GPT3Resources -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding -from kairon.shared.utils import Utility from kairon.shared.vector_embeddings.db.qdrant import Qdrant os.environ['ASYNC_TEST_TIMEOUT'] = "360" os.environ["system_file"] = "./tests/testing_data/system.yaml" client = TestClient(action) +OPENAI_EMBEDDING_OUTPUT = 1536 @pytest.fixture(autouse=True, scope='class') @@ -83,7 +85,8 @@ def test_live_agent_action_execution(aioresponses): aioresponses.add( method="POST", url=f"{Utility.environment['live_agent']['url']}/conversation/request", - payload={"success": True, "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": None }, "message": None, "error_code": 0}, + payload={"success": True, "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": None}, "message": None, + "error_code": 0}, body={'bot_id': '5f50fd0a56b698ca10d35d2z', 'sender_id': 'default', 'channel': 'messenger'}, status=200 ) @@ -186,7 +189,9 @@ def test_live_agent_action_execution_no_agent_available(aioresponses): aioresponses.add( method="POST", url=f"{Utility.environment['live_agent']['url']}/conversation/request", - payload={"success": True, "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": "live agent is not available" }, "message": None, "error_code": 0}, + payload={"success": True, + "data": {"identifier": "asjlbceuwvbalncouabvlvnlavni", "msg": "live agent is not available"}, + "message": None, "error_code": 0}, body={'bot_id': '5f50fd0a56b698ca10d35d2z', 'sender_id': 'default', 'channel': 'messenger'}, status=200 ) @@ -276,7 +281,6 @@ def test_live_agent_action_execution_no_agent_available(aioresponses): assert response_json['responses'][0]['text'] == 'live agent is not available' - def test_live_agent_action_execution_with_exception(aioresponses): bot_settings = BotSettings(bot='5f50fd0a56b698ca10d35d21', user='user') bot_settings.live_agent_enabled = True @@ -385,7 +389,9 @@ def test_live_agent_action_execution_with_exception(aioresponses): assert response.status_code == 200 assert len(response_json['responses']) == 1 assert response_json['responses'][0]['text'] == 'Connecting to live agent' - assert response_json == {'events': [], 'responses': [{'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}]} + assert response_json == {'events': [], 'responses': [ + {'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, + 'response': None, 'image': None, 'attachment': None}]} def test_live_agent_action_execution_with_exception(aioresponses): @@ -496,11 +502,12 @@ def test_live_agent_action_execution_with_exception(aioresponses): assert response.status_code == 200 assert len(response_json['responses']) == 1 assert response_json['responses'][0]['text'] == 'Connecting to live agent' - assert response_json == {'events': [], 'responses': [{'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}]} + assert response_json == {'events': [], 'responses': [ + {'text': 'Connecting to live agent', 'buttons': [], 'elements': [], 'custom': {}, 'template': None, + 'response': None, 'image': None, 'attachment': None}]} def test_retrieve_config_failure(): - patch('kairon.actions.definitions.live_agent.LiveAgentActionConfig.objects().get', side_effect=DoesNotExist) action_live_agent = ActionLiveAgent(bot='test_bot', name='test_action') with pytest.raises(ActionFailure, match="No Live Agent action found for given action and bot"): action_live_agent.retrieve_config() @@ -533,14 +540,22 @@ def test_pyscript_action_execution(): json={"success": True, "data": {"bot_response": {'numbers': [1, 2, 3, 4, 5], 'total': 15, 'i': 5}, "slots": {"location": "Bangalore", "langauge": "Kannada"}, "type": "json"}, "message": None, "error_code": 0}, - match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'chat_log': [], 'intent': 'pyscript_action', - 'kairon_user_msg': None, 'key_vault': {}, 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, - 'sender_id': 'default', 'session_started': None, - 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'langauge': 'Kannada', 'location': 'Bangalore'}, - 'user_message': 'get intents'} - - })] + match=[responses.matchers.json_params_matcher({'source_code': script, + 'predefined_objects': {'chat_log': [], + 'intent': 'pyscript_action', + 'kairon_user_msg': None, 'key_vault': {}, + 'latest_message': {'intent_ranking': [ + {'name': 'pyscript_action'}], + 'text': 'get intents'}, + 'sender_id': 'default', + 'session_started': None, + 'slot': { + 'bot': '5f50fd0a56b698ca10d35d2z', + 'langauge': 'Kannada', + 'location': 'Bangalore'}, + 'user_message': 'get intents'} + + })] ) request_object = { @@ -666,6 +681,7 @@ def test_pyscript_action_execution_with_multiple_utterances(): assert response_json['responses'][0]['custom'] == {'text': 'Hello!'} assert response_json['responses'][1]['text'] == 'How can I help you?' + @responses.activate def test_pyscript_action_execution_with_multiple_integer_utterances(): import textwrap @@ -779,7 +795,9 @@ def test_pyscript_action_execution_with_bot_response_none(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -854,7 +872,9 @@ def test_pyscript_action_execution_with_type_json_bot_response_none(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -929,7 +949,9 @@ def test_pyscript_action_execution_with_type_json_bot_response_str(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1006,7 +1028,9 @@ def test_pyscript_action_execution_with_other_type(): "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1081,7 +1105,9 @@ def test_pyscript_action_execution_with_slots_not_dict_type(): "slots": "invalid slots values"}, "message": None, "error_code": 0}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1176,7 +1202,7 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url(mock_trigger_l }, "version": "version" } - with patch("kairon.shared.utils.Utility.environment", new=mock_environment): + with mock.patch("kairon.shared.utils.Utility.environment", new=mock_environment): mock_trigger_lambda.return_value = \ {"Payload": {"body": {"bot_response": "Successfully Evaluated the pyscript", "slots": {"location": "Bangalore", "langauge": "Kannada"}}}, "StatusCode": 200} @@ -1186,15 +1212,17 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url(mock_trigger_l assert len(response_json['events']) == 3 assert len(response_json['responses']) == 1 assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'location', 'value': 'Bangalore'}, - {'event': 'slot', 'timestamp': None, 'name': 'langauge', 'value': 'Kannada'}, - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "Successfully Evaluated the pyscript"}] + {'event': 'slot', 'timestamp': None, 'name': 'location', 'value': 'Bangalore'}, + {'event': 'slot', 'timestamp': None, 'name': 'langauge', 'value': 'Kannada'}, + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "Successfully Evaluated the pyscript"}] assert response_json['responses'][0]['text'] == "Successfully Evaluated the pyscript" called_args = mock_trigger_lambda.call_args assert called_args.args[1] == \ {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {"bot": "5f50fd0a56b698ca10d35d2z", "location": "Bangalore", "langauge": "Kannada"}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, @@ -1253,7 +1281,7 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url_raise_exceptio }, "version": "version" } - with patch("kairon.shared.utils.Utility.environment", new=mock_environment): + with mock.patch("kairon.shared.utils.Utility.environment", new=mock_environment): mock_trigger_lambda.return_value = {"Payload": {"body": "Failed to evaluated the pyscript"}, "StatusCode": 422} response = client.post("/webhook", json=request_object) response_json = response.json() @@ -1261,8 +1289,8 @@ def test_pyscript_action_execution_without_pyscript_evaluator_url_raise_exceptio assert len(response_json['events']) == 1 assert len(response_json['responses']) == 1 assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "I have failed to process your request"}] + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "I have failed to process your request"}] log = ActionServerLogs.objects(action=action_name).get().to_mongo().to_dict() assert log['exception'] == "Failed to evaluated the pyscript" @@ -1297,7 +1325,9 @@ def raise_custom_exception(request): "POST", Utility.environment['evaluator']['pyscript']['url'], callback=raise_custom_exception, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1308,7 +1338,7 @@ def raise_custom_exception(request): "tracker": { "sender_id": "default", "conversation_id": "default", - "slots": {"bot": "5f50fd0a56b698ca10d35d2z", "location": "Bangalore", "langauge": "Kannada"}, + "slots": {"bot": "5f50fd0a56b698ca10d35d2z", "location": "Bangalore", "langauge": "Kannada"}, "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'pyscript_action'}]}, "latest_event_time": 1537645578.314389, "followup_action": "action_listen", @@ -1368,7 +1398,9 @@ def test_pyscript_action_execution_with_invalid_response(): "error_code": 422}, match=[responses.matchers.json_params_matcher( {'source_code': script, - 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], 'text': 'get intents'}, + 'predefined_objects': {'sender_id': 'default', 'user_message': 'get intents', + 'latest_message': {'intent_ranking': [{'name': 'pyscript_action'}], + 'text': 'get intents'}, 'slot': {'bot': '5f50fd0a56b698ca10d35d2z', 'location': 'Bangalore', 'langauge': 'Kannada'}, 'intent': 'pyscript_action', 'chat_log': [], 'key_vault': {}, 'kairon_user_msg': None, 'session_started': None}})] @@ -1410,7 +1442,8 @@ def test_pyscript_action_execution_with_invalid_response(): {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': 'I have failed to process your request'}] log = ActionServerLogs.objects(action=action_name).get().to_mongo().to_dict() - assert log['exception'] == 'Pyscript evaluation failed: {\'success\': False, \'data\': None, \'message\': \'Script execution error: ("Line 2: SyntaxError: invalid syntax at statement: for i in 10",)\', \'error_code\': 422}' + assert log[ + 'exception'] == 'Pyscript evaluation failed: {\'success\': False, \'data\': None, \'message\': \'Script execution error: ("Line 2: SyntaxError: invalid syntax at statement: for i in 10",)\', \'error_code\': 422}' def test_http_action_execution(aioresponses): @@ -1510,8 +1543,42 @@ def test_http_action_execution(aioresponses): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'slots', 'data': [{'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, {'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', 'evaluation_type': 'expression', 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot_response_log': ['evaluation_type: expression', 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, {'type': 'api_call', 'headers': {'botid': '**********************2e', 'userid': '****', 'tag': '******ot', 'email': '*******************om'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot', 'name': 'udit', 'contact': ''}, 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'status_code': 200}, {'type': 'params_list', 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot', 'name': 'udit', 'contact': ''}, 'request_params': {'bot': '**********************2e', 'user': '1011', 'tag': '******ot', 'name': '****', 'contact': None}}, {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', 'expression: ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'response: red']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'slots', 'data': [ + {'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, + {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, + {'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + 'evaluation_type': 'expression', + 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot_response_log': ['evaluation_type: expression', + 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, + {'type': 'api_call', + 'headers': {'botid': '**********************2e', 'userid': '****', 'tag': '******ot', + 'email': '*******************om'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot', 'name': 'udit', + 'contact': ''}, + 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, + 'status_code': 200}, {'type': 'params_list', + 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', + 'tag': 'from_bot', 'name': 'udit', 'contact': ''}, + 'request_params': {'bot': '**********************2e', 'user': '1011', + 'tag': '******ot', 'name': '****', 'contact': None}}, + {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, + 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', + 'expression: ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', + 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'response: red']}] + assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} def test_http_action_execution_returns_custom_json(aioresponses): @@ -1974,8 +2041,41 @@ def test_http_action_execution_no_response_dispatch(aioresponses): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'slots', 'data': [{'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, {'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', 'evaluation_type': 'expression', 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot_response_log': ['evaluation_type: expression', 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot'}, 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'status_code': 200}, {'type': 'params_list', 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot'}, 'request_params': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': '******ot'}}, {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', 'expression: ${data.a.b.d}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'response: red']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_no_response_dispatch', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'slots', 'data': [ + {'name': 'val_d', 'value': '${data.a.b.d}', 'evaluation_type': 'expression', 'slot_value': None}, + {'name': 'val_d_0', 'value': '${data.a.b.d.0}', 'evaluation_type': 'expression', 'slot_value': None}]}, + {'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', + 'data': 'The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + 'evaluation_type': 'expression', + 'response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot_response_log': ['evaluation_type: expression', + 'expression: The value of ${data.a.b.3} in ${data.a.b.d.0} is ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: The value of 2 in red is ['red', 'buggy', 'bumpers']"]}, + {'type': 'api_call', + 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, + 'method': 'GET', 'url': 'http://localhost:8081/mock', + 'payload': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', 'tag': 'from_bot'}, + 'response': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, + 'status_code': 200}, {'type': 'params_list', + 'request_body': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', + 'tag': 'from_bot'}, + 'request_params': {'bot': '5f50fd0a56b698ca10d35d2e', 'user': '1011', + 'tag': '******ot'}}, + {'type': 'filled_slots', 'data': {'val_d': "['red', 'buggy', 'bumpers']", 'val_d_0': 'red'}, + 'slot_eval_log': ['initiating slot evaluation', 'Slot: val_d', 'evaluation_type: expression', + 'expression: ${data.a.b.d}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + "response: ['red', 'buggy', 'bumpers']", 'Slot: val_d_0', + 'evaluation_type: expression', 'expression: ${data.a.b.d.0}', + "data: {'data': {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'buggy', 'bumpers']}}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'response: red']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_no_response_dispatch', 'sender': 'default', 'headers': {}, + 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': "The value of 2 in red is ['red', 'buggy', 'bumpers']", + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2074,8 +2174,22 @@ def test_http_action_execution_script_evaluation(aioresponses): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'params_list', 'request_body': {}, 'request_params': {}}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {}, 'response': {'a': 10, 'b': { + 'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, + {'type': 'params_list', 'request_body': {}, 'request_params': {}}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation', 'sender': 'default', 'headers': {}, + 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2200,8 +2314,34 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_post(aiores if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'POST', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_post', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'POST', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'POST', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_post', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'POST', + 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', + 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2325,8 +2465,34 @@ def test_http_action_execution_script_evaluation_with_dynamic_params(aioresponse if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params', 'sender': 'default', + 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', + 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2451,8 +2617,31 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_returns_cus if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'json', 'data': 'bot_response = data', 'evaluation_type': 'script', 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'bot_response_log': ['evaluation_type: script', 'script: bot_response = data', "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_returns_custom_json', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': "{'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}", 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [ + {'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'json', 'data': 'bot_response = data', + 'evaluation_type': 'script', + 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, + 'bot_response_log': ['evaluation_type: script', 'script: bot_response = data', + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, + {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, + 'method': 'GET', 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, + 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_returns_custom_json', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': "{'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}", + 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, + 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2577,8 +2766,34 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_no_response if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_no_response_dispatch', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} + assert events == [{'type': 'response', 'dispatch_bot_response': False, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert log == {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_no_response_dispatch', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', + 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', + 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200} @responses.activate @@ -2836,7 +3051,7 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_and_params_ resp_msg = json.dumps(data_obj) aioresponses.add( method=responses.GET, - url=http_url+"?intent=test_run&sender_id=default&user_message=get+intents", + url=http_url + "?intent=test_run&sender_id=default&user_message=get+intents", body=resp_msg, status=200 ) @@ -2901,8 +3116,35 @@ def test_http_action_execution_script_evaluation_with_dynamic_params_and_params_ if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': {'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', 'url': 'http://localhost:8081/mock', 'payload': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'response': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'status_code': 200}, {'type': 'dynamic_params', 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, 'slots': {}, 'request_params': ['evaluation_type: script', "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", 'raise_err_on_failure: True']}, {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] - assert not DeepDiff(log, {'type': 'http_action', 'intent': 'test_run', 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_and_params_list', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', 'http_status_code': 200}, ignore_order=True) + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': "bot_response = data['b']['name']", 'evaluation_type': 'script', 'response': 'Mayank', + 'bot_response_log': ['evaluation_type: script', "script: bot_response = data['b']['name']", + "data: {'data': {'a': 10, 'b': {'name': 'Mayank', 'arr': ['red', 'green', 'hotpink']}}, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 200}", + 'raise_err_on_failure: True']}, {'type': 'api_call', 'headers': { + 'botid': '5f50fd0a56b698ca10d35d2e', 'userid': '****', 'tag': '******ot'}, 'method': 'GET', + 'url': 'http://localhost:8081/mock', + 'payload': {'sender_id': 'default', + 'user_message': 'get intents', + 'intent': 'test_run'}, + 'response': {'a': 10, + 'b': {'name': 'Mayank', + 'arr': ['red', 'green', + 'hotpink']}}, + 'status_code': 200}, + {'type': 'dynamic_params', + 'data': "body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + 'response': {'sender_id': 'default', 'user_message': 'get intents', 'intent': 'test_run'}, + 'slots': {}, 'request_params': ['evaluation_type: script', + "script: body = {'sender_id': sender_id, 'user_message': user_message, 'intent': intent}", + "data: {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}", + 'raise_err_on_failure: True']}, + {'type': 'filled_slots', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']}] + assert not DeepDiff(log, {'type': 'http_action', 'intent': 'test_run', + 'action': 'test_http_action_execution_script_evaluation_with_dynamic_params_and_params_list', + 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8081/mock', + 'request_method': 'GET', 'bot_response': 'Mayank', 'bot': '5f50fd0a56b698ca10d35d2e', + 'status': 'SUCCESS', 'fail_reason': None, 'user_msg': 'get intents', + 'http_status_code': 200}, ignore_order=True) @responses.activate @@ -2942,7 +3184,7 @@ def test_http_action_execution_script_evaluation_failure_no_dispatch(aioresponse aioresponses.add( method=responses.GET, - url=http_url+"?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", + url=http_url + "?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", body=resp_msg, status=200 ) @@ -3040,7 +3282,7 @@ def test_http_action_execution_script_evaluation_failure_and_dispatch(aiorespons aioresponses.add( method=responses.GET, - url=http_url+"?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", + url=http_url + "?bot=5f50fd0a56b698ca10d35d2d&tag=from_bot&user=1011", body=resp_msg, status=200, ) @@ -3199,12 +3441,12 @@ def test_http_action_execution_script_evaluation_failure_and_dispatch_2(aiorespo assert response_json['events'] == [ {"event": "slot", "timestamp": None, "name": "kairon_action_response", "value": "I have failed to process your request"}, - {"event": "slot", "timestamp": None, "name": "http_status_code", "value": 200},] + {"event": "slot", "timestamp": None, "name": "http_status_code", "value": 200}, ] assert response_json['responses'][0]['text'] == "I have failed to process your request" -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.http.ActionHTTP.retrieve_config") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.http.ActionHTTP.retrieve_config") @mock.patch("kairon.shared.rest_client.AioRestClient._AioRestClient__trigger", autospec=True) def test_http_action_failed_execution(mock_trigger_request, mock_action_config, mock_action): action_name = "test_run_with_get" @@ -3272,8 +3514,18 @@ def _get_action(*arge, **kwargs): if event.get('time_elapsed') is not None: del event['time_elapsed'] print(events) - assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', 'data': 'The value of ${a.b.3} in ${a.b.d.0} is ${a.b.d}', 'evaluation_type': 'expression', 'exception': 'I have failed to process your request'}, {'type': 'api_call', 'headers': {}, 'method': 'GET', 'url': 'http://localhost:8800/mock', 'payload': {}, 'response': None, 'status_code': 408, 'exception': "Got non-200 status code:408 http_response:{'data': None, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 408}"}, {'type': 'params_list', 'request_body': {}, 'request_params': {}}, {'type': 'filled_slots'}] - assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_run_with_get', 'sender': 'default', 'headers': {}, 'url': 'http://localhost:8800/mock', 'request_method': 'GET', 'bot_response': 'I have failed to process your request', 'bot': '5f50fd0a56b698ca10d35d2e', 'status': 'FAILURE', 'fail_reason': 'Got non-200 status code:408 http_response:None', 'user_msg': 'get intents', 'time_elapsed': 0, 'http_status_code': 408} + assert events == [{'type': 'response', 'dispatch_bot_response': True, 'dispatch_type': 'text', + 'data': 'The value of ${a.b.3} in ${a.b.d.0} is ${a.b.d}', 'evaluation_type': 'expression', + 'exception': 'I have failed to process your request'}, + {'type': 'api_call', 'headers': {}, 'method': 'GET', 'url': 'http://localhost:8800/mock', + 'payload': {}, 'response': None, 'status_code': 408, + 'exception': "Got non-200 status code:408 http_response:{'data': None, 'context': {'sender_id': 'default', 'user_message': 'get intents', 'slot': {'bot': '5f50fd0a56b698ca10d35d2e'}, 'intent': 'test_run', 'chat_log': [], 'key_vault': {'EMAIL': 'uditpandey@digite.com', 'FIRSTNAME': 'udit'}, 'latest_message': {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, 'kairon_user_msg': None, 'session_started': None, 'bot': '5f50fd0a56b698ca10d35d2e'}, 'http_status_code': 408}"}, + {'type': 'params_list', 'request_body': {}, 'request_params': {}}, {'type': 'filled_slots'}] + assert log == {'type': 'http_action', 'intent': 'test_run', 'action': 'test_run_with_get', 'sender': 'default', + 'headers': {}, 'url': 'http://localhost:8800/mock', 'request_method': 'GET', + 'bot_response': 'I have failed to process your request', 'bot': '5f50fd0a56b698ca10d35d2e', + 'status': 'FAILURE', 'fail_reason': 'Got non-200 status code:408 http_response:None', + 'user_msg': 'get intents', 'time_elapsed': 0, 'http_status_code': 408} def test_http_action_missing_action_name(): @@ -3533,7 +3785,7 @@ def test_vectordb_action_execution_embedding_search_from_value(mock_embedding): BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value", bot="5f50fd0a56b698ca10d75d2e", user="user").save() embedding = list(np.random.random(Qdrant.__embedding__)) - mock_embedding.return_value = embedding + mock_embedding.return_value = {'data': [{'embedding': embedding}]} http_url = 'http://localhost:6333/collections/5f50fd0a56b698ca10d75d2e_test_vectordb_action_execution_faq_embd/points' resp_msg = json.dumps( @@ -3976,8 +4228,8 @@ def test_vectordb_action_execution_invalid_operation_type(): log.pop('timestamp') -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.database.ActionDatabase.retrieve_config") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.database.ActionDatabase.retrieve_config") def test_vectordb_action_failed_execution(mock_action_config, mock_action): action_name = "test_run_with_get_action" payload_body = {"ids": [0], "with_payload": True, "with_vector": True} @@ -3997,7 +4249,6 @@ def test_vectordb_action_failed_execution(mock_action_config, mock_action): BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value", bot="5f50fd0a56b697ca10d35d2e", user="user").save() - def _get_action_config(*arge, **kwargs): return action_config.to_mongo().to_dict(), bot_settings.to_mongo().to_dict() @@ -4167,9 +4418,9 @@ def _get_action(*arge, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "get_action") as mock_action: + with mock.patch.object(ActionUtility, "get_action") as mock_action: mock_action.side_effect = _get_action - with patch.object(ActionSetSlot, "retrieve_config") as mocked: + with mock.patch.object(ActionSetSlot, "retrieve_config") as mocked: mocked.side_effect = _get_action_config response = client.post("/webhook", json=request_object) response_json = response.json() @@ -4230,9 +4481,9 @@ def _get_action(*arge, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "get_action") as mock_action: + with mock.patch.object(ActionUtility, "get_action") as mock_action: mock_action.side_effect = _get_action - with patch.object(ActionSetSlot, "retrieve_config") as mocked: + with mock.patch.object(ActionSetSlot, "retrieve_config") as mocked: mocked.side_effect = _get_action_config response = client.post("/webhook", json=request_object) response_json = response.json() @@ -4292,9 +4543,9 @@ def _get_action(*arge, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "get_action") as mock_action: + with mock.patch.object(ActionUtility, "get_action") as mock_action: mock_action.side_effect = _get_action - with patch.object(ActionSetSlot, "retrieve_config") as mocked: + with mock.patch.object(ActionSetSlot, "retrieve_config") as mocked: mocked.side_effect = _get_action_config response = client.post("/webhook", json=request_object) response_json = response.json() @@ -5295,9 +5546,9 @@ def test_form_validation_action_with_is_required_true_and_semantics(): @responses.activate -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_script_evaluation(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['custom_text_mail'] = open('template/emails/custom_text_mail.html', 'rb').read().decode() @@ -5380,9 +5631,9 @@ def _get_action_config(*arge, **kwargs): assert str(args[2]).__contains__("Content-Type: text/html") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -5532,9 +5783,9 @@ def _get_action_config(*arge, **kwargs): assert str(args[2]).__contains__("Subject: default test") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_with_sender_email_from_slot(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -6834,8 +7085,8 @@ def _get_action_config(*arge, **kwargs): assert logs.status == "SUCCESS" -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") def test_email_action_failed_execution(mock_action_config, mock_action): action_name = "test_run_email_action" action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") @@ -7221,7 +7472,7 @@ def _run_action(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7250,7 +7501,7 @@ def _run_action(*args, **kwargs): 'intent_ranking': [{'name': 'test_run'}], "entities": [{"value": "my custom text", "entity": KAIRON_USER_MSG_ENTITY}] } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7277,7 +7528,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["latest_message"] = { 'text': '/action_google_search', 'intent_ranking': [{'name': 'test_run'}] } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7316,7 +7567,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "what is Kanban?" - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7351,7 +7602,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "what is Kanban?" - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7388,7 +7639,7 @@ def _run_action(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "what is Kanban?" - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7488,7 +7739,7 @@ def _run_action(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7591,7 +7842,7 @@ def _run_action(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_google_search") as mocked: + with mock.patch.object(ActionUtility, "perform_google_search") as mocked: mocked.side_effect = _run_action response = client.post("/webhook", json=request_object) response_json = response.json() @@ -7789,7 +8040,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8030,7 +8281,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8149,7 +8400,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8160,9 +8411,9 @@ def _perform_web_search(*args, **kwargs): {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': 'Data science combines math, statistics, programming, analytics, AI, and machine learning to uncover insights from data. Learn how data science works, what it entails, and how it differs from data science and BI.\nTo know more, please visit: What is Data Science? | IBM\n\nData science is an interdisciplinary field that uses algorithms, procedures, and processes to examine large amounts of data in order to uncover hidden patterns, generate insights, and direct decision-making.\nTo know more, please visit: What Is Data Science? Definition, Examples, Jobs, and More'}], 'responses': [{ - 'text': 'Data science combines math, statistics, programming, analytics, AI, and machine learning to uncover insights from data. Learn how data science works, what it entails, and how it differs from data science and BI.\nTo know more, please visit: What is Data Science? | IBM\n\nData science is an interdisciplinary field that uses algorithms, procedures, and processes to examine large amounts of data in order to uncover hidden patterns, generate insights, and direct decision-making.\nTo know more, please visit: What Is Data Science? Definition, Examples, Jobs, and More', - 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, - 'image': None, 'attachment': None}]} + 'text': 'Data science combines math, statistics, programming, analytics, AI, and machine learning to uncover insights from data. Learn how data science works, what it entails, and how it differs from data science and BI.\nTo know more, please visit: What is Data Science? | IBM\n\nData science is an interdisciplinary field that uses algorithms, procedures, and processes to examine large amounts of data in order to uncover hidden patterns, generate insights, and direct decision-making.\nTo know more, please visit: What Is Data Science? Definition, Examples, Jobs, and More', + 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, + 'image': None, 'attachment': None}]} log = ActionServerLogs.objects(bot=bot, type=ActionType.web_search_action.value, status="SUCCESS").get() assert log['user_msg'] == '/action_public_search' @@ -8190,7 +8441,7 @@ def _perform_web_search(*args, **kwargs): request_object["tracker"]["sender_id"] = user request_object["tracker"]["latest_message"]['text'] = "What is Python?" - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8293,7 +8544,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8397,7 +8648,7 @@ def _perform_web_search(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "perform_web_search") as mocked: + with mock.patch.object(ActionUtility, "perform_web_search") as mocked: mocked.side_effect = _perform_web_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8420,7 +8671,7 @@ def test_process_jira_action(): def _mock_response(*args, **kwargs): return None - with patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_response): + with mock.patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_response): Actions(name=action_name, type=ActionType.jira_action.value, bot=bot, user=user).save() JiraAction( name=action_name, bot=bot, user=user, url='https://test-digite.atlassian.net', @@ -8507,7 +8758,7 @@ def _mock_response(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "create_jira_issue") as mocked: + with mock.patch.object(ActionUtility, "create_jira_issue") as mocked: mocked.side_effect = _mock_response response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8529,7 +8780,7 @@ def _mock_validation(*args, **kwargs): def _mock_response(*args, **kwargs): raise JIRAError(status_code=404, url='https://test-digite.atlassian.net') - with patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_validation): + with mock.patch('kairon.shared.actions.data_objects.JiraAction.validate', new=_mock_validation): Actions(name=action_name, type=ActionType.jira_action.value, bot=bot, user='test_user').save() JiraAction( name=action_name, bot=bot, user=user, url='https://test-digite.atlassian.net', @@ -8616,7 +8867,7 @@ def _mock_response(*args, **kwargs): }, "version": "version" } - with patch.object(ActionUtility, "create_jira_issue") as mocked: + with mock.patch.object(ActionUtility, "create_jira_issue") as mocked: mocked.side_effect = _mock_response response = client.post("/webhook", json=request_object) response_json = response.json() @@ -8812,7 +9063,7 @@ def test_process_zendesk_action(): user = 'test_user' Actions(name=action_name, type=ActionType.zendesk_action.value, bot=bot, user='test_user').save() - with patch('zenpy.Zenpy'): + with mock.patch('zenpy.Zenpy'): ZendeskAction(name=action_name, subdomain='digite751', user_name='udit.pandey@digite.com', api_token=CustomActionRequestParameters(value='1234567890'), subject='new ticket', response='ticket created', @@ -8897,7 +9148,7 @@ def test_process_zendesk_action(): "version": "version" } - with patch('zenpy.Zenpy'): + with mock.patch('zenpy.Zenpy'): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -8914,7 +9165,7 @@ def test_process_zendesk_action_failure(): user = 'test_user' Actions(name=action_name, type=ActionType.zendesk_action.value, bot=bot, user='test_user').save() - with patch('zenpy.Zenpy'): + with mock.patch('zenpy.Zenpy') as zen: ZendeskAction(name=action_name, subdomain='digite751', user_name='udit.pandey@digite.com', api_token=CustomActionRequestParameters(value='1234567890'), subject='new ticket', response='ticket created', @@ -9003,8 +9254,8 @@ def __mock_zendesk_error(*args, **kwargs): from zenpy.lib.exception import APIException raise APIException({"error": {"title": "No help desk at digite751.zendesk.com"}}) - with patch('zenpy.Zenpy') as mock: - mock.side_effect = __mock_zendesk_error + with mock.patch('zenpy.Zenpy') as zen: + zen.side_effect = __mock_zendesk_error response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -9110,7 +9361,7 @@ def test_process_pipedrive_leads_action(): user = 'test_user' Actions(name=action_name, type=ActionType.pipedrive_leads_action.value, bot=bot, user='test_user').save() - with patch('pipedrive.client.Client'): + with mock.patch('pipedrive.client.Client'): metadata = {'name': 'name', 'org_name': 'organization', 'email': 'email', 'phone': 'phone'} PipedriveLeadsAction(name=action_name, domain='https://digite751.pipedrive.com/', api_token=CustomActionRequestParameters(value='1234567890'), @@ -9209,10 +9460,10 @@ def __mock_create_leads(*args, **kwargs): def __mock_create_note(*args, **kwargs): return {"success": True, "data": {"id": 2}} - with patch('pipedrive.organizations.Organizations.create_organization', __mock_create_organization): - with patch('pipedrive.persons.Persons.create_person', __mock_create_person): - with patch('pipedrive.leads.Leads.create_lead', __mock_create_leads): - with patch('pipedrive.notes.Notes.create_note', __mock_create_note): + with mock.patch('pipedrive.organizations.Organizations.create_organization', __mock_create_organization): + with mock.patch('pipedrive.persons.Persons.create_person', __mock_create_person): + with mock.patch('pipedrive.leads.Leads.create_lead', __mock_create_leads): + with mock.patch('pipedrive.notes.Notes.create_note', __mock_create_note): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -9386,7 +9637,7 @@ def test_process_pipedrive_leads_action_failure(): user = 'test_user' Actions(name=action_name, type=ActionType.pipedrive_leads_action.value, bot=bot, user='test_user').save() - with patch('pipedrive.client.Client'): + with mock.patch('pipedrive.client.Client'): metadata = {'name': 'name', 'org_name': 'organization', 'email': 'email', 'phone': 'phone'} PipedriveLeadsAction(name=action_name, domain='https://digite751.pipedrive.com/', api_token=CustomActionRequestParameters(value='1234567890'), @@ -9476,7 +9727,7 @@ def __mock_pipedrive_error(*args, **kwargs): from pipedrive.exceptions import BadRequestError raise BadRequestError('Invalid request raised', {'error_code': 402}) - with patch('pipedrive.organizations.Organizations.create_organization', __mock_pipedrive_error): + with mock.patch('pipedrive.organizations.Organizations.create_organization', __mock_pipedrive_error): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'events': [ @@ -9921,7 +10172,7 @@ def _mock_search(*args, **kwargs): {"text": "yes", "payload": "yes"}]: yield result - with patch.object(MongoProcessor, "search_training_examples") as mock_action: + with mock.patch.object(MongoProcessor, "search_training_examples") as mock_action: mock_action.side_effect = _mock_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -9933,7 +10184,7 @@ def _mock_search(*args, **kwargs): for _ in []: yield - with patch.object(MongoProcessor, "search_training_examples") as mock_action: + with mock.patch.object(MongoProcessor, "search_training_examples") as mock_action: mock_action.side_effect = _mock_search response = client.post("/webhook", json=request_object) response_json = response.json() @@ -10824,7 +11075,7 @@ def __mock_error(*args, **kwargs): "e2e_actions": []}, "version": "2.8.15" } - with patch.object(ActionUtility, "trigger_rephrase") as mock_utils: + with mock.patch.object(ActionUtility, "trigger_rephrase") as mock_utils: mock_utils.side_effect = __mock_error response = client.post("/webhook", json=request_object) @@ -11013,7 +11264,7 @@ async def mock_process_actions(*args, **kwargs): from rasa_sdk import ActionExecutionRejection raise ActionExecutionRejection("Action Execution Rejection") - with patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): + with mock.patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'error': "Custom action 'Action Execution Rejection' rejected execution.", @@ -11023,18 +11274,17 @@ async def mock_process_actions(*args, **kwargs): from rasa_sdk.interfaces import ActionNotFoundException raise ActionNotFoundException("Action Not Found Exception") - with patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): + with mock.patch('kairon.actions.handlers.action.ActionHandler.process_actions', mock_process_actions): response = client.post("/webhook", json=request_object) response_json = response.json() assert response_json == {'error': "No registered action found for name 'Action Not Found Exception'.", 'action_name': 'Action Not Found Exception'} -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_prompt_question_from_slot(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_response_action_with_prompt_question_from_slot" @@ -11058,9 +11308,9 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11088,31 +11338,20 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[ - 3] == "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n" - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': []} - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action" @@ -11136,9 +11375,9 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11166,32 +11405,21 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[ - 3] == "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n" - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': []} - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_bot_responses_with_instructions(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_with_bot_responses_with_instructions" @@ -11216,9 +11444,9 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11247,31 +11475,20 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[ - 3] == "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n" - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': ['Answer in a short way.', 'Keep it simple.']} - - -@mock.patch.object(GPT3Resources, "invoke", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_response_action_with_query_prompt" @@ -11295,20 +11512,18 @@ def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embed 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}, {'name': 'Query Prompt', - 'data': 'If there is no specific query, assume that user is aking about java programming.', + 'data': 'If there is no specific query, assume that user is asking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True} ] - mock_completion_for_query_prompt = rephrased_query, { - 'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]} + mock_completion_for_query_prompt = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]} - mock_completion_for_answer = generated_text, { - 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion_for_answer = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_completion.side_effect = [mock_completion_for_query_prompt, mock_completion_for_answer] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11343,10 +11558,9 @@ def test_prompt_action_response_action_with_query_prompt(mock_search, mock_embed 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Explain python is called high level programming language in laymen terms? \nA:"}] -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = GPT_LLM_FAQ @@ -11368,7 +11582,7 @@ def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): }, {'name': 'Data science prompt', 'instructions': 'Answer question based on the context above.', 'type': 'user', 'source': 'bot_content', - 'data': 'data_science'} + 'data': 'data_science'}, ] aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -11384,11 +11598,14 @@ def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): status=200, payload={ 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content_two}}]}) - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() - PromptAction(name=action_name, bot=bot, user=user, llm_prompts=llm_prompts).save() + PromptAction(name=action_name, + bot=bot, + user=user, + llm_prompts=llm_prompts).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() @@ -11408,11 +11625,10 @@ def test_prompt_response_action(mock_embedding, mock_completion, aioresponses): ] -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_response_action_with_instructions(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = 'test_prompt_response_action_with_instructions' @@ -11433,9 +11649,9 @@ def test_prompt_response_action_with_instructions(mock_search, mock_embedding, m 'is_enabled': True } ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11459,11 +11675,10 @@ def test_prompt_response_action_with_instructions(mock_search, mock_embedding, m ] -@mock.patch.object(GPT3Resources, "invoke", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = GPT_LLM_FAQ @@ -11489,9 +11704,9 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text, generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11513,22 +11728,16 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', - 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above.\n \nQ: What kind of language is python? \nA:"}], - 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': True, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} - assert mock_completion.call_args.args[1] == 'chat/completions' - - -@patch("kairon.shared.llm.gpt3.openai.ChatCompletion.create", autospec=True) -@patch("kairon.shared.llm.gpt3.openai.Embedding.create", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) -def test_prompt_response_action_failure(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'udit.pandeyy', 'bot': '5f50k90a56b698ca10d35d2e'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': True, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args.kwargs, expected, ignore_order=True) + + +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +def test_prompt_response_action_failure(mock_search): from uuid6 import uuid7 action_name = GPT_LLM_FAQ @@ -11537,10 +11746,7 @@ def test_prompt_response_action_failure(mock_search, mock_embedding, mock_comple user_msg = "What kind of language is python?" bot_content = "Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected." generated_text = "I don't know." - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = convert_to_openai_object(OpenAIResponse({'data': [{'embedding': embedding}]}, {})) - mock_completion.return_value = convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11611,13 +11817,10 @@ def test_prompt_action_response_action_does_not_exists(): assert len(response_json['responses']) == 0 -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_with_static_user_prompt(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -11646,8 +11849,7 @@ def test_prompt_action_response_action_with_static_user_prompt(mock_search, mock ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_search_cache(*args, **kwargs): return {'result': []} @@ -11660,9 +11862,9 @@ def __mock_cache_result(*args, **kwargs): mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.side_effect = [__mock_search_cache(), __mock_fetch_similar(), __mock_cache_result()] Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -11692,13 +11894,10 @@ def __mock_cache_result(*args, **kwargs): @responses.activate -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.llm.gpt3.GPT3FAQEmbedding.__collection_search__", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.llm.processor.LLMProcessor.__collection_search__", autospec=True) def test_prompt_action_response_action_with_action_prompt(mock_search, mock_embedding, mock_completion, aioresponses): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -11765,16 +11964,15 @@ def test_prompt_action_response_action_with_action_prompt(mock_search, mock_embe ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -11802,24 +12000,29 @@ def __mock_fetch_similar(*args, **kwargs): assert response_json['responses'] == [ {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}] - log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, status="SUCCESS").get() - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - with open('tests/testing_data/actions/action_prompt.txt', 'r') as file: - prompt_data = file.read() - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[3] == prompt_data - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, + status="SUCCESS").get().to_mongo().to_dict() + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) @mock.patch.object(ActionUtility, "perform_google_search", autospec=True) def test_kairon_faq_response_with_google_search_prompt(mock_google_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse - action_name = "kairon_faq_action" google_action_name = "custom_search_action" bot = "5u08kd0a56b698ca10hgjgjkhgjks" @@ -11860,12 +12063,11 @@ def _run_action(*args, **kwargs): PromptAction(name=action_name, bot=bot, user=user, llm_prompts=llm_prompts).save() def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - mock_completion.return_value = generated_text - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_google_search.side_effect = _run_action request_object = json.load(open("tests/testing_data/actions/action-request.json")) @@ -11882,12 +12084,24 @@ def mock_completion_for_answer(*args, **kwargs): 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None}] - log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, status="SUCCESS").get() - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What is kanban' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - assert mock_completion.call_args.args[ - 3] == 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n' + log = ActionServerLogs.objects(bot=bot, type=ActionType.prompt_action.value, + status="SUCCESS").get().to_mongo().to_dict() + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Kanban is a workflow management tool which visualizes both the process (the workflow) and the actual work passing through that process.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], + 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) def test_prompt_response_action_with_action_not_found(): @@ -11919,13 +12133,10 @@ def test_prompt_response_action_with_action_not_found(): log['exception'] = 'No action found for given bot and name' -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_dispatch_response_disabled(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -11949,17 +12160,16 @@ def test_prompt_action_dispatch_response_disabled(mock_search, mock_embedding, m ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -12001,23 +12211,28 @@ def __mock_fetch_similar(*args, **kwargs): {'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.'} }, {'type': 'slots_to_fill', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']} ] - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What is the name of prompt?' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - with open('tests/testing_data/actions/slot_prompt.txt', 'r') as file: - prompt_data = file.read() - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[3] == prompt_data - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.actions.utils.ActionUtility.compose_response", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.compose_response", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_set_slots(mock_search, mock_slot_set, mock_mock_embedding, mock_completion): - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse - action_name = "kairon_faq_action" bot = "5u80fd0a56c908ca10d35d2sjhjhjhj" user = "udit.pandey" @@ -12040,11 +12255,10 @@ def test_prompt_action_set_slots(mock_search, mock_slot_set, mock_mock_embedding ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_completion.return_value = mock_completion_for_answer() - mock_completion.return_value = generated_text + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} log1 = ['Slot: api_type', 'evaluation_type: expression', f"data: {generated_text}", 'response: filter'] log2 = ['Slot: query', 'evaluation_type: expression', f"data: {generated_text}", 'response: {\"must\": [{\"key\": \"Date Added\", \"match\": {\"value\": 1673721000.0}}]}'] @@ -12093,26 +12307,38 @@ def mock_completion_for_answer(*args, **kwargs): assert events == [ {'type': 'llm_response', 'response': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', - 'llm_response_log': {'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}'}}, - {'type': 'slots_to_fill', 'data': {'api_type': 'filter', 'query': '{"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}'}, - 'slot_eval_log': ['initiating slot evaluation', 'Slot: api_type', 'Slot: api_type', 'evaluation_type: expression', + 'llm_response_log': { + 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}'}}, + {'type': 'slots_to_fill', + 'data': {'api_type': 'filter', 'query': '{"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}'}, + 'slot_eval_log': ['initiating slot evaluation', 'Slot: api_type', 'Slot: api_type', + 'evaluation_type: expression', 'data: {"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'response: filter', 'Slot: query', 'Slot: query', 'evaluation_type: expression', 'data: {"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'response: {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}']} ] - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == user_msg - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], + 'raw_completion_response': {'choices': [{'message': { + 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_slot_prompt(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -12136,17 +12362,16 @@ def test_prompt_action_response_action_slot_prompt(mock_search, mock_embedding, ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -12191,22 +12416,27 @@ def __mock_fetch_similar(*args, **kwargs): {'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.'} }, {'type': 'slots_to_fill', 'data': {}, 'slot_eval_log': ['initiating slot evaluation']} ] - assert log['llm_logs'] == [] - assert mock_completion.call_args.args[1] == 'What is the name of prompt?' - assert mock_completion.call_args.args[2] == 'You are a personal assistant.\n' - with open('tests/testing_data/actions/slot_prompt.txt', 'r') as file: - prompt_data = file.read() - print(mock_completion.call_args.args[3]) - assert mock_completion.call_args.args[3] == prompt_data - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = [{'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', + 'role': 'assistant'}}]}, 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s'}, 'api_key': 'keyvalue', + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_user_message_in_slot(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from openai.util import convert_to_openai_object - from openai.openai_response import OpenAIResponse from uuid6 import uuid7 action_name = "kairon_faq_action" @@ -12226,17 +12456,16 @@ def test_prompt_action_user_message_in_slot(mock_search, mock_embedding, mock_co ] def mock_completion_for_answer(*args, **kwargs): - return convert_to_openai_object( - OpenAIResponse({'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, {})) + return {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} def __mock_fetch_similar(*args, **kwargs): return {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} mock_completion.return_value = mock_completion_for_answer() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = __mock_fetch_similar() Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -12260,28 +12489,23 @@ def __mock_fetch_similar(*args, **kwargs): {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - assert mock_completion.call_args[0][1] == 'Kanban And Scrum Together?' - assert mock_completion.call_args[0][2] == 'You are a personal assistant.\n' - print(mock_completion.call_args[0][3]) - assert mock_completion.call_args[0][3] == """ -Instructions on how to use Similarity Prompt: -['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.'] -Answer question based on the context above, if answer is not in the context go check previous logs. -""" - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) -def test_prompt_action_response_action_when_similarity_is_empty(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding - from uuid6 import uuid7 + expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', + 'content': "\nInstructions on how to use Similarity Prompt:\n['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Kanban And Scrum Together? \nA:"}], + 'metadata': {'user': user, 'bot': bot}, 'api_key': value, + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) +def test_prompt_action_response_action_when_similarity_is_empty(mock_search, mock_embedding, mock_completion): action_name = "test_prompt_action_response_action_when_similarity_is_empty" bot = "5f50fd0a56b698ca10d35d2C" user = "udit.pandey" value = "keyvalue" user_msg = "What kind of language is python?" - bot_content = "Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected." generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." llm_prompts = [ {'name': 'System Prompt', @@ -12297,9 +12521,9 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc 'is_enabled': True} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = {'result': []} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() @@ -12327,29 +12551,20 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc 'response': None, 'image': None, 'attachment': None} ] - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert not mock_completion.call_args.args[3] - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [ - {'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer question based on the context above, if answer is not in the context go check previous logs.', - 'collection': 'python', 'use_similarity_prompt': True, 'top_results': 10, 'similarity_threshold': 0.7}], - 'instructions': []} - - -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) -@mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) -@patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, + {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], + 'metadata': {'user': 'udit.pandey', 'bot': bot}, 'api_key': value, + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + +@mock.patch.object(litellm, "acompletion", autospec=True) +@mock.patch.object(litellm, "aembedding", autospec=True) +@mock.patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) def test_prompt_action_response_action_when_similarity_disabled(mock_search, mock_embedding, mock_completion): - from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from uuid6 import uuid7 action_name = "test_prompt_action_response_action_when_similarity_disabled" @@ -12373,10 +12588,11 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc 'is_enabled': False} ] - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_embedding.return_value = embedding - mock_completion.return_value = generated_text - mock_search.return_value = {'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} + embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_search.return_value = { + 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() BotSettings(llm_settings=LLMSettings(enable_faq=True), bot=bot, user=user).save() PromptAction(name=action_name, bot=bot, user=user, num_bot_responses=2, llm_prompts=llm_prompts).save() @@ -12402,17 +12618,11 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc {'text': generated_text, 'buttons': [], 'elements': [], 'custom': {}, 'template': None, 'response': None, 'image': None, 'attachment': None} ] - - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == """You are a personal assistant. Answer question based on the context below.\n""" - print(mock_completion.call_args.args[3]) - assert not mock_completion.call_args.args[3] - print(mock_completion.call_args.kwargs) - assert mock_completion.call_args.kwargs == { - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}, 'query_prompt': {}, - 'previous_bot_responses': [{'role': 'user', 'content': 'hello'}, - {'role': 'assistant', 'content': 'how are you'}], 'similarity_prompt': [], - 'instructions': []} \ No newline at end of file + expected = {'messages': [ + {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, + {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, + {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], + 'metadata': {'user': user, 'bot': bot}, 'api_key': value, + 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/integration_test/chat_service_test.py b/tests/integration_test/chat_service_test.py index 45597ff76..77b828a71 100644 --- a/tests/integration_test/chat_service_test.py +++ b/tests/integration_test/chat_service_test.py @@ -5,14 +5,18 @@ from datetime import datetime, timedelta from unittest import mock from urllib.parse import urlencode, quote_plus - -from kairon.shared.live_agent.live_agent import LiveAgentHandler from kairon.shared.utils import Utility - os.environ["system_file"] = "./tests/testing_data/system.yaml" os.environ["ASYNC_TEST_TIMEOUT"] = "3600" Utility.load_environment() +Utility.load_system_metadata() + +from kairon.shared.live_agent.live_agent import LiveAgentHandler + + + + import pytest import responses from mock import patch diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 08d9bbc43..7339b2e54 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -23,6 +23,9 @@ from pydantic import SecretStr from rasa.shared.utils.io import read_config_file from slack_sdk.web.slack_response import SlackResponse +from kairon.shared.utils import Utility, MailUtility + +Utility.load_system_metadata() from kairon.api.app.main import app from kairon.events.definitions.multilingual import MultilingualEvent @@ -70,7 +73,6 @@ from kairon.shared.multilingual.utils.translator import Translator from kairon.shared.organization.processor import OrgProcessor from kairon.shared.sso.clients.google import GoogleSSO -from kairon.shared.utils import Utility, MailUtility from urllib.parse import urlencode from deepdiff import DeepDiff @@ -4137,9 +4139,10 @@ def test_get_prompt_action(): assert actual["error_code"] == 0 assert not actual["message"] actual["data"][0].pop("_id") - assert actual["data"] == [ + assert not DeepDiff(actual["data"], [ {'name': 'test_update_prompt_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ @@ -4155,7 +4158,7 @@ def test_get_prompt_action(): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], 'dispatch_response': True, - 'status': True}] + 'status': True}], ignore_order=True) def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(monkeypatch): @@ -4228,10 +4231,11 @@ def _mock_get_bot_settings(*args, **kwargs): assert actual["error_code"] == 0 assert not actual["message"] actual["data"][1].pop("_id") - assert actual["data"][1] == { + assert not DeepDiff(actual["data"][1], { 'name': 'test_add_prompt_action_with_empty_collection_for_bot_content_prompt', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -4249,7 +4253,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], - 'dispatch_response': True, 'status': True} + 'dispatch_response': True, 'status': True}, ignore_order=True) def test_add_prompt_action_with_bot_content_prompt_with_payload(monkeypatch): @@ -4316,10 +4320,11 @@ def _mock_get_bot_settings(*args, **kwargs): ) actual = response.json() actual["data"][2].pop("_id") - assert actual["data"][2] == { + assert not DeepDiff(actual["data"][2], { 'name': 'test_add_prompt_action_with_bot_content_prompt_with_payload', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -4336,7 +4341,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], - 'set_slots': [], 'dispatch_response': True, 'status': True} + 'set_slots': [], 'dispatch_response': True, 'status': True}, ignore_order=True) assert actual["success"] assert actual["error_code"] == 0 assert not actual["message"] @@ -4407,10 +4412,11 @@ def _mock_get_bot_settings(*args, **kwargs): ) actual = response.json() actual["data"][3].pop("_id") - assert actual["data"][3] == { + assert not DeepDiff(actual["data"][3], { 'name': 'test_add_prompt_action_with_bot_content_prompt_with_content', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -4428,7 +4434,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], - 'dispatch_response': True, 'status': True} + 'dispatch_response': True, 'status': True}, ignore_order=True) assert actual["success"] assert actual["error_code"] == 0 assert not actual["message"] diff --git a/tests/unit_test/action/action_test.py b/tests/unit_test/action/action_test.py index d40eb9eb4..c10994445 100644 --- a/tests/unit_test/action/action_test.py +++ b/tests/unit_test/action/action_test.py @@ -6,6 +6,8 @@ import mock from googleapiclient.http import HttpRequest from pipedrive.exceptions import UnauthorizedError, BadRequestError +from kairon.shared.utils import Utility +Utility.load_system_metadata() from kairon.actions.definitions.email import ActionEmail from kairon.actions.definitions.factory import ActionFactory @@ -42,10 +44,10 @@ from kairon.actions.handlers.processor import ActionProcessor from kairon.shared.actions.utils import ActionUtility from kairon.shared.actions.exception import ActionFailure -from kairon.shared.utils import Utility from unittest.mock import patch from urllib.parse import urlencode + class TestActions: @pytest.fixture(autouse=True, scope='class') @@ -2663,6 +2665,7 @@ def test_get_prompt_action_config(self): 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', @@ -3954,6 +3957,7 @@ def test_get_prompt_action_config_2(self): 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'dispatch_response': True, 'set_slots': [], + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', diff --git a/tests/unit_test/api/api_processor_test.py b/tests/unit_test/api/api_processor_test.py index 34a46ad81..363272358 100644 --- a/tests/unit_test/api/api_processor_test.py +++ b/tests/unit_test/api/api_processor_test.py @@ -7,6 +7,8 @@ from unittest import mock from unittest.mock import patch from urllib.parse import urljoin +from kairon.shared.utils import Utility, MailUtility +Utility.load_system_metadata() import jwt import pytest @@ -22,7 +24,6 @@ from starlette.requests import Request from starlette.responses import RedirectResponse -from kairon.api.app.routers.idp import get_idp_config from kairon.api.models import RegisterAccount, EventConfig, IDPConfig, StoryRequest, HttpActionParameters, Password from kairon.exceptions import AppException from kairon.idp.data_objects import IdpConfig @@ -42,12 +43,6 @@ from kairon.shared.organization.processor import OrgProcessor from kairon.shared.sso.clients.facebook import FacebookSSO from kairon.shared.sso.clients.google import GoogleSSO -from kairon.shared.utils import Utility, MailUtility -from kairon.exceptions import AppException -import time -from kairon.idp.data_objects import IdpConfig -from kairon.api.models import RegisterAccount, EventConfig, IDPConfig, StoryRequest, HttpActionParameters, Password -from mongomock import MongoClient os.environ["system_file"] = "./tests/testing_data/system.yaml" diff --git a/tests/unit_test/augmentation/gpt_augmentation_test.py b/tests/unit_test/augmentation/gpt_augmentation_test.py index dfdb7d42e..e743fb49c 100644 --- a/tests/unit_test/augmentation/gpt_augmentation_test.py +++ b/tests/unit_test/augmentation/gpt_augmentation_test.py @@ -1,7 +1,7 @@ from augmentation.paraphrase.gpt3.generator import GPT3ParaphraseGenerator from augmentation.paraphrase.gpt3.models import GPTRequest from augmentation.paraphrase.gpt3.gpt import GPT -import openai +from openai.resources.completions import Completions import pytest import responses @@ -61,7 +61,7 @@ def test_questions_set_generation(monkeypatch): def test_generate_questions(monkeypatch): - monkeypatch.setattr(openai.Completion, 'create', mock_create) + monkeypatch.setattr(Completions, 'create', mock_create) request_data = GPTRequest(api_key="MockKey", data=["Are there any more test questions?"], num_responses=2) @@ -73,7 +73,7 @@ def test_generate_questions(monkeypatch): def test_generate_questions_empty_api_key(monkeypatch): - monkeypatch.setattr(openai.Completion, 'create', mock_create) + monkeypatch.setattr(Completions, 'create', mock_create) request_data = GPTRequest(api_key="", data=["Are there any more test questions?"], num_responses=2) @@ -84,7 +84,7 @@ def test_generate_questions_empty_api_key(monkeypatch): def test_generate_questions_empty_data(monkeypatch): - monkeypatch.setattr(openai.Completion, 'create', mock_create) + monkeypatch.setattr(Completions, 'create', mock_create) request_data = GPTRequest(api_key="MockKey", data=[], num_responses=2) @@ -125,6 +125,6 @@ def test_generate_questions_invalid_api_key(): data=["Are there any more test questions?"], num_responses=2) gpt3_generator = GPT3ParaphraseGenerator(request_data=request_data) - with pytest.raises(APIError, match=r'.*Incorrect API key provided: InvalidKey. You can find your API key at https://beta.openai.com..*'): + with pytest.raises(APIError, match=r'.*Incorrect API key provided: InvalidKey. You can find your API key at https://platform.openai.com/account/..*'): gpt3_generator.paraphrases() diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index a1db7c0a8..896ee9de4 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -8,6 +8,11 @@ from datetime import datetime, timedelta, timezone from io import BytesIO from typing import List +from kairon.shared.utils import Utility +os.environ["system_file"] = "./tests/testing_data/system.yaml" +Utility.load_environment() +Utility.load_system_metadata() + from mock import patch import numpy as np @@ -84,12 +89,9 @@ from kairon.shared.multilingual.processor import MultilingualLogProcessor from kairon.shared.test.data_objects import ModelTestingLogs from kairon.shared.test.processor import ModelTestingLogProcessor -from kairon.shared.utils import Utility from kairon.train import train_model_for_bot, start_training - -os.environ["system_file"] = "./tests/testing_data/system.yaml" -Utility.load_environment() from deepdiff import DeepDiff +import litellm class TestMongoProcessor: @@ -213,7 +215,7 @@ def test_add_prompt_action_with_invalid_slots(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_slots', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -241,7 +243,7 @@ def test_add_prompt_action_with_invalid_http_action(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_http_action', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -270,7 +272,7 @@ def test_add_prompt_action_with_invalid_similarity_threshold(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_prompt_action_similarity', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -300,7 +302,7 @@ def test_add_prompt_action_with_invalid_top_results(self): user = 'test_user' request = {'name': 'test_prompt_action_invalid_top_results', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -348,7 +350,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): processor.add_prompt_action(request, bot, user) prompt_action = processor.get_prompt_action(bot) prompt_action[0].pop("_id") - assert prompt_action == [ + assert not DeepDiff(prompt_action, [ {'name': 'test_add_prompt_action_with_empty_collection_for_bot_content_prompt', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", @@ -356,6 +358,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'data': 'default', @@ -366,7 +369,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): 'is_enabled': True}, {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True}] + 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True}], ignore_order=True) def test_add_prompt_action_with_bot_content_prompt(self): processor = MongoProcessor() @@ -392,8 +395,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): processor.add_prompt_action(request, bot, user) prompt_action = processor.get_prompt_action(bot) prompt_action[1].pop("_id") - print(prompt_action) - assert prompt_action[1] == { + assert not DeepDiff(prompt_action[1], { 'name': 'test_add_prompt_action_with_bot_content_prompt', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", @@ -401,6 +403,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity Prompt', 'data': 'Bot_collection', @@ -411,7 +414,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): 'is_enabled': True}, {'name': 'Query Prompt', 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], - 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True} + 'instructions': [], 'set_slots': [], 'dispatch_response': True, 'status': True}, ignore_order=True) def test_add_prompt_action_with_invalid_query_prompt(self): processor = MongoProcessor() @@ -596,7 +599,7 @@ def test_add_prompt_action_with_empty_llm_prompts(self): user = 'test_user' request = {'name': 'test_add_prompt_action_with_empty_llm_prompts', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -621,13 +624,13 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) pytest.action_id = processor.add_prompt_action(request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_add_prompt_action_faq_action_with_default_values', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_add_prompt_action_faq_action_with_default_values', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -635,7 +638,7 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [{'name': 'gpt_result', 'value': '${data}', 'evaluation_type': 'expression'}, {'name': 'gpt_result_type', 'value': '${data.type}', - 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}] + 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}], ignore_order=True) def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): processor = MongoProcessor() @@ -644,14 +647,14 @@ def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_temperature_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="Temperature must be between 0.0 and 2.0!"): + with pytest.raises(ValidationError, match=re.escape("['temperature']: 3.0 is greater than the maximum of 2.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_stop_hyperparameter(self): @@ -661,7 +664,7 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_stop_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': ["\n", ".", "?", "!", ";"], 'presence_penalty': 0.0, @@ -670,7 +673,7 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} with pytest.raises(ValidationError, - match="Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers."): + match=re.escape('[\'stop\']: ["\\n",".","?","!",";"] is not valid under any of the schemas listed in the \'anyOf\' keyword')): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): @@ -681,14 +684,14 @@ def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): request = {'name': 'test_add_prompt_action_with_invalid_presence_penalty_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': -3.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="Presence penalty must be between -2.0 and 2.0!"): + with pytest.raises(ValidationError, match=re.escape("['presence_penalty']: -3.0 is less than the minimum of -2.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): @@ -699,14 +702,14 @@ def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): request = {'name': 'test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 3.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="Frequency penalty must be between -2.0 and 2.0!"): + with pytest.raises(ValidationError, match=re.escape("['frequency_penalty']: 3.0 is greater than the maximum of 2.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): @@ -716,14 +719,14 @@ def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_max_tokens_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="max_tokens must be between 5 and 4096 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['max_tokens']: 2 is less than the minimum of 5")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): @@ -733,14 +736,14 @@ def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_zero_max_tokens_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="max_tokens must be between 5 and 4096 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['max_tokens']: 0 is less than the minimum of 5")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): @@ -750,14 +753,14 @@ def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_top_p_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt-3.5-turbo', 'top_p': 3.0, 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="top_p must be between 0.0 and 1.0!"): + with pytest.raises(ValidationError, match=re.escape("['top_p']: 3.0 is greater than the maximum of 1.0")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_n_hyperparameter(self): @@ -767,14 +770,14 @@ def test_add_prompt_action_with_invalid_n_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_n_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 7, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="n must be between 1 and 5 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['n']: 7 is greater than the maximum of 5")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_zero_n_hyperparameter(self): @@ -784,14 +787,14 @@ def test_add_prompt_action_with_zero_n_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_zero_n_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 0, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="n must be between 1 and 5 and should not be 0!"): + with pytest.raises(ValidationError, match=re.escape("['n']: 0 is less than the minimum of 1")): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): @@ -801,14 +804,14 @@ def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): BotSettings(bot=bot, user=user, llm_settings=LLMSettings(enable_faq=True)).save() request = {'name': 'test_add_prompt_action_with_invalid_logit_bias_hyperparameter', 'num_bot_responses': 5, 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt - 3.5 - turbo', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 2, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': 'a'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'History Prompt', 'type': 'user', 'source': 'history', 'is_enabled': True}]} - with pytest.raises(ValidationError, match="logit_bias must be a dictionary!"): + with pytest.raises(ValidationError, match=re.escape('[\'logit_bias\']: "a" is not of type "object"')): processor.add_prompt_action(request, bot, user) def test_add_prompt_action_faq_action_already_exist(self): @@ -867,7 +870,7 @@ def test_edit_prompt_action_faq_action(self): 'source': 'static', 'is_enabled': True}], "failure_message": "updated_failure_message", "use_query_prompt": True, "use_bot_responses": True, "query_prompt": "updated_query_prompt", - "num_bot_responses": 5, "hyperparameters": Utility.get_llm_hyperparameters(), + "num_bot_responses": 5, "hyperparameters": Utility.get_llm_hyperparameters('openai'), "set_slots": [{"name": "gpt_result", "value": "${data}", "evaluation_type": "expression"}, {"name": "gpt_result_type", "value": "${data.type}", "evaluation_type": "script"}], "dispatch_response": False @@ -875,12 +878,12 @@ def test_edit_prompt_action_faq_action(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -898,7 +901,8 @@ def test_edit_prompt_action_faq_action(self): 'is_enabled': True}], 'instructions': [], 'set_slots': [{'name': 'gpt_result', 'value': '${data}', 'evaluation_type': 'expression'}, {'name': 'gpt_result_type', 'value': '${data.type}', - 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}] + 'evaluation_type': 'script'}], 'dispatch_response': False, 'status': True}], + ignore_order=True) request = {'name': 'test_edit_prompt_action_faq_action_again', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -907,10 +911,10 @@ def test_edit_prompt_action_faq_action(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_faq_action_again', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action_again', 'num_bot_responses': 5, 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, + 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, @@ -918,7 +922,7 @@ def test_edit_prompt_action_faq_action(self): {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'set_slots': [], - 'dispatch_response': True, 'status': True}] + 'dispatch_response': True, 'status': True}], ignore_order=True) def test_edit_prompt_action_with_less_hyperparameters(self): processor = MongoProcessor() @@ -951,13 +955,13 @@ def test_edit_prompt_action_with_less_hyperparameters(self): processor.edit_prompt_action(pytest.action_id, request, bot, user) action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -973,7 +977,7 @@ def test_edit_prompt_action_with_less_hyperparameters(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': [], 'set_slots': [], 'dispatch_response': True, - 'status': True}] + 'status': True}], ignore_order=True) def test_get_prompt_action_does_not_exist(self): processor = MongoProcessor() @@ -986,13 +990,13 @@ def test_get_prompt_faq_action(self): bot = 'test_bot' action = list(processor.get_prompt_action(bot)) action[0].pop("_id") - print(action) - assert action == [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, + assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_with_less_hyperparameters', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, + 'llm_type': 'openai', 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -1008,8 +1012,7 @@ def test_get_prompt_faq_action(self): 'data': 'If there is no specific query, assume that user is aking about java programming.', 'instructions': 'Answer according to the context', 'type': 'query', 'source': 'static', 'is_enabled': True}], 'instructions': [], 'set_slots': [], 'dispatch_response': True, - 'status': True}] - + 'status': True}], ignore_order=True) def test_delete_prompt_action(self): processor = MongoProcessor() bot = 'test_bot' @@ -2674,7 +2677,7 @@ def test_start_training_fail(self): assert model_training.__len__() == 1 assert model_training.first().exception in str("Training data does not exists!") - @patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @patch.object(litellm, "aembedding", autospec=True) @patch("kairon.shared.rest_client.AioRestClient.request", autospec=True) @patch("kairon.shared.account.processor.AccountProcessor.get_bot", autospec=True) @patch("kairon.train.train_model_for_bot", autospec=True) @@ -2695,8 +2698,8 @@ def test_start_training_with_llm_faq( settings = BotSettings.objects(bot=bot).get() settings.llm_settings = LLMSettings(enable_faq=True) settings.save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - mock_openai.return_value = embedding + embedding = list(np.random.random(1532)) + mock_openai.return_value = {'data': [{'embedding': embedding}]} mock_bot.return_value = {"account": 1} mock_train.return_value = f"/models/{bot}" start_training(bot, user) diff --git a/tests/unit_test/events/events_test.py b/tests/unit_test/events/events_test.py index 98ae5ff10..4f7a63438 100644 --- a/tests/unit_test/events/events_test.py +++ b/tests/unit_test/events/events_test.py @@ -17,15 +17,15 @@ from rasa.shared.constants import DEFAULT_DOMAIN_PATH, DEFAULT_DATA_PATH, DEFAULT_CONFIG_PATH from rasa.shared.importers.rasa import RasaFileImporter from responses import matchers +from kairon.shared.utils import Utility + +Utility.load_system_metadata() from kairon.shared.channels.broadcast.whatsapp import WhatsappBroadcast from kairon.shared.chat.data_objects import ChannelLogs os.environ["system_file"] = "./tests/testing_data/system.yaml" -from kairon.events.definitions.message_broadcast import MessageBroadcastEvent - -from kairon.shared.chat.broadcast.processor import MessageBroadcastProcessor from kairon.events.definitions.data_importer import TrainingDataImporterEvent from kairon.events.definitions.faq_importer import FaqDataImporterEvent from kairon.events.definitions.history_delete import DeleteHistoryEvent @@ -42,7 +42,6 @@ from kairon.shared.data.processor import MongoProcessor from kairon.shared.importer.processor import DataImporterLogProcessor from kairon.shared.test.processor import ModelTestingLogProcessor -from kairon.shared.utils import Utility from kairon.test.test_models import ModelTester os.environ["system_file"] = "./tests/testing_data/system.yaml" @@ -2020,7 +2019,7 @@ def test_execute_message_broadcast_with_pyscript_failure(self, mock_is_exist, mo bot = 'test_execute_message_broadcast_with_pyscript_failure' user = 'test_user' script = """ - import time + import os """ script = textwrap.dedent(script) config = { @@ -2057,7 +2056,7 @@ def test_execute_message_broadcast_with_pyscript_failure(self, mock_is_exist, mo logs[0][0].pop("timestamp", None) assert logs[0][0] == {"event_id": event_id, 'reference_id': reference_id, 'log_type': 'common', 'bot': bot, 'status': 'Fail', - 'user': user, "exception": "Script execution error: import of 'time' is unauthorized"} + 'user': user, "exception": "Script execution error: import of 'os' is unauthorized"} with pytest.raises(AppException, match="Notification settings not found!"): MessageBroadcastProcessor.get_settings(event_id, bot) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 7494e1c0a..c8f727db3 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -7,16 +7,17 @@ import ujson as json from aiohttp import ClientConnectionError from mongoengine import connect +from kairon.shared.utils import Utility +Utility.load_system_metadata() from kairon.exceptions import AppException from kairon.shared.admin.constants import BotSecretType from kairon.shared.admin.data_objects import BotSecrets from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT -from kairon.shared.data.data_objects import LLMSettings -from kairon.shared.llm.factory import LLMFactory -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding, LLMBase -from kairon.shared.utils import Utility +from kairon.shared.llm.processor import LLMProcessor +import litellm +from deepdiff import DeepDiff class TestLLM: @@ -26,36 +27,9 @@ def init_connection(self): Utility.load_environment() connect(**Utility.mongoengine_connection(Utility.environment['database']["url"])) - def test_llm_base_train(self): - with pytest.raises(Exception): - base = LLMBase("Test") - base.train() - - def test_llm_base_predict(self): - with pytest.raises(Exception): - base = LLMBase('Test') - base.predict("Sample") - - def test_llm_factory_invalid_type(self): - with pytest.raises(Exception): - LLMFactory.get_instance("sample")("test", LLMSettings(provider="openai").to_mongo().to_dict()) - - def test_llm_factory_faq_type(self): - BotSecrets(secret_type=BotSecretType.gpt_key.value, value='value', bot='test', user='test').save() - inst = LLMFactory.get_instance("faq")("test", LLMSettings(provider="openai").to_mongo().to_dict()) - assert isinstance(inst, GPT3FAQEmbedding) - assert inst.db_url == Utility.environment['vector']['db'] - assert inst.headers == {} - - def test_llm_factory_faq_type_set_vector_key(self): - with mock.patch.dict(Utility.environment, {'vector': {"db": "http://test:6333", 'key': 'test'}}): - inst = LLMFactory.get_instance("faq")("test", LLMSettings(provider="openai").to_mongo().to_dict()) - assert isinstance(inst, GPT3FAQEmbedding) - assert inst.db_url == Utility.environment['vector']['db'] - assert inst.headers == {'api-key': Utility.environment['vector']['key']} - @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): bot = "test_embed_faq" user = "test" value = "nupurkhare" @@ -64,19 +38,11 @@ async def test_gpt3_faq_embedding_train(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} - + embedding = list(np.random.random(LLMProcessor.__embedding__)) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -100,23 +66,26 @@ async def test_gpt3_faq_embedding_train(self, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train() + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": test_content.data} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { 'points': [{'id': test_content.vector_id, 'vector': embedding, - 'payload': {"collection_name": f"{gpt3.bot}{gpt3.suffix}", 'content': test_content.data} + 'payload': {'content': test_content.data} }]} + expected = {"model": "text-embedding-3-small", + "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aioresponses): bot = "test_embed_faq_text" user = "test" value = "nupurkhare" @@ -149,18 +118,9 @@ async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]}, - repeat=True - ) - - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + embedding = list(np.random.random(LLMProcessor.__embedding__)) + mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]} + gpt3 = LLMProcessor(bot) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -205,39 +165,36 @@ async def test_gpt3_faq_embedding_train_payload_text(self, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train() + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 3 assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'name': f"{gpt3.bot}_country_details{gpt3.suffix}", 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": '{"country":"Spain","lang":"spanish"}'} - assert list(aioresponses.requests.values())[3][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][1].kwargs['json'] == {"model": "text-embedding-3-small", - 'input': '{"lang":"spanish","role":"ds"}'} - assert list(aioresponses.requests.values())[3][1].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][2].kwargs['json'] == {"model": "text-embedding-3-small", - "input": '{"name":"Nupur","city":"Pune"}'} - assert list(aioresponses.requests.values())[3][2].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[4][0].kwargs['json'] == {'points': [{'id': test_content_two.vector_id, + assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content_two.vector_id, 'vector': embedding, - 'payload': {'collection_name': f"{gpt3.bot}_country_details{gpt3.suffix}", - 'country': 'Spain'}}]} - assert list(aioresponses.requests.values())[4][1].kwargs['json'] == {'points': [{'id': test_content_three.vector_id, + 'payload': {'country': 'Spain'}}]} + assert list(aioresponses.requests.values())[3][1].kwargs['json'] == {'points': [{'id': test_content_three.vector_id, 'vector': embedding, - 'payload': {'collection_name': f"{gpt3.bot}_country_details{gpt3.suffix}", 'role': 'ds'}}]} + 'payload': {'role': 'ds'}}]} - assert list(aioresponses.requests.values())[5][0].kwargs['json'] == {'name': f"{gpt3.bot}_user_details{gpt3.suffix}", + assert list(aioresponses.requests.values())[4][0].kwargs['json'] == {'name': f"{gpt3.bot}_user_details{gpt3.suffix}", 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[6][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + assert list(aioresponses.requests.values())[5][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, 'vector': embedding, - 'payload': {'collection_name': f"{gpt3.bot}_user_details{gpt3.suffix}", - 'name': 'Nupur'}}]} + 'payload': {'name': 'Nupur'}}]} assert response['faq'] == 3 + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + print(mock_embedding.call_args) + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_payload_with_int(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, aioresponses): bot = "test_embed_faq_json" user = "test" value = "nupurkhare" @@ -254,17 +211,11 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} + embedding = list(np.random.random(LLMProcessor.__embedding__)) input = {"name": "Ram", "color": "red"} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), method="PUT", @@ -282,21 +233,25 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, aioresponses): ) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - response = await gpt3.train() + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_embed_faq_json_payload_with_int_faq_embd', 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": json.dumps(input)} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, 'vector': embedding, - 'payload': {'name': 'Ram', 'age': 23, 'color': 'red', "collection_name": "test_embed_faq_json_payload_with_int_faq_embd"} + 'payload': {'name': 'Ram', 'age': 23, 'color': 'red'} }]} + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_int(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): bot = "test_int" user = "test" value = "nupurkhare" @@ -313,18 +268,11 @@ async def test_gpt3_faq_embedding_train_int(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": "Bearer nupurkhare"} + embedding = list(np.random.random(LLMProcessor.__embedding__)) input = {"name": "Ram", "color": "red"} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - + mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -354,28 +302,32 @@ async def test_gpt3_faq_embedding_train_int(self, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train() + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_int_embd_int_faq_embd', 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": json.dumps(input)} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header expected_payload = test_content.data - expected_payload['collection_name'] = 'test_int_embd_int_faq_embd' - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + #expected_payload['collection_name'] = 'test_int_embd_int_faq_embd' + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { 'points': [{'id': test_content.vector_id, 'vector': embedding, 'payload': expected_payload }]} + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + def test_gpt3_faq_embedding_train_failure(self): with pytest.raises(AppException, match=f"Bot secret '{BotSecretType.gpt_key.value}' not configured!"): - GPT3FAQEmbedding('test_failure', LLMSettings(provider="openai").to_mongo().to_dict()) + LLMProcessor('test_failure') @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_upsert_error(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aioresponses): bot = "test_embed_faq_not_exists" user = "test" value = "nupurk" @@ -384,19 +336,12 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, aioresponses): bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - - request_header = {"Authorization": "Bearer nupurk"} + embedding = list(np.random.random(LLMProcessor.__embedding__)) - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -423,17 +368,21 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, aioresponses): ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train() + await gpt3.train(user=user, bot=bot) assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": test_content.data} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, 'payload': {'collection_name': f"{bot}{gpt3.suffix}",'content': test_content.data}}]} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + 'vector': embedding, 'payload': {'content': test_content.data}}]} + expected = {"model": "text-embedding-3-small", + "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, aioresponses): + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_embedding, aioresponses): bot = "payload_upsert_error" user = "test" value = "nupurk" @@ -450,19 +399,11 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, aiorespo bot=bot, user=user).save() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - - request_header = {"Authorization": "Bearer nupurk"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + embedding = list(np.random.random(LLMProcessor.__embedding__)) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -489,21 +430,27 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, aiorespo ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train() + await gpt3.train(user=user, bot=bot) assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": json.dumps(test_content.data)} - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header expected_payload = test_content.data - expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + #expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, 'vector': embedding, 'payload': expected_payload }]} + expected = {"model": "text-embedding-3-small", + "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict" user = "test" @@ -516,6 +463,7 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", @@ -523,8 +471,9 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}]} - hyperparameters = Utility.get_llm_hyperparameters() + 'collection': 'python'}], + "hyperparameters": hyperparameters + } mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, @@ -532,24 +481,10 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), @@ -558,24 +493,29 @@ async def test_gpt3_faq_embedding_predict(self, aioresponses): {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response['content'] == generated_text - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = value + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_default_collection(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict_with_default_collection" user = "test" @@ -588,15 +528,17 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, aiorespo generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" - + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'default'}]} - hyperparameters = Utility.get_llm_hyperparameters() + 'collection': 'default'}], + 'hyperparameters': hyperparameters + } + mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, @@ -604,24 +546,10 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, aiorespo 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -631,40 +559,52 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, aiorespo {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) assert response['content'] == generated_text - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": value, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = value + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_embed_faq_predict", user="test").save() + collection='python', bot="test_gpt3_faq_embedding_predict_with_values", user="test").save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() + key = 'test' + user = "tests" + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}]} + 'collection': 'python'}], + "hyperparameters": hyperparameters + } - hyperparameters = Utility.get_llm_hyperparameters() mock_completion_request = {"messages": [ {"role": "system", "content": "You are a personal assistant. Answer the question according to the below context"}, @@ -672,24 +612,11 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), @@ -698,7 +625,7 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=gpt3.bot, **k_faq_action_config) assert response['content'] == generated_text assert gpt3.logs == [ {'messages': [{'role': 'system', @@ -714,25 +641,36 @@ async def test_gpt3_faq_embedding_predict_with_values(self, aioresponses): 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': gpt3.bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "test_gpt3_faq_embedding_predict_with_values_with_instructions" + key = 'test' test_content = CognitionData( data="Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ", - collection='java', bot="test_embed_faq_predict", user="test").save() - + collection='java', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", @@ -740,9 +678,10 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, ai 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', "collection": "java"}], - 'instructions': ['Answer in a short way.', 'Keep it simple.']} + 'instructions': ['Answer in a short way.', 'Keep it simple.'], + "hyperparameters": hyperparameters + } - hyperparameters = Utility.get_llm_hyperparameters() mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, @@ -750,186 +689,210 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, ai 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url="https://api.openai.com/v1/chat/completions", + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + payload={'result': [ + {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) - - aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), - method="POST", - payload={'result': [ - {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} - ) - - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response['content'] == generated_text - assert gpt3.logs == [ - {'messages': [{'role': 'system', - 'content': 'You are a personal assistant. Answer the question according to the below context'}, - {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': { - 'choices': [{'message': {'content': 'Python is dynamically typed, garbage-collected, ' - 'high level, general purpose programming.', - 'role': 'assistant'}}]}, - 'type': 'answer_query', - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}}] - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) + assert response['content'] == generated_text + assert gpt3.logs == [ + {'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': { + 'choices': [{'message': {'content': 'Python is dynamically typed, garbage-collected, ' + 'high level, general purpose programming.', + 'role': 'assistant'}}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_answer", autospec=True) - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + embedding = list(np.random.random(LLMProcessor.__embedding__)) + bot = "test_gpt3_faq_embedding_predict_completion_connection_error" + user = 'test' + key = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_embed_faq_predict", user="test").save() + collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() + hyperparameters = Utility.get_default_llm_hyperparameters() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}]} + "collection": 'python'}], + "hyperparameters": hyperparameters + } def __mock_connection_error(*args, **kwargs): - import openai - - raise openai.error.APIConnectionError("Connection reset by peer!") + raise Exception("Connection reset by peer!") - mock_embedding.return_value = embedding + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.side_effect = __mock_connection_error - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) - - aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), - method="POST", - payload={'result': [ - {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} - ) + gpt3 = LLMProcessor(test_content.bot) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - print(mock_completion.call_args.args[3]) + aioresponses.add( + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + method="POST", + payload={'result': [ + {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} + ) - assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} - assert mock_embedding.call_args.args[1] == query + assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] - assert mock_completion.call_args.args[1] == 'What kind of language is python?' - assert mock_completion.call_args.args[ - 2] == 'You are a personal assistant. Answer the question according to the below context' - assert mock_completion.call_args.args[3] == """Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n""" - assert mock_completion.call_args.kwargs == {'similarity_prompt': [ - {'top_results': 10, 'similarity_threshold': 0.7, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', 'collection': 'python'}]} - assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = {'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], + 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error'}, + 'api_key': 'test', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio @mock.patch("kairon.shared.rest_client.AioRestClient._AioRestClient__trigger", autospec=True) - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock_llm_request): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user ="test" + bot = "test_gpt3_faq_embedding_predict_exact_match" + key = 'test' test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_embed_faq_predict", user="test").save() + collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() query = "What kind of language is python?" + hyperparameters = Utility.get_default_llm_hyperparameters() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}]} + "collection": 'python'}], + "hyperparameters": hyperparameters + } - mock_embedding.return_value = embedding + mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_llm_request.side_effect = ClientConnectionError() - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} + response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_exact_match", **k_faq_action_config) + assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} - assert mock_embedding.call_args.args[1] == query - assert gpt3.logs == [] - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - @mock.patch.object(GPT3FAQEmbedding, "_GPT3FAQEmbedding__get_embedding", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_embedding): - import openai - - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "test_gpt3_faq_embedding_predict_embedding_connection_error" + key = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - bot="test_embed_faq_predict", user="test").save() + bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() + hyperparameters = Utility.get_default_llm_hyperparameters() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" + k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", - "context_prompt": "Based on below context answer question, if answer not in context check previous logs."} + "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", + "hyperparameters": hyperparameters + } + mock_embedding.side_effect = [Exception("Connection reset by peer!"), {'data': [{'embedding': embedding}]}] - mock_embedding.side_effect = [openai.error.APIConnectionError("Connection reset by peer!"), embedding] + gpt3 = LLMProcessor(test_content.bot) + mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': 'test'}}): - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_embedding_connection_error", **k_faq_action_config) + assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} + assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - assert mock_embedding.call_args.args[1] == query - assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) - bot = "test_embed_faq_predict" + bot = "test_gpt3_faq_embedding_predict_with_previous_bot_responses" user = "test" + key = "test" + hyperparameters = Utility.get_default_llm_hyperparameters() test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" @@ -940,9 +903,10 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior ], "similarity_prompt": [{'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}] + "collection": 'python'}], + "hyperparameters": hyperparameters } - hyperparameters = Utility.get_llm_hyperparameters() + mock_completion_request = {"messages": [ {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below'}, {'role': 'user', 'content': 'hello'}, @@ -951,23 +915,10 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior 'content': "Answer question based on the context below, if answer is not in the context go check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - ) - - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), @@ -976,44 +927,55 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, aior {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - print(list(aioresponses.requests.values())[2][0].kwargs['json']) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) assert response['content'] == generated_text - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + @pytest.mark.asyncio - async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding, mock_completion, aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) - bot = "test_embed_faq_predict" + bot = "test_gpt3_faq_embedding_predict_with_query_prompt" user = "test" + key = "test" test_content = CognitionData( data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", collection='python', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" rephrased_query = "Explain python is called high level programming language in laymen terms?" + hyperparameters = Utility.get_default_llm_hyperparameters() + k_faq_action_config = {"query_prompt": { "query_prompt": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", "use_query_prompt": True}, - "similarity_prompt": [ - {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}] - } - hyperparameters = Utility.get_llm_hyperparameters() + "similarity_prompt": [ + {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], + "hyperparameters": hyperparameters + } + mock_rephrase_request = {"messages": [ {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, @@ -1029,31 +991,11 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): ]} mock_rephrase_request.update(hyperparameters) mock_completion_request.update(hyperparameters) - request_header = {"Authorization": "Bearer knupur"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={'data': [{'embedding': embedding}]} - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]} - ) - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]}, - repeat=True - ) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = GPT3FAQEmbedding(test_content.bot, LLMSettings(provider="openai").to_mongo().to_dict()) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), @@ -1062,18 +1004,21 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, aioresponses): {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, **k_faq_action_config) - print(list(aioresponses.requests.values())[2][1].kwargs['json']) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) assert response['content'] == generated_text - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {"model": "text-embedding-3-small", - "input": query} - assert list(aioresponses.requests.values())[0][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == mock_rephrase_request - assert list(aioresponses.requests.values())[2][0].kwargs['headers'] == request_header - assert list(aioresponses.requests.values())[2][1].kwargs['json'] == mock_completion_request - assert list(aioresponses.requests.values())[2][1].kwargs['headers'] == request_header assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/unit_test/utility_test.py b/tests/unit_test/utility_test.py index b32c4a3b8..a0de42a47 100644 --- a/tests/unit_test/utility_test.py +++ b/tests/unit_test/utility_test.py @@ -8,6 +8,9 @@ from io import BytesIO from unittest.mock import patch, MagicMock from urllib.parse import urlencode +from kairon.shared.utils import Utility, MailUtility + +Utility.load_system_metadata() import numpy as np import pandas as pd @@ -36,12 +39,7 @@ from kairon.shared.data.data_objects import EventConfig, Slots, LLMSettings, DemoRequestLogs from kairon.shared.data.processor import MongoProcessor from kairon.shared.data.utils import DataUtility -from kairon.shared.llm.clients.azure import AzureGPT3Resources -from kairon.shared.llm.clients.factory import LLMClientFactory -from kairon.shared.llm.clients.gpt3 import GPT3Resources -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from kairon.shared.models import TemplateType -from kairon.shared.utils import Utility, MailUtility from kairon.shared.verification.email import QuickEmailVerification @@ -2122,7 +2120,7 @@ def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, user=user, ws_url="http://localhost:5000/event_url" @@ -2134,14 +2132,14 @@ def publish_auditlog(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user, action="save" ).count() - assert count == 2 + assert count == 1 def test_save_and_publish_auditlog_action_save_another(self, monkeypatch): def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2157,14 +2155,14 @@ def publish_auditlog(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user, action="save" ).count() - assert count == 3 + assert count == 2 def test_save_and_publish_auditlog_action_update(self, monkeypatch): def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2179,14 +2177,14 @@ def publish_auditlog(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user, action="update" ).count() - assert count == 2 + assert count == 1 def test_save_and_publish_auditlog_total_count(self, monkeypatch): def publish_auditlog(*args, **kwargs): return None monkeypatch.setattr(AuditDataProcessor, "publish_auditlog", publish_auditlog) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2208,7 +2206,7 @@ def execute_http_request(*args, **kwargs): return None monkeypatch.setattr(Utility, "execute_http_request", execute_http_request) - bot = "tests" + bot = "publish_auditlog" user = "testuser" event_config = EventConfig( bot=bot, @@ -2223,11 +2221,11 @@ def execute_http_request(*args, **kwargs): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user ).count() - assert count >= 3 + assert count >= 2 @responses.activate def test_publish_auditlog(self): - bot = "secret" + bot = "publish_auditlog" user = "secret_user" config = { "bot_user_oAuth_token": "xoxb-801939352912-801478018484-v3zq6MYNu62oSs8vammWOY8K", @@ -2263,7 +2261,7 @@ def test_publish_auditlog(self): count = AuditLogData.objects( attributes=[{"key": "bot", "value": bot}], user=user ).count() - assert count == 4 + assert count == 1 @pytest.mark.asyncio async def test_messageConverter_messenger_button_one(self): @@ -2956,7 +2954,7 @@ def test_verify_email_enable_valid_email(self): Utility.verify_email(email) def test_get_llm_hyperparameters(self): - hyperparameters = Utility.get_llm_hyperparameters() + hyperparameters = Utility.get_llm_hyperparameters("openai") assert hyperparameters == { "temperature": 0.0, "max_tokens": 300, @@ -2971,508 +2969,10 @@ def test_get_llm_hyperparameters(self): } def test_get_llm_hyperparameters_not_found(self, monkeypatch): - monkeypatch.setitem(Utility.environment["llm"], "faq", None) - with pytest.raises( - AppException, match="Could not find any hyperparameters for configured LLM." - ): - Utility.get_llm_hyperparameters() - - @pytest.mark.asyncio - async def test_trigger_gpt3_client_completion_with_generated_text( - self, aioresponses - ): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - messages = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = messages - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - - resp = await GPT3Resources("test").invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert resp[0] == generated_text - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gpt3_client_completion_with_response(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - - formatted_response, raw_response = await GPT3Resources("test").invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == { - "choices": [{"message": {"content": generated_text, "role": "assistant"}}] - } - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_completion(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - formatted_response, raw_response = await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == { - "choices": [{"message": {"content": generated_text, "role": "assistant"}}] - } - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_completion_failure(self, aioresponses): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=504, - payload={"error": {"message": "Server unavailable!", "id": 876543456789}}, - ) - with pytest.raises( - AppException, match="Failed to connect to service: api.openai.com" - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=201, - body="openai".encode(), - repeat=True, - ) - with pytest.raises(AppException): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_embedding(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"Authorization": f"Bearer {api_key}"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=200, - payload={"data": [{"embedding": embedding}]}, - ) - formatted_response, raw_response = await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - assert formatted_response == embedding - assert raw_response == {"data": [{"embedding": embedding}]} - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_embedding_failure(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - request_header = {"Authorization": f"Bearer {api_key}"} - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", method="POST", status=504 - ) - - with pytest.raises( - AppException, match="Failed to connect to service: api.openai.com" - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - - aioresponses.add( - url="https://api.openai.com/v1/embeddings", - method="POST", - status=204, - payload={"error": {"message": "Server unavailable!", "id": 876543456789}}, - repeat=True, - ) - - with pytest.raises( - AppException, match="Server unavailable!. Request id: 876543456789" - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - assert list(aioresponses.requests.values())[0][1].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][1].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_streaming_completion(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - content = """data: {"choices": [{"delta": {"role": "assistant"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": "Python"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " is"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " dynamically"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " typed"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " garbage-collected"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " high"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " level"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " general"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " purpose"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": " programming"}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {"content": "."}, "index": 0, "finish_reason": null}]}\n\n -data: {"choices": [{"delta": {}, "index": 0, "finish_reason": "stop"}]}\n\n -data: [DONE]\n\n""" - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - body=content.encode(), - content_type="text/event-stream", - ) - - formatted_response, raw_response = await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == [ - b'data: {"choices": [{"delta": {"role": "assistant"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": "Python"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " is"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " dynamically"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " typed"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " garbage-collected"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " high"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " level"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": ","}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " general"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " purpose"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": " programming"}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {"content": "."}, "index": 0, "finish_reason": null}]}\n', - b"\n", - b"\n", - b'data: {"choices": [{"delta": {}, "index": 0, "finish_reason": "stop"}]}\n', - b"\n", - b"\n", - b"data: [DONE]\n", - b"\n", - ] - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_streaming_connection_error(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=401, - ) - - with pytest.raises( - AppException, - match=re.escape( - "Failed to execute the url: 401, message='Unauthorized', url=URL('https://api.openai.com/v1/chat/completions')" - ), - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_streaming_completion_failure(self, aioresponses): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - content = "data: {'choices': [{'delta': {'role': 'assistant'}}]}\n\n" - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=200, - body=content.encode(), - content_type="text/event-stream", - ) - with pytest.raises( - AppException, - match=re.escape( - "Failed to parse streaming response: b\"data: {'choices': [{'delta': {'role': 'assistant'}}]}\\n\"" - ), - ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_gp3_client_completion_failure_invalid_json( - self, aioresponses - ): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"Authorization": f"Bearer {api_key}"} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - mock_completion_request["stream"] = True - - content = "data: {'choices': [{'delta': {'role': 'assistant'}}]}\n\n" - aioresponses.add( - url="https://api.openai.com/v1/chat/completions", - method="POST", - status=504, - body=content.encode(), - ) with pytest.raises( - AppException, match="Failed to connect to service: api.openai.com" + AppException, match="Could not find any hyperparameters for claude LLM." ): - await GPT3Resources(api_key).invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) + Utility.get_llm_hyperparameters("claude") def test_get_client_ip_with_request_client(self): request = MagicMock() @@ -3481,217 +2981,6 @@ def test_get_client_ip_with_request_client(self): ip = Utility.get_client_ip(request) assert "58.0.127.89" == ip - def test_llm_resource_provider_factory(self): - client = LLMClientFactory.get_resource_provider(LLMResourceProvider.azure.value) - assert isinstance(client("test"), AzureGPT3Resources) - - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.openai.value - ) - assert isinstance(client("test"), GPT3Resources) - - def test_llm_resource_provider_not_implemented(self): - with pytest.raises(AppException, match="aws client not supported"): - LLMClientFactory.get_resource_provider("aws") - - @pytest.mark.asyncio - async def test_trigger_azure_client_completion(self, aioresponses): - api_key = "test" - generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"api-key": api_key} - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['chat_completion_model_id']}/{GPT3ResourceTypes.chat_completion.value}?api-version={llm_settings['api_version']}", - method="POST", - status=200, - payload={ - "choices": [ - {"message": {"content": generated_text, "role": "assistant"}} - ] - }, - ) - - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - formatted_response, raw_response = await client.invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - assert formatted_response == generated_text - assert raw_response == { - "choices": [{"message": {"content": generated_text, "role": "assistant"}}] - } - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_azure_client_embedding(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - embedding = list(np.random.random(GPT3FAQEmbedding.__embedding__)) - request_header = {"api-key": api_key} - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['embeddings_model_id']}/{GPT3ResourceTypes.embeddings.value}?api-version={llm_settings['api_version']}", - method="POST", - status=200, - payload={"data": [{"embedding": embedding}]}, - ) - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - formatted_response, raw_response = await client.invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - assert formatted_response == embedding - assert raw_response == {"data": [{"embedding": embedding}]} - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_azure_client_embedding_failure(self, aioresponses): - api_key = "test" - query = "What kind of language is python?" - request_header = {"api-key": api_key} - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['embeddings_model_id']}/{GPT3ResourceTypes.embeddings.value}?api-version={llm_settings['api_version']}", - method="POST", - status=504, - ) - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - - with pytest.raises( - AppException, match="Failed to connect to service: kairon.openai.azure.com" - ): - await client.invoke( - GPT3ResourceTypes.embeddings.value, - model="text-embedding-3-small", - input=query, - ) - - assert list(aioresponses.requests.values())[0][0].kwargs["json"] == { - "model": "text-embedding-3-small", - "input": query, - } - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) - - @pytest.mark.asyncio - async def test_trigger_azure_client_completion_failure(self, aioresponses): - api_key = "test" - hyperparameters = Utility.get_llm_hyperparameters() - request_header = {"api-key": api_key} - llm_settings = ( - LLMSettings( - enable_faq=True, - provider="azure", - embeddings_model_id="openaimodel_embd", - chat_completion_model_id="openaimodel_completion", - api_version="2023-03-16", - ) - .to_mongo() - .to_dict() - ) - mock_completion_request = { - "messages": [ - {"role": "system", "content": DEFAULT_SYSTEM_PROMPT}, - { - "role": "user", - "content": "Answer question based on the context below, if answer is not in the context go check previous logs.\nSimilarity Prompt:\nPython is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.\nInstructions on how to use Similarity Prompt: Answer according to this context.\n \n Q: Explain python is called high level programming language in laymen terms?\n A:", - }, - ] - } - mock_completion_request.update(hyperparameters) - - aioresponses.add( - url=f"https://kairon.openai.azure.com/openai/deployments/{llm_settings['chat_completion_model_id']}/{GPT3ResourceTypes.chat_completion.value}?api-version={llm_settings['api_version']}", - method="POST", - status=504, - payload={"error": {"message": "Server unavailable!", "id": 876543456789}}, - ) - - client = LLMClientFactory.get_resource_provider( - LLMResourceProvider.azure.value - )(api_key, **llm_settings) - with pytest.raises( - AppException, match="Failed to connect to service: kairon.openai.azure.com" - ): - await client.invoke( - GPT3ResourceTypes.chat_completion.value, **mock_completion_request - ) - - assert ( - list(aioresponses.requests.values())[0][0].kwargs["json"] - == mock_completion_request - ) - assert ( - list(aioresponses.requests.values())[0][0].kwargs["headers"] - == request_header - ) @pytest.mark.asyncio async def test_messageConverter_whatsapp_dropdown(self): diff --git a/tests/unit_test/validator/training_data_validator_test.py b/tests/unit_test/validator/training_data_validator_test.py index 2633c01f1..f3aa8dc66 100644 --- a/tests/unit_test/validator/training_data_validator_test.py +++ b/tests/unit_test/validator/training_data_validator_test.py @@ -3,6 +3,8 @@ import pytest import yaml from mongoengine import connect +from kairon.shared.utils import Utility +Utility.load_system_metadata() from kairon.exceptions import AppException from kairon.importer.validator.file_validator import TrainingDataValidator @@ -797,55 +799,34 @@ def test_validate_custom_actions_with_errors(self): assert len(error_summary['google_search_actions']) == 2 assert len(error_summary['zendesk_actions']) == 2 assert len(error_summary['pipedrive_leads_actions']) == 3 - assert len(error_summary['prompt_actions']) == 49 + assert len(error_summary['prompt_actions']) == 36 assert len(error_summary['razorpay_actions']) == 3 assert len(error_summary['pyscript_actions']) == 3 assert len(error_summary['database_actions']) == 6 - required_fields_error = error_summary["prompt_actions"][21] - assert re.match(r"Required fields .* not found in action: prompt_action_with_no_llm_prompts", required_fields_error) - del error_summary["prompt_actions"][21] - print(error_summary['prompt_actions']) - assert error_summary['prompt_actions'] == ['top_results should not be greater than 30 and of type int!', - 'similarity_threshold should be within 0.3 and 1.0 and of type int or float!', - 'Collection is required for bot content prompts!', - 'System prompt is required', 'Query prompt must have static source', - 'Name cannot be empty', 'System prompt is required', - 'num_bot_responses should not be greater than 5 and of type int: prompt_action_invalid_num_bot_responses', - 'Collection is required for bot content prompts!', - 'data field in prompts should of type string.', - 'data is required for static prompts', - 'Temperature must be between 0.0 and 2.0!', - 'max_tokens must be between 5 and 4096!', - 'top_p must be between 0.0 and 1.0!', 'n must be between 1 and 5!', - 'presence_penality must be between -2.0 and 2.0!', - 'frequency_penalty must be between -2.0 and 2.0!', - 'logit_bias must be a dictionary!', - 'System prompt must have static source', - 'Collection is required for bot content prompts!', - 'Collection is required for bot content prompts!', - 'Duplicate action found: test_add_prompt_action_one', - 'Invalid action configuration format. Dictionary expected.', - 'Temperature must be between 0.0 and 2.0!', - 'max_tokens must be between 5 and 4096!', - 'top_p must be between 0.0 and 1.0!', 'n must be between 1 and 5!', - 'Stop must be None, a string, an integer, or an array of 4 or fewer strings or integers.', - 'presence_penality must be between -2.0 and 2.0!', - 'frequency_penalty must be between -2.0 and 2.0!', - 'logit_bias must be a dictionary!', - 'Only one system prompt can be present', 'Invalid prompt type', - 'Invalid prompt source', 'Only one system prompt can be present', - 'Invalid prompt type', 'Invalid prompt source', - 'type in LLM Prompts should be of type string.', - 'source in LLM Prompts should be of type string.', - 'Instructions in LLM Prompts should be of type string.', - 'Only one system prompt can be present', - 'Data must contain action name', - 'Only one system prompt can be present', - 'Data must contain slot name', - 'Only one system prompt can be present', - 'Only one system prompt can be present', - 'Only one system prompt can be present', - 'Only one history source can be present'] + expected_errors = ['top_results should not be greater than 30 and of type int!', + 'similarity_threshold should be within 0.3 and 1.0 and of type int or float!', + 'Collection is required for bot content prompts!', 'System prompt is required', + 'Query prompt must have static source', 'Name cannot be empty', 'System prompt is required', + 'num_bot_responses should not be greater than 5 and of type int: prompt_action_invalid_num_bot_responses', + 'Collection is required for bot content prompts!', + 'data field in prompts should of type string.', 'data is required for static prompts', + "['frequency_penalty']: 5 is greater than the maximum of 2.0", + 'System prompt must have static source', 'Collection is required for bot content prompts!', + 'Collection is required for bot content prompts!', + "Required fields ['llm_prompts', 'name'] not found in action: prompt_action_with_no_llm_prompts", + 'Duplicate action found: test_add_prompt_action_one', + 'Invalid action configuration format. Dictionary expected.', + "['frequency_penalty']: 5 is greater than the maximum of 2.0", + 'Only one system prompt can be present', 'Invalid prompt type', + 'Only one system prompt can be present', 'Invalid prompt type', 'Invalid prompt source', + 'type in LLM Prompts should be of type string.', + 'source in LLM Prompts should be of type string.', + 'Instructions in LLM Prompts should be of type string.', + 'Only one system prompt can be present', 'Data must contain action name', + 'Only one system prompt can be present', 'Data must contain slot name', + 'Only one system prompt can be present', 'Only one system prompt can be present', + 'Only one system prompt can be present', 'Only one history source can be present'] + assert not DeepDiff(error_summary['prompt_actions'], expected_errors, ignore_order=True) assert component_count == {'http_actions': 7, 'slot_set_actions': 10, 'form_validation_actions': 9, 'email_actions': 5, 'google_search_actions': 5, 'jira_actions': 6, 'zendesk_actions': 4, 'pipedrive_leads_actions': 5, 'prompt_actions': 8, diff --git a/training_data/ReadMe.md b/training_data/ReadMe.md deleted file mode 100644 index f827d7c61..000000000 --- a/training_data/ReadMe.md +++ /dev/null @@ -1 +0,0 @@ -Trained Data Directory \ No newline at end of file From e3d52c144ed93028e7ac4816c520b89f31ea4e36 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Mon, 24 Jun 2024 19:00:46 +0530 Subject: [PATCH 30/57] 1. added missing test cases 2. removed stream from hyperparameters from prompt action --- kairon/actions/definitions/database.py | 2 +- kairon/actions/definitions/prompt.py | 1 - kairon/api/models.py | 10 +- kairon/shared/actions/utils.py | 3 +- kairon/shared/llm/base.py | 4 +- kairon/shared/llm/processor.py | 41 +- kairon/shared/rest_client.py | 2 +- kairon/shared/utils.py | 4 +- kairon/shared/vector_embeddings/db/base.py | 8 +- kairon/shared/vector_embeddings/db/qdrant.py | 18 +- kairon/train.py | 2 +- metadata/integrations.yml | 4 - tests/conftest.py | 1 - tests/integration_test/action_service_test.py | 35 +- tests/integration_test/services_test.py | 243 +++++++++-- tests/unit_test/action/action_test.py | 4 +- .../data_processor/data_processor_test.py | 44 +- tests/unit_test/llm_test.py | 390 ++++++++++++------ .../vector_embeddings/qdrant_test.py | 31 +- 19 files changed, 584 insertions(+), 263 deletions(-) diff --git a/kairon/actions/definitions/database.py b/kairon/actions/definitions/database.py index ebdf83510..0d54abd49 100644 --- a/kairon/actions/definitions/database.py +++ b/kairon/actions/definitions/database.py @@ -83,7 +83,7 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma request_body = ActionUtility.get_payload(payload, tracker) msg_logger.append(request_body) tracker_data = ActionUtility.build_context(tracker, True) - response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id, bot=self.bot) + response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id) logger.info("response: " + str(response)) response_context = self.__add_user_context_to_http_response(response, tracker_data) bot_response, bot_resp_log, _ = ActionUtility.compose_response(vector_action_config['response'], response_context) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 381e6f543..4c7bf6bc4 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -71,7 +71,6 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma llm_processor = LLMProcessor(self.bot) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, user=tracker.sender_id, - bot=self.bot, **llm_params) status = "FAILURE" if llm_response.get("is_failure", False) is True else status exception = llm_response.get("exception") diff --git a/kairon/api/models.py b/kairon/api/models.py index 53ce8d62c..d61a98126 100644 --- a/kairon/api/models.py +++ b/kairon/api/models.py @@ -15,8 +15,7 @@ ACTIVITY_STATUS, INTEGRATION_STATUS, FALLBACK_MESSAGE, - DEFAULT_NLU_FALLBACK_RESPONSE, - DEFAULT_LLM + DEFAULT_NLU_FALLBACK_RESPONSE ) from ..shared.actions.models import ( ActionParameterType, @@ -1059,8 +1058,8 @@ class PromptActionConfigRequest(BaseModel): num_bot_responses: int = 5 failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE user_question: UserQuestionModel = UserQuestionModel() - llm_type: str = DEFAULT_LLM - hyperparameters: dict = None + llm_type: str + hyperparameters: dict llm_prompts: List[LlmPromptRequest] instructions: List[str] = [] set_slots: List[SetSlotsUsingActionResponse] = [] @@ -1089,7 +1088,8 @@ def validate_llm_type(cls, v, values, **kwargs): @validator("hyperparameters") def validate_llm_hyperparameters(cls, v, values, **kwargs): - Utility.validate_llm_hyperparameters(v, kwargs['llm_type'], ValueError) + if values.get('llm_type'): + Utility.validate_llm_hyperparameters(v, values['llm_type'], ValueError) @root_validator def check(cls, values): diff --git a/kairon/shared/actions/utils.py b/kairon/shared/actions/utils.py index 00911f55c..d0c97d72f 100644 --- a/kairon/shared/actions/utils.py +++ b/kairon/shared/actions/utils.py @@ -5,6 +5,8 @@ import re from datetime import datetime from typing import Any, List, Text, Dict +from ..utils import Utility +Utility.load_system_metadata() import requests from aiohttp import ContentTypeError @@ -26,7 +28,6 @@ from ..data.data_objects import Slots, KeyVault from ..plugins.factory import PluginFactory from ..rest_client import AioRestClient -from ..utils import Utility from ...exceptions import AppException diff --git a/kairon/shared/llm/base.py b/kairon/shared/llm/base.py index f07eceda0..006e38a3d 100644 --- a/kairon/shared/llm/base.py +++ b/kairon/shared/llm/base.py @@ -8,9 +8,9 @@ def __init__(self, bot: Text): self.bot = bot @abstractmethod - async def train(self, user, bot, *args, **kwargs) -> Dict: + async def train(self, user, *args, **kwargs) -> Dict: pass @abstractmethod - async def predict(self, query, user, bot, *args, **kwargs) -> Dict: + async def predict(self, query, user, *args, **kwargs) -> Dict: pass diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index ffc48e2eb..c6e7fa8af 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -1,4 +1,4 @@ -import random +from secrets import randbelow, choice import time from typing import Text, Dict, List, Tuple from urllib.parse import urljoin @@ -39,7 +39,7 @@ def __init__(self, bot: Text): self.EMBEDDING_CTX_LENGTH = 8191 self.__logs = [] - async def train(self, user, bot, *args, **kwargs) -> Dict: + async def train(self, user, *args, **kwargs) -> Dict: await self.__delete_collections() count = 0 processor = CognitionDataProcessor() @@ -59,26 +59,25 @@ async def train(self, user, bot, *args, **kwargs) -> Dict: content['data'], metadata) else: search_payload, embedding_payload = {'content': content["data"]}, content["data"] - #search_payload['collection_name'] = collection - embeddings = await self.get_embedding(embedding_payload, user, bot) + embeddings = await self.get_embedding(embedding_payload, user) points = [{'id': content['vector_id'], 'vector': embeddings, 'payload': search_payload}] await self.__collection_upsert__(collection, {'points': points}, err_msg="Unable to train FAQ! Contact support") count += 1 return {"faq": count} - async def predict(self, query: Text, user, bot, *args, **kwargs) -> Tuple: + async def predict(self, query: Text, user, *args, **kwargs) -> Tuple: start_time = time.time() embeddings_created = False try: - query_embedding = await self.get_embedding(query, user, bot) + query_embedding = await self.get_embedding(query, user) embeddings_created = True system_prompt = kwargs.pop('system_prompt', DEFAULT_SYSTEM_PROMPT) context_prompt = kwargs.pop('context_prompt', DEFAULT_CONTEXT_PROMPT) context = await self.__attach_similarity_prompt_if_enabled(query_embedding, context_prompt, **kwargs) - answer = await self.__get_answer(query, system_prompt, context, user, bot, **kwargs) + answer = await self.__get_answer(query, system_prompt, context, user, **kwargs) response = {"content": answer} except Exception as e: logging.exception(e) @@ -100,11 +99,11 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def get_embedding(self, text: Text, user, bot) -> List[float]: + async def get_embedding(self, text: Text, user) -> List[float]: truncated_text = self.truncate_text(text) result = await litellm.aembedding(model="text-embedding-3-small", input=[truncated_text], - metadata={'user': user, 'bot': bot}, + metadata={'user': user, 'bot': self.bot}, api_key=self.api_key, num_retries=3) return result["data"][0]["embedding"] @@ -112,24 +111,25 @@ async def get_embedding(self, text: Text, user, bot) -> List[float]: async def __parse_completion_response(self, response, **kwargs): if kwargs.get("stream"): formatted_response = '' - msg_choice = random.randint(0, kwargs.get("n", 1) - 1) + msg_choice = randbelow(kwargs.get("n", 1)) if response["choices"][0].get("index") == msg_choice and response["choices"][0]['delta'].get('content'): formatted_response = f"{response['choices'][0]['delta']['content']}" else: - msg_choice = random.choice(response['choices']) + msg_choice = choice(response['choices']) formatted_response = msg_choice['message']['content'] return formatted_response - async def __get_completion(self, messages, hyperparameters, user, bot, **kwargs): + async def __get_completion(self, messages, hyperparameters, user, **kwargs): response = await litellm.acompletion(messages=messages, - metadata={'user': user, 'bot': bot}, + metadata={'user': user, 'bot': self.bot}, api_key=self.api_key, num_retries=3, **hyperparameters) - formatted_response = await self.__parse_completion_response(response, **kwargs) + formatted_response = await self.__parse_completion_response(response, + **hyperparameters) return formatted_response, response - async def __get_answer(self, query, system_prompt: Text, context: Text, user, bot, **kwargs): + async def __get_answer(self, query, system_prompt: Text, context: Text, user, **kwargs): use_query_prompt = False query_prompt = '' if kwargs.get('query_prompt', {}): @@ -144,8 +144,7 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, bo if use_query_prompt and query_prompt: query = await self.__rephrase_query(query, system_prompt, query_prompt, hyperparameters=hyperparameters, - user=user, - bot=bot) + user=user) messages = [ {"role": "system", "content": system_prompt}, ] @@ -156,13 +155,12 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, bo completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user, - bot=bot) + user=user) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'answer_query', 'hyperparameters': hyperparameters}) return completion - async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, bot, **kwargs): + async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, **kwargs): messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": f"{query_prompt}\n\n Q: {query}\n A:"} @@ -171,8 +169,7 @@ async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user, - bot=bot) + user=user) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'rephrase_query', 'hyperparameters': hyperparameters}) return completion diff --git a/kairon/shared/rest_client.py b/kairon/shared/rest_client.py index a76faad82..301e11323 100644 --- a/kairon/shared/rest_client.py +++ b/kairon/shared/rest_client.py @@ -60,7 +60,7 @@ async def request(self, request_method: str, http_url: str, request_body: Union[ headers: dict = None, return_json: bool = True, **kwargs): max_retries = kwargs.get("max_retries", 1) - status_forcelist = kwargs.get("status_forcelist", [104, 502, 503, 504]) + status_forcelist = set(kwargs.get("status_forcelist", [104, 502, 503, 504])) timeout = ClientTimeout(total=kwargs['timeout']) if kwargs.get('timeout') else None is_streaming_resp = kwargs.pop("is_streaming_resp", False) content_type = kwargs.pop("content_type", HttpRequestContentType.json.value) diff --git a/kairon/shared/utils.py b/kairon/shared/utils.py index 2db890379..acf3ff92a 100644 --- a/kairon/shared/utils.py +++ b/kairon/shared/utils.py @@ -2069,12 +2069,12 @@ def get_llm_hyperparameters(llm_type): @staticmethod def validate_llm_hyperparameters(hyperparameters: dict, llm_type: str, exception_class): - from jsonschema_rs import JSONSchema, ValidationError + from jsonschema_rs import JSONSchema, ValidationError as JValidationError schema = Utility.system_metadata["llm"][llm_type] try: validator = JSONSchema(schema) validator.validate(hyperparameters) - except ValidationError as e: + except JValidationError as e: message = f"{e.instance_path}: {e.message}" raise exception_class(message) diff --git a/kairon/shared/vector_embeddings/db/base.py b/kairon/shared/vector_embeddings/db/base.py index d1c2a1e97..887be41bb 100644 --- a/kairon/shared/vector_embeddings/db/base.py +++ b/kairon/shared/vector_embeddings/db/base.py @@ -8,16 +8,16 @@ class VectorEmbeddingsDbBase(ABC): @abstractmethod - async def embedding_search(self, request_body: Dict, **kwargs): + async def embedding_search(self, request_body: Dict, user: str, **kwargs): raise NotImplementedError("Provider not implemented") @abstractmethod - async def payload_search(self, request_body: Dict, **kwargs): + async def payload_search(self, request_body: Dict, user: str, **kwargs): raise NotImplementedError("Provider not implemented") - async def perform_operation(self, op_type: Text, request_body: Dict, **kwargs): + async def perform_operation(self, op_type: Text, request_body: Dict, user: str, **kwargs): supported_ops = {DbActionOperationType.payload_search.value: self.payload_search, DbActionOperationType.embedding_search.value: self.embedding_search} if op_type not in supported_ops.keys(): raise AppException("Operation type not supported") - return await supported_ops[op_type](request_body, **kwargs) + return await supported_ops[op_type](request_body, user, **kwargs) diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index 893a310ad..12b5268e2 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -29,23 +29,15 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 - def truncate_text(self, text: Text) -> Text: - """ - Truncate text to 8191 tokens for openai - """ - tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] - return self.tokenizer.decode(tokens) + async def __get_embedding(self, text: Text, user: str, **kwargs) -> List[float]: + return await self.llm.get_embedding(text, user=user) - async def __get_embedding(self, text: Text, **kwargs) -> List[float]: - result, _ = await self.llm.get_embedding(text, user=kwargs.get('user'), bot=kwargs.get('bot')) - return result - - async def embedding_search(self, request_body: Dict, **kwargs): + async def embedding_search(self, request_body: Dict, user: str, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points") if request_body.get("text"): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/search") user_msg = request_body.get("text") - vector = await self.__get_embedding(user_msg, **kwargs) + vector = await self.__get_embedding(user_msg, user, **kwargs) request_body = {'vector': vector, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} embedding_search_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', @@ -53,7 +45,7 @@ async def embedding_search(self, request_body: Dict, **kwargs): request_body=request_body) return embedding_search_result - async def payload_search(self, request_body: Dict, **kwargs): + async def payload_search(self, request_body: Dict, user, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/scroll") payload_filter_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', diff --git a/kairon/train.py b/kairon/train.py index 0276f7bc5..3ddcf9eb0 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -102,7 +102,7 @@ def start_training(bot: str, user: str, token: str = None): settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: llm_processor = LLMProcessor(bot) - faqs = asyncio.run(llm_processor.train(user=user, bot=bot)) + faqs = asyncio.run(llm_processor.train(user=user)) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) agent_url = Utility.environment['model']['agent'].get('url') diff --git a/metadata/integrations.yml b/metadata/integrations.yml index 227b4c413..6ba78b15f 100644 --- a/metadata/integrations.yml +++ b/metadata/integrations.yml @@ -129,10 +129,6 @@ llm: minimum: 1 maximum: 5 description: "The n hyperparameter controls the number of different response options that are generated by the model." - stream: - type: boolean - default: false - description: "the stream hyperparameter controls whether the model generates a continuous stream of text or a single response." stop: anyOf: - type: "string" diff --git a/tests/conftest.py b/tests/conftest.py index c613d74a5..10bd6434c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ from kairon.shared.concurrency.actors.factory import ActorFactory import pytest -from mock import patch import os diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index 8d0219d28..13dc8d7e7 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -11344,7 +11344,7 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11411,7 +11411,7 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11481,7 +11481,7 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11706,7 +11706,7 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion.return_value = {'choices': [{'delta': {'role': 'assistant', 'content': generated_text}, 'finish_reason': None, 'index': 0}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11722,6 +11722,7 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m response = client.post("/webhook", json=request_object) response_json = response.json() + print(response_json['events']) assert response_json['events'] == [ {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': generated_text}] assert response_json['responses'] == [ @@ -12008,14 +12009,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12092,7 +12093,7 @@ def mock_completion_for_answer(*args, **kwargs): 'content': 'Kanban is a workflow management tool which visualizes both the process (the workflow) and the actual work passing through that process.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) @@ -12100,7 +12101,7 @@ def mock_completion_for_answer(*args, **kwargs): 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12217,14 +12218,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12324,14 +12325,14 @@ def mock_completion_for_answer(*args, **kwargs): 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12422,14 +12423,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12493,7 +12494,7 @@ def __mock_fetch_similar(*args, **kwargs): 'content': "\nInstructions on how to use Similarity Prompt:\n['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Kanban And Scrum Together? \nA:"}], 'metadata': {'user': user, 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12557,7 +12558,7 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], 'metadata': {'user': 'udit.pandey', 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12624,5 +12625,5 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], 'metadata': {'user': user, 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 7339b2e54..7f944d109 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -47,6 +47,7 @@ KAIRON_TWO_STAGE_FALLBACK, FeatureMappings, DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_LLM ) from kairon.shared.data.data_objects import ( Stories, @@ -1610,7 +1611,6 @@ def test_get_live_agent_with_no_live_agent(): def test_enable_live_agent(): - bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.live_agent_enabled = True bot_settings.save() @@ -1633,7 +1633,6 @@ def test_enable_live_agent(): assert actual["success"] - def test_get_live_agent_after_enabled_no_bot_settings_enabled(): bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.live_agent_enabled = False @@ -1650,6 +1649,7 @@ def test_get_live_agent_after_enabled_no_bot_settings_enabled(): assert not actual["message"] assert actual["success"] + def test_get_live_agent_after_enabled(): bot_settings = BotSettings.objects(bot=pytest.bot).get() bot_settings.live_agent_enabled = True @@ -2386,7 +2386,9 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters()} response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -3123,6 +3125,8 @@ def _mock_get_bot_settings(*args, **kwargs): ], "num_bot_responses": 5, "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3180,6 +3184,8 @@ def _mock_get_bot_settings(*args, **kwargs): ], "num_bot_responses": 5, "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3224,6 +3230,8 @@ def test_add_prompt_action_with_invalid_query_prompt(): ], "num_bot_responses": 5, "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3281,6 +3289,8 @@ def test_add_prompt_action_with_invalid_num_bot_responses(): ], "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "num_bot_responses": 10, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3337,7 +3347,9 @@ def test_add_prompt_action_with_invalid_system_prompt_source(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3402,7 +3414,9 @@ def test_add_prompt_action_with_multiple_system_prompt(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3459,7 +3473,9 @@ def test_add_prompt_action_with_empty_llm_prompt_name(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3516,7 +3532,9 @@ def test_add_prompt_action_with_empty_data_for_static_prompt(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3577,7 +3595,9 @@ def test_add_prompt_action_with_multiple_history_source_prompts(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3637,6 +3657,8 @@ def test_add_prompt_action_with_gpt_feature_disabled(): "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, "top_results": 10, "similarity_threshold": 0.70, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3653,6 +3675,138 @@ def test_add_prompt_action_with_gpt_feature_disabled(): assert actual["error_code"] == 422 +def test_add_prompt_action_with_invalid_llm_type(monkeypatch): + def _mock_get_bot_settings(*args, **kwargs): + return BotSettings( + bot=pytest.bot, + user="integration@demo.ai", + llm_settings=LLMSettings(enable_faq=True), + ) + + monkeypatch.setattr(MongoProcessor, "get_bot_settings", _mock_get_bot_settings) + action = { + "name": "test_add_prompt_action_with_invalid_llm_type", 'user_question': {'type': 'from_user_message'}, + "llm_prompts": [ + { + "name": "System Prompt", + "data": "You are a personal assistant.", + "type": "system", + "source": "static", + "is_enabled": True, + }, + { + "name": "Similarity Prompt", + "data": "Bot_collection", + "instructions": "Answer question based on the context above, if answer is not in the context go check previous logs.", + "type": "user", + "source": "bot_content", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "If there is no specific query, assume that user is aking about java programming.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + ], + "instructions": ["Answer in a short manner.", "Keep it simple."], + "num_bot_responses": 5, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": "test", + "hyperparameters": Utility.get_default_llm_hyperparameters() + } + response = client.post( + f"/api/bot/{pytest.bot}/action/prompt", + json=action, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + assert not DeepDiff(actual["message"], + [{'loc': ['body', 'llm_type'], 'msg': 'Invalid llm type', 'type': 'value_error'}], + ignore_order=True) + assert not actual["success"] + assert not actual["data"] + assert actual["error_code"] == 422 + + +def test_add_prompt_action_with_invalid_hyperameters(monkeypatch): + temp = Utility.get_default_llm_hyperparameters() + temp['temperature'] = 3.0 + + def _mock_get_bot_settings(*args, **kwargs): + return BotSettings( + bot=pytest.bot, + user="integration@demo.ai", + llm_settings=LLMSettings(enable_faq=True), + ) + + monkeypatch.setattr(MongoProcessor, "get_bot_settings", _mock_get_bot_settings) + action = { + "name": "test_add_prompt_action_with_invalid_hyperameters", 'user_question': {'type': 'from_user_message'}, + "llm_prompts": [ + { + "name": "System Prompt", + "data": "You are a personal assistant.", + "type": "system", + "source": "static", + "is_enabled": True, + }, + { + "name": "Similarity Prompt", + "data": "Bot_collection", + "instructions": "Answer question based on the context above, if answer is not in the context go check previous logs.", + "type": "user", + "source": "bot_content", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + { + "name": "Query Prompt", + "data": "If there is no specific query, assume that user is aking about java programming.", + "instructions": "Answer according to the context", + "type": "query", + "source": "static", + "is_enabled": True, + }, + ], + "instructions": ["Answer in a short manner.", "Keep it simple."], + "num_bot_responses": 5, + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": temp + } + response = client.post( + f"/api/bot/{pytest.bot}/action/prompt", + json=action, + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + assert not DeepDiff(actual["message"], + [{'loc': ['body', 'hyperparameters'], + 'msg': "['temperature']: 3.0 is greater than the maximum of 2.0", 'type': 'value_error'}], + ignore_order=True) + assert not actual["success"] + assert not actual["data"] + assert actual["error_code"] == 422 + + def test_add_prompt_action(monkeypatch): def _mock_get_bot_settings(*args, **kwargs): return BotSettings( @@ -3699,7 +3853,9 @@ def _mock_get_bot_settings(*args, **kwargs): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3760,7 +3916,9 @@ def _mock_get_bot_settings(*args, **kwargs): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -3811,7 +3969,9 @@ def test_update_prompt_action_does_not_exist(): }, ], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/61512cc2c6219f0aae7bba3d", @@ -3863,7 +4023,9 @@ def test_update_prompt_action_with_invalid_similarity_threshold(): }, ], "num_bot_responses": 5, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -3908,7 +4070,9 @@ def test_update_prompt_action_with_invalid_top_results(): }, ], "num_bot_responses": 5, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -3952,7 +4116,9 @@ def test_update_prompt_action_with_invalid_num_bot_responses(): }, ], "num_bot_responses": 50, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4001,6 +4167,8 @@ def test_update_prompt_action_with_invalid_query_prompt(): "num_bot_responses": 5, "use_query_prompt": True, "query_prompt": "", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4058,6 +4226,8 @@ def test_update_prompt_action_with_query_prompt_with_false(): }, ], "dispatch_response": False, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4114,7 +4284,9 @@ def test_update_prompt_action(): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": "updated_failure_message" + "failure_message": "updated_failure_message", + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.put( f"/api/bot/{pytest.bot}/action/prompt/{pytest.action_id}", @@ -4144,7 +4316,7 @@ def test_get_prompt_action(): 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, {'name': 'Similarity_analytical Prompt', 'data': 'Bot_collection', @@ -4208,7 +4380,9 @@ def _mock_get_bot_settings(*args, **kwargs): ], "instructions": ["Answer in a short manner.", "Keep it simple."], "num_bot_responses": 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", @@ -4237,7 +4411,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'user_question': {'type': 'from_user_message'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -4306,7 +4480,10 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() + } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -4326,7 +4503,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'user_question': {'type': 'from_user_message'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -4398,7 +4575,10 @@ def _mock_get_bot_settings(*args, **kwargs): 'source': 'static', 'is_enabled': True}], 'instructions': ['Answer in a short manner.', 'Keep it simple.'], 'num_bot_responses': 5, - "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE} + "failure_message": DEFAULT_NLU_FALLBACK_RESPONSE, + "llm_type": DEFAULT_LLM, + "hyperparameters": Utility.get_default_llm_hyperparameters() + } response = client.post( f"/api/bot/{pytest.bot}/action/prompt", json=action, @@ -4418,7 +4598,7 @@ def _mock_get_bot_settings(*args, **kwargs): 'user_question': {'type': 'from_user_message'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -5073,7 +5253,8 @@ def test_get_data_importer_logs(): assert actual['data']["logs"][3]['event_status'] == EVENT_STATUS.COMPLETED.value assert actual['data']["logs"][3]['status'] == 'Failure' assert set(actual['data']["logs"][3]['files_received']) == {'rules', 'stories', 'nlu', 'domain', 'config', - 'actions', 'chat_client_config', 'multiflow_stories', 'bot_content'} + 'actions', 'chat_client_config', 'multiflow_stories', + 'bot_content'} assert actual['data']["logs"][3]['is_data_uploaded'] assert actual['data']["logs"][3]['start_timestamp'] assert actual['data']["logs"][3]['end_timestamp'] @@ -5105,7 +5286,8 @@ def test_get_data_importer_logs(): ] assert actual['data']["logs"][3]['is_data_uploaded'] assert set(actual['data']["logs"][3]['files_received']) == {'rules', 'stories', 'nlu', 'config', 'domain', - 'actions', 'chat_client_config', 'multiflow_stories','bot_content'} + 'actions', 'chat_client_config', 'multiflow_stories', + 'bot_content'} @responses.activate @@ -6234,6 +6416,7 @@ def test_add_story_lone_intent(): } ] + def test_add_story_consecutive_intents(): response = client.post( f"/api/bot/{pytest.bot}/stories", @@ -9887,6 +10070,7 @@ def test_login_for_verified(): pytest.access_token = actual["data"]["access_token"] pytest.token_type = actual["data"]["token_type"] + def test_list_bots_for_different_user(): response = client.get( "/api/account/bot", @@ -19005,7 +19189,7 @@ def test_set_templates_with_sysadmin_as_user(): intents = Intents.objects(bot=pytest.bot) intents = [{k: v for k, v in intent.to_mongo().to_dict().items() if k not in ['_id', 'bot', 'timestamp']} for - intent in intents] + intent in intents] assert intents == [ {'name': 'greet', 'user': 'sysadmin', 'status': True, 'is_integration': False, 'use_entities': False}, @@ -19081,7 +19265,6 @@ def test_add_channel_config(monkeypatch): def test_add_bot_with_template_with_sysadmin_as_user(monkeypatch): - def mock_reload_model(*args, **kwargs): mock_reload_model.called_with = (args, kwargs) return None @@ -19120,7 +19303,7 @@ def mock_reload_model(*args, **kwargs): rules = Rules.objects(bot=bot_id) rules = [{k: v for k, v in rule.to_mongo().to_dict().items() if k not in ['_id', 'bot', 'timestamp']} for - rule in rules] + rule in rules] assert rules == [ {'block_name': 'ask the user to rephrase whenever they send a message with low nlu confidence', @@ -19134,7 +19317,7 @@ def mock_reload_model(*args, **kwargs): utterances = Utterances.objects(bot=bot_id) utterances = [{k: v for k, v in utterance.to_mongo().to_dict().items() if k not in ['_id', 'bot', 'timestamp']} for - utterance in utterances] + utterance in utterances] assert utterances == [ {'name': 'utter_please_rephrase', 'user': 'sysadmin', 'status': True}, @@ -20249,7 +20432,7 @@ def test_get_bot_settings(): 'whatsapp': 'meta', 'cognition_collections_limit': 3, 'cognition_columns_per_collection_limit': 5, - 'integrations_per_user_limit':3 } + 'integrations_per_user_limit': 3} def test_update_analytics_settings_with_empty_value(): @@ -20327,7 +20510,7 @@ def test_update_analytics_settings(): 'live_agent_enabled': False, 'cognition_collections_limit': 3, 'cognition_columns_per_collection_limit': 5, - 'integrations_per_user_limit':3 } + 'integrations_per_user_limit': 3} def test_delete_channels_config(): diff --git a/tests/unit_test/action/action_test.py b/tests/unit_test/action/action_test.py index c10994445..6b361ad44 100644 --- a/tests/unit_test/action/action_test.py +++ b/tests/unit_test/action/action_test.py @@ -2662,7 +2662,7 @@ def test_get_prompt_action_config(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'bot': 'test_action_server', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', @@ -3954,7 +3954,7 @@ def test_get_prompt_action_config_2(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'bot': 'test_bot_action_test', 'user': 'test_user_action_test', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'dispatch_response': True, 'set_slots': [], 'llm_type': 'openai', diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 896ee9de4..b42dc6b06 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -217,7 +217,7 @@ def test_add_prompt_action_with_invalid_slots(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', @@ -245,7 +245,7 @@ def test_add_prompt_action_with_invalid_http_action(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', @@ -274,7 +274,7 @@ def test_add_prompt_action_with_invalid_similarity_threshold(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -304,7 +304,7 @@ def test_add_prompt_action_with_invalid_top_results(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -356,7 +356,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', @@ -401,7 +401,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', @@ -601,7 +601,7 @@ def test_add_prompt_action_with_empty_llm_prompts(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': []} with pytest.raises(ValidationError, match="llm_prompts are required!"): @@ -628,7 +628,7 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -649,7 +649,7 @@ def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -666,7 +666,7 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': ["\n", ".", "?", "!", ";"], + 'n': 1, 'stop': ["\n", ".", "?", "!", ";"], 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -686,7 +686,7 @@ def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': -3.0, + 'n': 1, 'stop': '?', 'presence_penalty': -3.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -704,7 +704,7 @@ def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 3.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -721,7 +721,7 @@ def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -738,7 +738,7 @@ def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -755,7 +755,7 @@ def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt-3.5-turbo', 'top_p': 3.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -772,7 +772,7 @@ def test_add_prompt_action_with_invalid_n_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 7, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 7, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -789,7 +789,7 @@ def test_add_prompt_action_with_zero_n_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 0, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 0, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -806,7 +806,7 @@ def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 2, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 2, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': 'a'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -881,7 +881,7 @@ def test_edit_prompt_action_faq_action(self): assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -916,7 +916,7 @@ def test_edit_prompt_action_faq_action(self): 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -959,7 +959,7 @@ def test_edit_prompt_action_with_less_hyperparameters(self): 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -994,7 +994,7 @@ def test_get_prompt_faq_action(self): 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index c8f727db3..7738dfeb0 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -8,6 +8,7 @@ from aiohttp import ClientConnectionError from mongoengine import connect from kairon.shared.utils import Utility + Utility.load_system_metadata() from kairon.exceptions import AppException @@ -66,7 +67,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, @@ -119,14 +120,16 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() embedding = list(np.random.random(LLMProcessor.__embedding__)) - mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]} + mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, { + 'data': [{'embedding': embedding}]} gpt3 = LLMProcessor(bot) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), method="GET", - payload={"time": 0, "status": "ok", "result": {"collections": [{"name": "test_embed_faq_text_swift_faq_embd"}, - {"name": "example_bot_swift_faq_embd"}]}} + payload={"time": 0, "status": "ok", + "result": {"collections": [{"name": "test_embed_faq_text_swift_faq_embd"}, + {"name": "example_bot_swift_faq_embd"}]}} ) aioresponses.add( @@ -141,19 +144,22 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_user_details{gpt3.suffix}/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_user_details{gpt3.suffix}/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_country_details{gpt3.suffix}"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_country_details{gpt3.suffix}"), method="PUT", status=200 ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_country_details{gpt3.suffix}/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_country_details{gpt3.suffix}/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) @@ -165,24 +171,29 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 3 - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'name': f"{gpt3.bot}_country_details{gpt3.suffix}", - 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'name': f"{gpt3.bot}_country_details{gpt3.suffix}", + 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content_two.vector_id, - 'vector': embedding, - 'payload': {'country': 'Spain'}}]} - assert list(aioresponses.requests.values())[3][1].kwargs['json'] == {'points': [{'id': test_content_three.vector_id, - 'vector': embedding, - 'payload': {'role': 'ds'}}]} + assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + 'points': [{'id': test_content_two.vector_id, + 'vector': embedding, + 'payload': {'country': 'Spain'}}]} + assert list(aioresponses.requests.values())[3][1].kwargs['json'] == { + 'points': [{'id': test_content_three.vector_id, + 'vector': embedding, + 'payload': {'role': 'ds'}}]} - assert list(aioresponses.requests.values())[4][0].kwargs['json'] == {'name': f"{gpt3.bot}_user_details{gpt3.suffix}", - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[5][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, - 'payload': {'name': 'Nupur'}}]} + assert list(aioresponses.requests.values())[4][0].kwargs['json'] == { + 'name': f"{gpt3.bot}_user_details{gpt3.suffix}", + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[5][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, + 'payload': {'name': 'Nupur'}}]} assert response['faq'] == 3 expected = {"model": "text-embedding-3-small", @@ -217,7 +228,8 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a gpt3 = LLMProcessor(bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), method="PUT", status=200 ) @@ -227,18 +239,21 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a payload={"time": 0, "status": "ok", "result": {"collections": []}}) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/test_embed_faq_json_payload_with_int_faq_embd/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 1 - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_embed_faq_json_payload_with_int_faq_embd', - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == { + 'name': 'test_embed_faq_json_payload_with_int_faq_embd', + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, 'vector': embedding, 'payload': {'name': 'Ram', 'age': 23, 'color': 'red'} }]} @@ -302,7 +317,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_int_embd_int_faq_embd', @@ -363,16 +378,18 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aiore url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}{gpt3.suffix}/points"), method="PUT", payload={"result": None, - 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, - "time": 0.003612634} + 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, + "time": 0.003612634} ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train(user=user, bot=bot) + await gpt3.train(user=user) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, 'payload': {'content': test_content.data}}]} + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, 'payload': {'content': test_content.data}}]} expected = {"model": "text-embedding-3-small", "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, @@ -412,33 +429,38 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb aioresponses.add( method="DELETE", - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd"), ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd"), method="PUT", status=200 ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd/points"), method="PUT", payload={"result": None, - 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, - "time": 0.003612634} + 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, + "time": 0.003612634} ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train(user=user, bot=bot) + await gpt3.train(user=user) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == { + 'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} expected_payload = test_content.data #expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, - 'payload': expected_payload - }]} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, + 'payload': expected_payload + }]} expected = {"model": "text-embedding-3-small", "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, @@ -449,7 +471,7 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict" @@ -469,9 +491,9 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], "hyperparameters": hyperparameters } mock_completion_request = {"messages": [ @@ -487,13 +509,14 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, @@ -514,7 +537,8 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict_with_default_collection" @@ -559,7 +583,7 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_emb {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, @@ -599,11 +623,11 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], "hyperparameters": hyperparameters - } + } mock_completion_request = {"messages": [ {"role": "system", @@ -615,17 +639,18 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + with mock.patch.dict(Utility.environment, {'vector': {"key": "test", 'db': "http://localhost:6333"}}): gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=gpt3.bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert gpt3.logs == [ {'messages': [{'role': 'system', @@ -638,10 +663,12 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 @@ -659,25 +686,133 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) - user = "test" - bot = "test_gpt3_faq_embedding_predict_with_values_with_instructions" - key = 'test' + test_content = CognitionData( - data="Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ", - collection='java', bot=bot, user=user).save() - BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() + data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", + collection='python', bot="test_gpt3_faq_embedding_predict_with_values_and_stream", user="test").save() + generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" hyperparameters = Utility.get_default_llm_hyperparameters() + hyperparameters['stream'] = True + key = 'test' + user = "tests" + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": "java"}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], + "hyperparameters": hyperparameters + } + + mock_completion_request = {"messages": [ + {"role": "system", + "content": "You are a personal assistant. Answer the question according to the below context"}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} + ]} + mock_completion_request.update(hyperparameters) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.side_effect = [{'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[29:60]}, + 'finish_reason': None, 'index': 0}]}, + {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[60:]}, + 'finish_reason': 'stop', 'index': 0}]} + ] + + with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + gpt3 = LLMProcessor(test_content.bot) + + aioresponses.add( + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + method="POST", + payload={'result': [ + {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} + ) + + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + assert response['content'] == "Python is dynamically typed, " + assert gpt3.logs == [ + {'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': True, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} + + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': gpt3.bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + @pytest.mark.asyncio + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, + mock_embedding, + mock_completion, + aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "payload_with_instruction" + key = 'test' + CognitionSchema( + metadata=[{"column_name": "name", "data_type": "str", "enable_search": True, "create_embeddings": True}, + {"column_name": "city", "data_type": "str", "enable_search": True, "create_embeddings": True}], + collection_name="User_details", + bot=bot, user=user + ).save() + test_content1 = CognitionData( + data={"name": "Nupur", "city": "Pune"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + test_content2 = CognitionData( + data={"name": "Fahad", "city": "Mumbai"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + test_content3 = CognitionData( + data={"name": "Hitesh", "city": "Mumbai"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=bot, user=user).save() + + generated_text = "Hitesh and Fahad lives in mumbai city." + query = "List all the user lives in mumbai city" + hyperparameters = Utility.get_default_llm_hyperparameters() + k_faq_action_config = { + "system_prompt": "You are a personal assistant. Answer the question according to the below context", + "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", + "similarity_prompt": [{"top_results": 10, + "similarity_threshold": 0.70, + 'use_similarity_prompt': True, + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": "user_details"}], 'instructions': ['Answer in a short way.', 'Keep it simple.'], "hyperparameters": hyperparameters } @@ -686,39 +821,39 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mo {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"} + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n[{'name': 'Fahad', 'city': 'Mumbai'}, {'name': 'Hitesh', 'city': 'Mumbai'}]\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: List all the user lives in mumbai city \nA:"} ]} mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot) - + gpt3 = LLMProcessor(bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content1.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ - {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} + {'id': test_content2.vector_id, 'score': 0.80, "payload": test_content2.data}, + {'id': test_content3.vector_id, 'score': 0.80, "payload": test_content3.data} + ]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text - assert gpt3.logs == [ - {'messages': [{'role': 'system', - 'content': 'You are a personal assistant. Answer the question according to the below context'}, - {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': { - 'choices': [{'message': {'content': 'Python is dynamically typed, garbage-collected, ' - 'high level, general purpose programming.', - 'role': 'assistant'}}]}, - 'type': 'answer_query', - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}}] - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert gpt3.logs == [{'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n[{'name': 'Fahad', 'city': 'Mumbai'}, {'name': 'Hitesh', 'city': 'Mumbai'}]\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: List all the user lives in mumbai city \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Hitesh and Fahad lives in mumbai city.', 'role': 'assistant'}}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', + 'top_p': 0.0, 'n': 1, 'stop': None, + 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 @@ -736,7 +871,8 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mo @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_gpt3_faq_embedding_predict_completion_connection_error" user = 'test' @@ -755,11 +891,11 @@ async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters - } + } def __mock_connection_error(*args, **kwargs): raise Exception("Connection reset by peer!") @@ -770,18 +906,21 @@ def __mock_connection_error(*args, **kwargs): gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", @@ -795,7 +934,7 @@ def __mock_connection_error(*args, **kwargs): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error'}, 'api_key': 'test', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, - 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -804,7 +943,7 @@ def __mock_connection_error(*args, **kwargs): @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock_llm_request): embedding = list(np.random.random(LLMProcessor.__embedding__)) - user ="test" + user = "test" bot = "test_gpt3_faq_embedding_predict_exact_match" key = 'test' test_content = CognitionData( @@ -818,9 +957,9 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters } @@ -829,16 +968,17 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock gpt3 = LLMProcessor(test_content.bot) - response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_exact_match", **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} - assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] + assert gpt3.logs == [ + {'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, - "api_key": key, - "num_retries": 3} + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio @@ -867,7 +1007,7 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ gpt3 = LLMProcessor(test_content.bot) mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] - response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_embedding_connection_error", **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] @@ -882,7 +1022,8 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_gpt3_faq_embedding_predict_with_previous_bot_responses" @@ -921,13 +1062,14 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, @@ -970,11 +1112,11 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding "query_prompt": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", "use_query_prompt": True}, "similarity_prompt": [ - {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters - } + } mock_rephrase_request = {"messages": [ {"role": "system", @@ -993,18 +1135,20 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, { + 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, diff --git a/tests/unit_test/vector_embeddings/qdrant_test.py b/tests/unit_test/vector_embeddings/qdrant_test.py index 7bf166116..667285715 100644 --- a/tests/unit_test/vector_embeddings/qdrant_test.py +++ b/tests/unit_test/vector_embeddings/qdrant_test.py @@ -14,7 +14,9 @@ from kairon.shared.data.data_objects import LLMSettings from kairon.shared.vector_embeddings.db.factory import VectorEmbeddingsDbFactory from kairon.shared.vector_embeddings.db.qdrant import Qdrant - +import litellm +from kairon.shared.llm.processor import LLMProcessor +import numpy as np class TestQdrant: @@ -25,16 +27,21 @@ def init_connection(self): connect(**Utility.mongoengine_connection(Utility.environment['database']["url"])) @pytest.mark.asyncio + @mock.patch.dict(Utility.environment, {'vector': {"key": "TEST", 'db': 'http://localhost:6333'}}) + @mock.patch.object(litellm, "aembedding", autospec=True) @mock.patch.object(ActionUtility, "execute_http_request", autospec=True) - async def test_embedding_search_valid_request_body(self, mock_http_request): + async def test_embedding_search_valid_request_body(self, mock_http_request, mock_embedding): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" Utility.load_environment() secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value="key_value", bot="5f50fd0a56v098ca10d75d2g", user="user").save() qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) - request_body = {"ids": [0], "with_payload": True, "with_vector": True} + request_body = {"ids": [0], "with_payload": True, "with_vector": True, 'text': "Hi"} mock_http_request.return_value = 'expected_result' - result = await qdrant.embedding_search(request_body) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + result = await qdrant.embedding_search(request_body, user=user) assert result == 'expected_result' @pytest.mark.asyncio @@ -46,29 +53,31 @@ async def test_payload_search_valid_request_body(self, mock_http_request): request_body = {"filter": {"should": [{"key": "city", "match": {"value": "London"}}, {"key": "color", "match": {"value": "red"}}]}} mock_http_request.return_value = 'expected_result' - result = await qdrant.payload_search(request_body) + result = await qdrant.payload_search(request_body, user="test") assert result == 'expected_result' @pytest.mark.asyncio @mock.patch.object(ActionUtility, "execute_http_request", autospec=True) async def test_perform_operation_valid_op_type_and_request_body(self, mock_http_request): Utility.load_environment() + user = "test" qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) request_body = {} mock_http_request.return_value = 'expected_result' - result_embedding = await qdrant.perform_operation('embedding_search', request_body) + result_embedding = await qdrant.perform_operation('embedding_search', request_body, user=user) assert result_embedding == 'expected_result' - result_payload = await qdrant.perform_operation('payload_search', request_body) + result_payload = await qdrant.perform_operation('payload_search', request_body, user=user) assert result_payload == 'expected_result' @pytest.mark.asyncio async def test_embedding_search_empty_request_body(self): Utility.load_environment() + user = "test" qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) with pytest.raises(ActionFailure): - await qdrant.embedding_search({}) + await qdrant.embedding_search({}, user=user) @pytest.mark.asyncio async def test_payload_search_empty_request_body(self): @@ -76,7 +85,7 @@ async def test_payload_search_empty_request_body(self): qdrant = Qdrant('5f50fd0a56v098ca10d75d2g', '5f50fd0a56v098ca10d75d2g', LLMSettings(provider="openai").to_mongo().to_dict()) with pytest.raises(ActionFailure): - await qdrant.payload_search({}) + await qdrant.payload_search({}, user="test") @pytest.mark.asyncio async def test_perform_operation_invalid_op_type(self): @@ -85,7 +94,7 @@ async def test_perform_operation_invalid_op_type(self): LLMSettings(provider="openai").to_mongo().to_dict()) request_body = {} with pytest.raises(AppException, match="Operation type not supported"): - await qdrant.perform_operation("vector_search", request_body) + await qdrant.perform_operation("vector_search", request_body, user="test") def test_get_instance_raises_exception_when_db_not_implemented(self): with pytest.raises(AppException, match="Database not yet implemented!"): @@ -99,7 +108,7 @@ async def test_embedding_search_valid_request_body_payload(self, mock_http_reque LLMSettings(provider="openai").to_mongo().to_dict()) request_body = {'ids': [0], 'with_payload': True, 'with_vector': True} mock_http_request.return_value = 'expected_result' - result = await qdrant.embedding_search(request_body) + result = await qdrant.embedding_search(request_body, user="test") assert result == 'expected_result' mock_http_request.assert_called_once() From 2b4a3bbddbfa130d6c8e8c48fc3ae18243c12118 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 10:40:04 +0530 Subject: [PATCH 31/57] test cased fixed --- tests/unit_test/data_processor/data_processor_test.py | 6 ++++-- tests/unit_test/utility_test.py | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index b42dc6b06..0f0c563ad 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -60,7 +60,7 @@ from kairon.shared.data.constant import UTTERANCE_TYPE, EVENT_STATUS, STORY_EVENT, ALLOWED_DOMAIN_FORMATS, \ ALLOWED_CONFIG_FORMATS, ALLOWED_NLU_FORMATS, ALLOWED_STORIES_FORMATS, ALLOWED_RULES_FORMATS, REQUIREMENTS, \ DEFAULT_NLU_FALLBACK_RULE, SLOT_TYPE, KAIRON_TWO_STAGE_FALLBACK, AuditlogActions, TOKEN_TYPE, GPT_LLM_FAQ, \ - DEFAULT_CONTEXT_PROMPT, DEFAULT_NLU_FALLBACK_RESPONSE, DEFAULT_SYSTEM_PROMPT + DEFAULT_CONTEXT_PROMPT, DEFAULT_NLU_FALLBACK_RESPONSE, DEFAULT_SYSTEM_PROMPT, DEFAULT_LLM from kairon.shared.data.data_objects import (TrainingExamples, Slots, Entities, EntitySynonyms, RegexFeatures, @@ -8552,7 +8552,9 @@ def test_delete_action_with_attached_http_action(self): 'data': 'tester_action', 'instructions': 'Answer according to the context', 'type': 'user', 'source': 'action', - 'is_enabled': True}] + 'is_enabled': True}], + llm_type=DEFAULT_LLM, + hyperparameters=Utility.get_default_llm_hyperparameters() ) processor.add_http_action_config(http_action_config.dict(), user, bot) processor.add_prompt_action(prompt_action_config.dict(), bot, user) diff --git a/tests/unit_test/utility_test.py b/tests/unit_test/utility_test.py index a0de42a47..5fa44f128 100644 --- a/tests/unit_test/utility_test.py +++ b/tests/unit_test/utility_test.py @@ -2961,7 +2961,6 @@ def test_get_llm_hyperparameters(self): "model": "gpt-3.5-turbo", "top_p": 0.0, "n": 1, - "stream": False, "stop": None, "presence_penalty": 0.0, "frequency_penalty": 0.0, From f5e1e945d004e6995d6a6185cbbe35e67dc13d97 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 15:48:54 +0530 Subject: [PATCH 32/57] 1. added missing test case 2. updated litellm --- kairon/shared/llm/logger.py | 27 ++++++++++++--------------- requirements/prod.txt | 2 +- tests/unit_test/llm_test.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py index 06b720b48..dbb93e469 100644 --- a/kairon/shared/llm/logger.py +++ b/kairon/shared/llm/logger.py @@ -1,14 +1,10 @@ from litellm.integrations.custom_logger import CustomLogger from .data_objects import LLMLogs import ujson as json +from loguru import logger class LiteLLMLogger(CustomLogger): - def log_pre_api_call(self, model, messages, kwargs): - pass - - def log_post_api_call(self, kwargs, response_obj, start_time, end_time): - pass def log_stream_event(self, kwargs, response_obj, start_time, end_time): self.__logs_litellm(**kwargs) @@ -29,15 +25,16 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti self.__logs_litellm(**kwargs) def __logs_litellm(self, **kwargs): - litellm_params = kwargs['litellm_params'] - self.__save_logs(**{'response': json.loads(kwargs['original_response']), - 'start_time': kwargs['start_time'], - 'end_time': kwargs['end_time'], - 'cost': kwargs["response_cost"], - 'llm_call_id': litellm_params['litellm_call_id'], - 'llm_provider': litellm_params['custom_llm_provider'], - 'model_params': kwargs["additional_args"]["complete_input_dict"], - 'metadata': litellm_params['metadata']}) + logger.info("logging llms call") + litellm_params = kwargs.get('litellm_params') + self.__save_logs(**{'response': json.loads(kwargs.get('original_response')) if kwargs.get('original_response') else None, + 'start_time': kwargs.get('start_time'), + 'end_time': kwargs.get('end_time'), + 'cost': kwargs.get("response_cost"), + 'llm_call_id': litellm_params.get('litellm_call_id'), + 'llm_provider': litellm_params.get('custom_llm_provider'), + 'model_params': kwargs.get("additional_args", {}).get("complete_input_dict"), + 'metadata': litellm_params.get('metadata')}) def __save_logs(self, **kwargs): - LLMLogs(**kwargs).save() + print(LLMLogs(**kwargs).save().to_mongo().to_dict()) diff --git a/requirements/prod.txt b/requirements/prod.txt index e00d448a9..13fcc40e7 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -64,6 +64,6 @@ opentelemetry-instrumentation-requests==0.46b0 opentelemetry-instrumentation-sklearn==0.46b0 pykwalify==1.8.0 gunicorn==22.0.0 -litellm==1.38.11 +litellm==1.39.5 jsonschema_rs==0.18.0 mongoengine-jsonschema==0.1.3 \ No newline at end of file diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 7738dfeb0..bb446e802 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1,4 +1,5 @@ import os +import time from urllib.parse import urljoin import mock @@ -17,6 +18,7 @@ from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT from kairon.shared.llm.processor import LLMProcessor +from kairon.shared.llm.data_objects import LLMLogs import litellm from deepdiff import DeepDiff @@ -1166,3 +1168,33 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + @pytest.mark.asyncio + async def test_llm_logging(self): + from kairon.shared.llm.logger import LiteLLMLogger + bot = "test_llm_logging" + user = "test" + litellm.callbacks = [LiteLLMLogger()] + + result = await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + metadata={'user': user, 'bot': bot}) + assert result + + result = litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + metadata={'user': user, 'bot': bot}) + assert result + + result = litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + stream=True, + metadata={'user': user, 'bot': bot}) + for chunk in result: + print(chunk["choices"][0]["delta"]["content"]) + assert chunk["choices"][0]["delta"]["content"] + + assert list(LLMLogs.objects(metadata__bot=bot)) From 270b0abc9de9e99ce884e103c677fb36c3e4f0cb Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 17:28:09 +0530 Subject: [PATCH 33/57] 1. added missing test case --- tests/unit_test/llm_test.py | 56 +++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index bb446e802..09eea5168 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -688,7 +688,8 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) test_content = CognitionData( @@ -720,7 +721,8 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe ]} mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.side_effect = [{'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + mock_completion.side_effect = [{'choices': [ + {'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[29:60]}, 'finish_reason': None, 'index': 0}]}, {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[60:]}, @@ -745,7 +747,9 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe 'content': 'You are a personal assistant. Answer the question according to the below context'}, {'role': 'user', 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + 'raw_completion_response': {'choices': [ + {'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, + 'index': 0}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': True, 'stop': None, 'presence_penalty': 0.0, @@ -1177,17 +1181,17 @@ async def test_llm_logging(self): litellm.callbacks = [LiteLLMLogger()] result = await litellm.acompletion(messages=["Hi"], - model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", - metadata={'user': user, 'bot': bot}) - assert result - - result = litellm.completion(messages=["Hi"], model="gpt-3.5-turbo", mock_response="Hi, How may i help you?", metadata={'user': user, 'bot': bot}) assert result + result = litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + metadata={'user': user, 'bot': bot}) + assert result + result = litellm.completion(messages=["Hi"], model="gpt-3.5-turbo", mock_response="Hi, How may i help you?", @@ -1197,4 +1201,38 @@ async def test_llm_logging(self): print(chunk["choices"][0]["delta"]["content"]) assert chunk["choices"][0]["delta"]["content"] + result = await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + stream=True, + metadata={'user': user, 'bot': bot}) + async for chunk in result: + print(chunk["choices"][0]["delta"]["content"]) + assert chunk["choices"][0]["delta"]["content"] + assert list(LLMLogs.objects(metadata__bot=bot)) + + with pytest.raises(Exception) as e: + await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + metadata={'user': user, 'bot': bot}) + + assert str(e) == "Authentication error" + + with pytest.raises(Exception) as e: + litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + metadata={'user': user, 'bot': bot}) + + assert str(e) == "Authentication error" + + with pytest.raises(Exception) as e: + await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + stream=True, + metadata={'user': user, 'bot': bot}) + + assert str(e) == "Authentication error" From db4a710c3e22cf68903275d6981d907376f42489 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 18:50:36 +0530 Subject: [PATCH 34/57] 1. added missing test case --- tests/unit_test/llm_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 09eea5168..bcc2ed661 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1210,8 +1210,6 @@ async def test_llm_logging(self): print(chunk["choices"][0]["delta"]["content"]) assert chunk["choices"][0]["delta"]["content"] - assert list(LLMLogs.objects(metadata__bot=bot)) - with pytest.raises(Exception) as e: await litellm.acompletion(messages=["Hi"], model="gpt-3.5-turbo", From 5b83405ea05774645130da6f9612865b3c945efc Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 11:10:39 +0530 Subject: [PATCH 35/57] 1. added missing test case 2. removed deprecated api --- kairon/shared/rest_client.py | 29 +++---------------- tests/integration_test/action_service_test.py | 2 +- tests/integration_test/chat_service_test.py | 2 +- tests/integration_test/event_service_test.py | 2 +- .../integration_test/history_services_test.py | 2 +- tests/integration_test/services_test.py | 4 +-- tests/unit_test/action/action_test.py | 1 - tests/unit_test/chat/chat_test.py | 27 +++++++++-------- tests/unit_test/cli_test.py | 17 +++++------ .../data_processor/agent_processor_test.py | 2 +- .../data_processor/data_processor_test.py | 2 +- .../unit_test/data_processor/history_test.py | 2 +- tests/unit_test/events/definitions_test.py | 4 +-- tests/unit_test/events/events_test.py | 2 +- tests/unit_test/events/scheduler_test.py | 2 +- tests/unit_test/idp/test_idp_helper.py | 1 - tests/unit_test/llm_test.py | 7 ++--- tests/unit_test/plugins_test.py | 3 +- tests/unit_test/rest_client_test.py | 17 +++++++++++ tests/unit_test/verification_test.py | 2 +- 20 files changed, 59 insertions(+), 71 deletions(-) diff --git a/kairon/shared/rest_client.py b/kairon/shared/rest_client.py index 301e11323..5c144a0df 100644 --- a/kairon/shared/rest_client.py +++ b/kairon/shared/rest_client.py @@ -32,14 +32,6 @@ def __init__(self, close_session_with_rqst_completion=True): self._time_elapsed = None self._status_code = None - @property - def streaming_response(self): - return self._streaming_response - - @streaming_response.setter - def streaming_response(self, resp): - self._streaming_response = resp - @property def time_elapsed(self): return self._time_elapsed @@ -124,12 +116,9 @@ async def __trigger(self, client, *args, **kwargs) -> ClientResponse: logger.debug(f"Content-type: {response.headers['content-type']}") logger.debug(f"Status code: {str(response.status)}") self.status_code = response.status - if is_streaming_resp: - streaming_resp = await AioRestClient.parse_streaming_response(response) - self.streaming_response = streaming_resp - logger.debug(f"Raw streaming response: {streaming_resp}") - text = await response.text() - logger.debug(f"Raw response: {text}") + if not is_streaming_resp: + text = await response.text() + logger.debug(f"Raw response: {text}") return response def __validate_response(self, response: ClientResponse, **kwargs): @@ -149,14 +138,4 @@ async def cleanup(self): Close underlying connector to release all acquired resources. """ if not self.session.closed: - await self.session.close() - - @staticmethod - async def parse_streaming_response(response): - chunks = [] - async for chunk in response.content: - if not chunk: - break - chunks.append(chunk) - - return chunks + await self.session.close() \ No newline at end of file diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index 13dc8d7e7..e71a95494 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -3,7 +3,7 @@ from urllib.parse import urlencode, urljoin import litellm -import mock +from unittest import mock import numpy as np import pytest import responses diff --git a/tests/integration_test/chat_service_test.py b/tests/integration_test/chat_service_test.py index 77b828a71..04d65276a 100644 --- a/tests/integration_test/chat_service_test.py +++ b/tests/integration_test/chat_service_test.py @@ -19,7 +19,7 @@ import pytest import responses -from mock import patch +from unittest.mock import patch from mongoengine import connect from slack_sdk.web.slack_response import SlackResponse from starlette.exceptions import HTTPException diff --git a/tests/integration_test/event_service_test.py b/tests/integration_test/event_service_test.py index 445a37949..f03f65817 100644 --- a/tests/integration_test/event_service_test.py +++ b/tests/integration_test/event_service_test.py @@ -3,7 +3,7 @@ from dramatiq.brokers.stub import StubBroker from loguru import logger -from mock import patch +from unittest.mock import patch from starlette.testclient import TestClient from kairon.shared.constants import EventClass, EventExecutor diff --git a/tests/integration_test/history_services_test.py b/tests/integration_test/history_services_test.py index 2d8cc807a..46e683b43 100644 --- a/tests/integration_test/history_services_test.py +++ b/tests/integration_test/history_services_test.py @@ -8,7 +8,7 @@ from mongomock import MongoClient from kairon.history.processor import HistoryProcessor from pymongo.collection import Collection -import mock +from unittest import mock from urllib.parse import urlencode diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index 7f944d109..d0645e60b 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -6,11 +6,11 @@ import tempfile from datetime import datetime, timedelta from io import BytesIO -from mock import patch +from unittest.mock import patch from urllib.parse import urljoin from zipfile import ZipFile -import mock +from unittest import mock import pytest import responses from botocore.exceptions import ClientError diff --git a/tests/unit_test/action/action_test.py b/tests/unit_test/action/action_test.py index 6b361ad44..221d6d1aa 100644 --- a/tests/unit_test/action/action_test.py +++ b/tests/unit_test/action/action_test.py @@ -3,7 +3,6 @@ import re from unittest import mock -import mock from googleapiclient.http import HttpRequest from pipedrive.exceptions import UnauthorizedError, BadRequestError from kairon.shared.utils import Utility diff --git a/tests/unit_test/chat/chat_test.py b/tests/unit_test/chat/chat_test.py index 9abbcab6b..6336152bb 100644 --- a/tests/unit_test/chat/chat_test.py +++ b/tests/unit_test/chat/chat_test.py @@ -3,7 +3,7 @@ import ujson as json import os from re import escape -from unittest.mock import patch +from unittest import mock from urllib.parse import urlencode, quote_plus import mongomock @@ -21,7 +21,6 @@ from kairon.shared.data.constant import ACCESS_ROLES, TOKEN_TYPE from kairon.shared.data.utils import DataUtility from kairon.shared.utils import Utility -import mock from pymongo.errors import ServerSelectionTimeoutError @@ -49,7 +48,7 @@ def test_save_channel_config_invalid(self): "test", "test" ) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -93,7 +92,7 @@ def test_save_channel_config_invalid(self): "test" ) - @patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) + @mock.patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) def test_save_channel_config_slack_team_id_error(self, mock_slack_info): mock_slack_info.side_effect = AppException("The request to the Slack API failed. ") with pytest.raises(AppException, match="The request to the Slack API failed.*"): @@ -108,7 +107,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -133,7 +132,7 @@ def __mock_get_bot(*args, **kwargs): "client_secret": "a23456789sfdghhtyutryuivcbn", "is_primary": True}}, "test", "test" ) - @patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) + @mock.patch("kairon.shared.utils.Utility.get_slack_team_info", autospec=True) def test_save_channel_config_slack_secondary_app_team_id_error(self, mock_slack_info ): mock_slack_info.side_effect = AppException("The request to the Slack API failed. ") with pytest.raises(AppException, match="The request to the Slack API failed.*"): @@ -149,7 +148,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -252,7 +251,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -317,7 +316,7 @@ def __mock_get_bot(*args, **kwargs): return {"account": 1000} monkeypatch.setattr(AccountProcessor, "get_bot", __mock_get_bot) - with patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: + with mock.patch("slack_sdk.web.client.WebClient.team_info") as mock_slack_resp: mock_slack_resp.return_value = SlackResponse( client=self, http_verb="POST", @@ -384,7 +383,7 @@ def test_save_channel_config_telegram(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/telegram/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): ChatDataProcessor.save_channel_config({"connector_type": "telegram", "config": { "access_token": access_token, @@ -405,7 +404,7 @@ def test_save_channel_config_telegram_invalid(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/telegram/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): ChatDataProcessor.save_channel_config({"connector_type": "telegram", "config": { "access_token": access_token, @@ -487,7 +486,7 @@ def test_save_channel_config_business_messages_with_invalid_private_key(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/business_messages/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): channel_endpoint = ChatDataProcessor.save_channel_config( { "connector_type": "business_messages", @@ -515,7 +514,7 @@ def test_save_channel_config_business_messages(self): def __mock_endpoint(*args): return f"https://test@test.com/api/bot/business_messages/tests/test" - with patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): + with mock.patch('kairon.shared.data.utils.DataUtility.get_channel_endpoint', __mock_endpoint): channel_endpoint = ChatDataProcessor.save_channel_config( { "connector_type": "business_messages", @@ -603,7 +602,7 @@ def test_get_channel_end_point_whatsapp(self, monkeypatch): def _mock_generate_integration_token(*arge, **kwargs): return "testtoken", "ignore" - with patch.object(Authentication, "generate_integration_token", _mock_generate_integration_token): + with mock.patch.object(Authentication, "generate_integration_token", _mock_generate_integration_token): channel_url = ChatDataProcessor.save_channel_config({ "connector_type": "whatsapp", "config": { "app_secret": "app123", diff --git a/tests/unit_test/cli_test.py b/tests/unit_test/cli_test.py index 80f1e6bf2..1f28f1fd2 100644 --- a/tests/unit_test/cli_test.py +++ b/tests/unit_test/cli_test.py @@ -1,8 +1,10 @@ +import argparse +import os from datetime import datetime -from unittest.mock import patch +from unittest import mock import pytest -import os +from mongoengine import connect from kairon import cli from kairon.cli.conversations_deletion import initiate_history_deletion_archival @@ -10,22 +12,17 @@ from kairon.cli.delete_logs import delete_logs from kairon.cli.importer import validate_and_import from kairon.cli.message_broadcast import send_notifications -from kairon.cli.training import train from kairon.cli.testing import run_tests_on_model +from kairon.cli.training import train from kairon.cli.translator import translate_multilingual_bot from kairon.events.definitions.data_generator import DataGenerationEvent from kairon.events.definitions.data_importer import TrainingDataImporterEvent from kairon.events.definitions.history_delete import DeleteHistoryEvent -from kairon.events.definitions.message_broadcast import MessageBroadcastEvent from kairon.events.definitions.model_testing import ModelTestingEvent from kairon.events.definitions.multilingual import MultilingualEvent from kairon.shared.concurrency.actors.factory import ActorFactory -from kairon.shared.utils import Utility -from mongoengine import connect -import mock -import argparse - from kairon.shared.constants import EventClass +from kairon.shared.utils import Utility class TestTrainingCli: @@ -395,7 +392,7 @@ def test_message_broadcast_no_event_id(self, monkeypatch): return_value=argparse.Namespace(func=send_notifications, bot="test_cli", user="testUser", event_id="65432123456789876543")) def test_message_broadcast_all_arguments(self, mock_namespace): - with patch('kairon.events.definitions.message_broadcast.MessageBroadcastEvent.execute', autospec=True): + with mock.patch('kairon.events.definitions.message_broadcast.MessageBroadcastEvent.execute', autospec=True): cli() for proxy in ActorFactory._ActorFactory__actors.values(): diff --git a/tests/unit_test/data_processor/agent_processor_test.py b/tests/unit_test/data_processor/agent_processor_test.py index ec62c2dec..8d37b1541 100644 --- a/tests/unit_test/data_processor/agent_processor_test.py +++ b/tests/unit_test/data_processor/agent_processor_test.py @@ -16,7 +16,7 @@ from kairon.shared.data.constant import EVENT_STATUS from kairon.shared.data.model_processor import ModelProcessor -from mock import patch +from unittest.mock import patch from mongoengine import connect diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 0f0c563ad..4b6844865 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -14,7 +14,7 @@ Utility.load_system_metadata() -from mock import patch +from unittest.mock import patch import numpy as np import pandas as pd import pytest diff --git a/tests/unit_test/data_processor/history_test.py b/tests/unit_test/data_processor/history_test.py index 3428736b9..8d7532bc2 100644 --- a/tests/unit_test/data_processor/history_test.py +++ b/tests/unit_test/data_processor/history_test.py @@ -2,7 +2,7 @@ import os from datetime import datetime -import mock +from unittest import mock import mongomock import pytest diff --git a/tests/unit_test/events/definitions_test.py b/tests/unit_test/events/definitions_test.py index aff9455f7..988b14072 100644 --- a/tests/unit_test/events/definitions_test.py +++ b/tests/unit_test/events/definitions_test.py @@ -4,11 +4,11 @@ from io import BytesIO from urllib.parse import urljoin -import mock +from unittest import mock import pytest import responses from fastapi import UploadFile -from mock.mock import patch +from unittest.mock import patch from mongoengine import connect from augmentation.utils import WebsiteParser diff --git a/tests/unit_test/events/events_test.py b/tests/unit_test/events/events_test.py index 4f7a63438..7f1448a9d 100644 --- a/tests/unit_test/events/events_test.py +++ b/tests/unit_test/events/events_test.py @@ -8,7 +8,7 @@ from unittest.mock import patch from urllib.parse import urljoin -import mock +from unittest import mock import mongomock import pytest import responses diff --git a/tests/unit_test/events/scheduler_test.py b/tests/unit_test/events/scheduler_test.py index ec6426415..678c11d2e 100644 --- a/tests/unit_test/events/scheduler_test.py +++ b/tests/unit_test/events/scheduler_test.py @@ -1,7 +1,7 @@ import os import re -from mock import patch +from unittest.mock import patch import pytest from apscheduler.jobstores.mongodb import MongoDBJobStore diff --git a/tests/unit_test/idp/test_idp_helper.py b/tests/unit_test/idp/test_idp_helper.py index db6ff9609..0d74cc842 100644 --- a/tests/unit_test/idp/test_idp_helper.py +++ b/tests/unit_test/idp/test_idp_helper.py @@ -17,7 +17,6 @@ from kairon.shared.organization.processor import OrgProcessor from kairon.shared.utils import Utility from stress_test.data_objects import User -from mock import patch def get_user(): diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index bcc2ed661..16daaec64 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1,13 +1,13 @@ import os -import time +from unittest import mock from urllib.parse import urljoin -import mock import numpy as np import pytest import ujson as json from aiohttp import ClientConnectionError from mongoengine import connect + from kairon.shared.utils import Utility Utility.load_system_metadata() @@ -18,7 +18,6 @@ from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT from kairon.shared.llm.processor import LLMProcessor -from kairon.shared.llm.data_objects import LLMLogs import litellm from deepdiff import DeepDiff @@ -1233,4 +1232,4 @@ async def test_llm_logging(self): stream=True, metadata={'user': user, 'bot': bot}) - assert str(e) == "Authentication error" + assert str(e) == "Authentication error" \ No newline at end of file diff --git a/tests/unit_test/plugins_test.py b/tests/unit_test/plugins_test.py index cc24f8f04..bb9de7443 100644 --- a/tests/unit_test/plugins_test.py +++ b/tests/unit_test/plugins_test.py @@ -1,7 +1,7 @@ import os import re -import mock +from unittest import mock import pytest import requests import responses @@ -11,7 +11,6 @@ from kairon.shared.constants import PluginTypes from kairon.shared.plugins.factory import PluginFactory from kairon.shared.utils import Utility -from mongomock import MongoClient class TestUtility: diff --git a/tests/unit_test/rest_client_test.py b/tests/unit_test/rest_client_test.py index 0d6afa160..6daa1e5e4 100644 --- a/tests/unit_test/rest_client_test.py +++ b/tests/unit_test/rest_client_test.py @@ -1,4 +1,5 @@ import asyncio +import ujson as json from unittest import mock import pytest @@ -101,3 +102,19 @@ async def test_aio_rest_client_timeout_error(self, aioresponses): with pytest.raises(AppException, match="Request timed out: Request timed out"): await AioRestClient().request("get", url, request_body={"name": "udit.pandey", "loc": "blr"}, headers={"Authorization": "Bearer sasdfghjkytrtyui"}, max_retries=3) + + @pytest.mark.asyncio + async def test_aio_rest_client_post_request_stream(self, aioresponses): + url = 'http://kairon.com' + aioresponses.post("http://kairon.com", status=200, body=json.dumps({'data': 'hi!'})) + resp = await AioRestClient().request("post", url, request_body={"name": "udit.pandey", "loc": "blr"}, + headers={"Authorization": "Bearer sasdfghjkytrtyui"}, is_streaming_resp=True) + response = '' + async for content in resp.content: + response += content.decode() + + assert json.loads(response) == {"data": "hi!"} + assert list(aioresponses.requests.values())[0][0].kwargs == {'allow_redirects': True, 'headers': { + 'Authorization': 'Bearer sasdfghjkytrtyui'}, 'json': {'loc': 'blr', 'name': 'udit.pandey'}, 'timeout': None, + 'data': None, + 'trace_request_ctx': {'current_attempt': 1}} \ No newline at end of file diff --git a/tests/unit_test/verification_test.py b/tests/unit_test/verification_test.py index 4e6c2b360..6462ae579 100644 --- a/tests/unit_test/verification_test.py +++ b/tests/unit_test/verification_test.py @@ -2,7 +2,7 @@ import responses from kairon.shared.verification.email import QuickEmailVerification from urllib.parse import urlencode -import mock +from unittest import mock from kairon.shared.utils import Utility import os From e57d183249eef2d975d470ef1e821415b04d0359 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 13:38:29 +0530 Subject: [PATCH 36/57] test cases fixed --- kairon/shared/llm/logger.py | 3 ++- tests/unit_test/llm_test.py | 46 +++++++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py index dbb93e469..3af7a7870 100644 --- a/kairon/shared/llm/logger.py +++ b/kairon/shared/llm/logger.py @@ -37,4 +37,5 @@ def __logs_litellm(self, **kwargs): 'metadata': litellm_params.get('metadata')}) def __save_logs(self, **kwargs): - print(LLMLogs(**kwargs).save().to_mongo().to_dict()) + logs = LLMLogs(**kwargs).save().to_mongo().to_dict() + logger.info(f"llm logs: {logs}") diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 16daaec64..1042be91d 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1179,38 +1179,50 @@ async def test_llm_logging(self): user = "test" litellm.callbacks = [LiteLLMLogger()] - result = await litellm.acompletion(messages=["Hi"], + messages = [{"role":"user", "content":"Hi"}] + expected = "Hi, How may i help you?" + + result = await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, metadata={'user': user, 'bot': bot}) - assert result + assert result['choices'][0]['message']['content'] == expected - result = litellm.completion(messages=["Hi"], + result = litellm.completion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, metadata={'user': user, 'bot': bot}) - assert result + assert result['choices'][0]['message']['content'] == expected - result = litellm.completion(messages=["Hi"], + result = litellm.completion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, stream=True, metadata={'user': user, 'bot': bot}) + response = '' for chunk in result: - print(chunk["choices"][0]["delta"]["content"]) - assert chunk["choices"][0]["delta"]["content"] + content = chunk["choices"][0]["delta"]["content"] + if content: + response = response + content + + assert response == expected - result = await litellm.acompletion(messages=["Hi"], + result = await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, stream=True, metadata={'user': user, 'bot': bot}) + response = '' async for chunk in result: - print(chunk["choices"][0]["delta"]["content"]) - assert chunk["choices"][0]["delta"]["content"] + content = chunk["choices"][0]["delta"]["content"] + print(chunk) + if content: + response += content + + assert response.__contains__(expected) with pytest.raises(Exception) as e: - await litellm.acompletion(messages=["Hi"], + await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), metadata={'user': user, 'bot': bot}) @@ -1218,7 +1230,7 @@ async def test_llm_logging(self): assert str(e) == "Authentication error" with pytest.raises(Exception) as e: - litellm.completion(messages=["Hi"], + litellm.completion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), metadata={'user': user, 'bot': bot}) @@ -1226,7 +1238,7 @@ async def test_llm_logging(self): assert str(e) == "Authentication error" with pytest.raises(Exception) as e: - await litellm.acompletion(messages=["Hi"], + await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), stream=True, From c75d3ef319de19f310214425bc273ef1a3b76d45 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 14:53:59 +0530 Subject: [PATCH 37/57] removed unused variable --- kairon/actions/definitions/prompt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 4c7bf6bc4..6323589b9 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -66,7 +66,6 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma k_faq_action_config, bot_settings = self.retrieve_config() user_question = k_faq_action_config.get('user_question') user_msg = self.__get_user_msg(tracker, user_question) - llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) llm_processor = LLMProcessor(self.bot) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, From 423ec461136c6120f803e7bc3a88b5a1cd90aaed Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 15:15:10 +0530 Subject: [PATCH 38/57] fixed unused variable --- kairon/actions/definitions/prompt.py | 3 +- kairon/shared/llm/processor.py | 3 +- kairon/shared/vector_embeddings/db/qdrant.py | 6 ++-- kairon/train.py | 4 +-- tests/unit_test/llm_test.py | 36 ++++++++++---------- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 6323589b9..6441d607f 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -66,8 +66,9 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma k_faq_action_config, bot_settings = self.retrieve_config() user_question = k_faq_action_config.get('user_question') user_msg = self.__get_user_msg(tracker, user_question) + llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) - llm_processor = LLMProcessor(self.bot) + llm_processor = LLMProcessor(self.bot, llm_type) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, user=tracker.sender_id, **llm_params) diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index c6e7fa8af..adbb039ae 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -26,13 +26,14 @@ class LLMProcessor(LLMBase): __embedding__ = 1536 - def __init__(self, bot: Text): + def __init__(self, bot: Text, llm_type: str): super().__init__(bot) self.db_url = Utility.environment['vector']['db'] self.headers = {} if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.suffix = "_faq_embd" + self.llm_type = llm_type self.vector_config = {'size': self.__embedding__, 'distance': 'Cosine'} self.api_key = Sysadmin.get_bot_secret(bot, BotSecretType.gpt_key.value, raise_err=True) self.tokenizer = get_encoding("cl100k_base") diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index 12b5268e2..454eeb8fe 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -5,10 +5,8 @@ from kairon import Utility from kairon.shared.actions.utils import ActionUtility -from kairon.shared.admin.constants import BotSecretType -from kairon.shared.admin.processor import Sysadmin -from kairon.shared.constants import GPT3ResourceTypes from kairon.shared.llm.processor import LLMProcessor +from kairon.shared.data.constant import DEFAULT_LLM from kairon.shared.vector_embeddings.db.base import VectorEmbeddingsDbBase @@ -25,7 +23,7 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.llm_settings = llm_settings - self.llm = LLMProcessor(self.bot) + self.llm = LLMProcessor(self.bot, DEFAULT_LLM) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 diff --git a/kairon/train.py b/kairon/train.py index 3ddcf9eb0..d8360ce67 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -6,7 +6,7 @@ from rasa.api import train from rasa.model import DEFAULT_MODELS_PATH from rasa.shared.constants import DEFAULT_CONFIG_PATH, DEFAULT_DATA_PATH, DEFAULT_DOMAIN_PATH - +from kairon.shared.data.constant import DEFAULT_LLM from kairon.chat.agent.agent import KaironAgent from kairon.exceptions import AppException from kairon.shared.account.processor import AccountProcessor @@ -101,7 +101,7 @@ def start_training(bot: str, user: str, token: str = None): settings = processor.get_bot_settings(bot, user) settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: - llm_processor = LLMProcessor(bot) + llm_processor = LLMProcessor(bot, DEFAULT_LLM) faqs = asyncio.run(llm_processor.train(user=user)) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 1042be91d..a385106cd 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -16,7 +16,7 @@ from kairon.shared.admin.constants import BotSecretType from kairon.shared.admin.data_objects import BotSecrets from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema -from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT +from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_LLM from kairon.shared.llm.processor import LLMProcessor import litellm from deepdiff import DeepDiff @@ -44,7 +44,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM)[[]] aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -123,7 +123,7 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore embedding = list(np.random.random(LLMProcessor.__embedding__)) mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, { 'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -227,7 +227,7 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a input = {"name": "Ram", "color": "red"} mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), @@ -288,7 +288,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): input = {"name": "Ram", "color": "red"} mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -339,7 +339,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): def test_gpt3_faq_embedding_train_failure(self): with pytest.raises(AppException, match=f"Bot secret '{BotSecretType.gpt_key.value}' not configured!"): - LLMProcessor('test_failure') + LLMProcessor('test_failure', DEFAULT_LLM) @pytest.mark.asyncio @mock.patch.object(litellm, "aembedding", autospec=True) @@ -357,7 +357,7 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aiore mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -421,7 +421,7 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -507,7 +507,7 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -574,7 +574,7 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_emb mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -641,7 +641,7 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'vector': {"key": "test", 'db': "http://localhost:6333"}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -729,7 +729,7 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe ] with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -832,7 +832,7 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content1.collection}{gpt3.suffix}/points/search"), @@ -908,7 +908,7 @@ def __mock_connection_error(*args, **kwargs): mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.side_effect = __mock_connection_error - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -971,7 +971,7 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_llm_request.side_effect = ClientConnectionError() - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} @@ -1009,7 +1009,7 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ } mock_embedding.side_effect = [Exception("Connection reset by peer!"), {'data': [{'embedding': embedding}]}] - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) @@ -1064,7 +1064,7 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -1143,7 +1143,7 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, { 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], From e7cd631fc51c9f5bfb730ab73dc4e37af0a50217 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 16:28:11 +0530 Subject: [PATCH 39/57] fixed unused variable --- tests/unit_test/llm_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index a385106cd..53c31bace 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -44,7 +44,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM)[[]] + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), From a05ebd1a993011544a547a3ce5b5ef3e783a6162 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 28 Jun 2024 12:56:08 +0530 Subject: [PATCH 40/57] added test cased for fetching logs --- kairon/api/app/routers/bot/bot.py | 23 +++++++++++-- kairon/shared/llm/processor.py | 24 +++++++++++++ tests/integration_test/services_test.py | 45 ++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/kairon/api/app/routers/bot/bot.py b/kairon/api/app/routers/bot/bot.py index 6dfeda5c5..276404377 100644 --- a/kairon/api/app/routers/bot/bot.py +++ b/kairon/api/app/routers/bot/bot.py @@ -26,7 +26,7 @@ from kairon.shared.actions.data_objects import ActionServerLogs from kairon.shared.auth import Authentication from kairon.shared.constants import TESTER_ACCESS, DESIGNER_ACCESS, CHAT_ACCESS, UserActivityType, ADMIN_ACCESS, \ - VIEW_ACCESS, EventClass, AGENT_ACCESS + EventClass, AGENT_ACCESS from kairon.shared.data.assets_processor import AssetsProcessor from kairon.shared.data.audit.processor import AuditDataProcessor from kairon.shared.data.constant import EVENT_STATUS, ENDPOINT_TYPE, TOKEN_TYPE, ModelTestType, \ @@ -38,10 +38,12 @@ from kairon.shared.data.utils import DataUtility from kairon.shared.importer.data_objects import ValidationLogs from kairon.shared.importer.processor import DataImporterLogProcessor +from kairon.shared.live_agent.live_agent import LiveAgentHandler from kairon.shared.models import User, TemplateType from kairon.shared.test.processor import ModelTestingLogProcessor from kairon.shared.utils import Utility -from kairon.shared.live_agent.live_agent import LiveAgentHandler +from kairon.shared.llm.processor import LLMProcessor +from kairon.shared.llm.data_objects import LLMLogs router = APIRouter() @@ -1668,3 +1670,20 @@ async def get_live_agent_token(current_user: User = Security(Authentication.get_ data = await LiveAgentHandler.authenticate_agent(current_user.get_user(), current_user.get_bot()) return Response(data=data) + +@router.get("/llm/logs", response_model=Response) +async def get_llm_logs( + start_idx: int = 0, page_size: int = 10, + current_user: User = Security(Authentication.get_current_user_and_bot, scopes=TESTER_ACCESS) +): + """ + Get data llm event logs. + """ + logs = list(LLMProcessor.get_logs(current_user.get_bot(), start_idx, page_size)) + row_cnt = LLMProcessor.get_row_count(current_user.get_bot()) + data = { + "logs": logs, + "total": row_cnt + } + return Response(data=data) + diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index adbb039ae..7365e0aa7 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -16,6 +16,7 @@ from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_CONTEXT_PROMPT from kairon.shared.llm.base import LLMBase from kairon.shared.llm.logger import LiteLLMLogger +from kairon.shared.llm.data_objects import LLMLogs from kairon.shared.models import CognitionDataType from kairon.shared.rest_client import AioRestClient from kairon.shared.utils import Utility @@ -260,3 +261,26 @@ async def __attach_similarity_prompt_if_enabled(self, query_embedding, context_p similarity_context = f"Instructions on how to use {similarity_prompt_name}:\n{extracted_values}\n{similarity_prompt_instructions}\n" context_prompt = f"{context_prompt}\n{similarity_context}" return context_prompt + + @staticmethod + def get_logs(bot: str, start_idx: int = 0, page_size: int = 10): + """ + Get all logs for data importer event. + @param bot: bot id. + @param start_idx: start index + @param page_size: page size + @return: list of logs. + """ + for log in LLMLogs.objects(metadata__bot=bot).order_by("-start_time").skip(start_idx).limit(page_size): + llm_log = log.to_mongo().to_dict() + llm_log.pop('_id') + yield llm_log + + @staticmethod + def get_row_count(bot: str): + """ + Gets the count of rows in a LLMLogs for a particular bot. + :param bot: bot id + :return: Count of rows + """ + return LLMLogs.objects(metadata__bot=bot).count() diff --git a/tests/integration_test/services_test.py b/tests/integration_test/services_test.py index d0645e60b..1f968ec39 100644 --- a/tests/integration_test/services_test.py +++ b/tests/integration_test/services_test.py @@ -1,3 +1,5 @@ +import time + import ujson as json import os import re @@ -23661,6 +23663,47 @@ def test_trigger_widget(): assert actual["error_code"] == 0 assert len(actual["data"]) == 2 assert not actual["message"] + + +def test_get_llm_logs(): + from kairon.shared.llm.logger import LiteLLMLogger + import litellm + import asyncio + + loop = asyncio.new_event_loop() + user = "test" + litellm.callbacks = [LiteLLMLogger()] + + messages = [{"role": "user", "content": "Hi"}] + expected = "Hi, How may i help you?" + + result = loop.run_until_complete(litellm.acompletion(messages=messages, + model="gpt-3.5-turbo", + mock_response=expected, + metadata={'user': user, 'bot': pytest.bot})) + assert result['choices'][0]['message']['content'] == expected + + time.sleep(2) + + response = client.get( + f"/api/bot/{pytest.bot}/llm/logs?start_idx=0&page_size=10", + headers={"Authorization": pytest.token_type + " " + pytest.access_token}, + ) + actual = response.json() + print(actual) + assert actual["success"] + assert actual["error_code"] == 0 + assert len(actual["data"]["logs"]) == 1 + assert actual["data"]["total"] == 1 + assert actual["data"]["logs"][0]['start_time'] + assert actual["data"]["logs"][0]['end_time'] + assert actual["data"]["logs"][0]['cost'] + assert actual["data"]["logs"][0]['llm_call_id'] + assert actual["data"]["logs"][0]["llm_provider"] == "openai" + assert not actual["data"]["logs"][0].get("model") + assert actual["data"]["logs"][0]["model_params"] == {} + assert actual["data"]["logs"][0]["metadata"]['bot'] == pytest.bot + assert actual["data"]["logs"][0]["metadata"]['user'] == "test" def test_add_custom_widget_invalid_config(): @@ -24560,4 +24603,4 @@ def test_list_system_metadata(): actual = response.json() assert actual["error_code"] == 0 assert actual["success"] - assert len(actual["data"]) == 17 + assert len(actual["data"]) == 17 \ No newline at end of file From c286989fce999b25158ffdf5cac67dfad8e4ae18 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 28 Jun 2024 15:26:15 +0530 Subject: [PATCH 41/57] added test cased for fetching logs --- tests/unit_test/data_processor/data_processor_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 4b6844865..3c93e1044 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -82,7 +82,6 @@ from kairon.shared.data.utils import DataUtility from kairon.shared.importer.processor import DataImporterLogProcessor from kairon.shared.live_agent.live_agent import LiveAgentHandler -from kairon.shared.llm.gpt3 import GPT3FAQEmbedding from kairon.shared.metering.constants import MetricType from kairon.shared.metering.data_object import Metering from kairon.shared.models import StoryEventType, HttpContentType, CognitionDataType From f48dc2ee837eb9ad643be483031b0fe390b60d21 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 21 Jun 2024 20:21:05 +0530 Subject: [PATCH 42/57] litellm base version --- kairon/actions/definitions/database.py | 2 +- kairon/actions/definitions/prompt.py | 3 +- kairon/api/models.py | 36 +- kairon/shared/llm/base.py | 4 +- kairon/shared/llm/logger.py | 28 +- kairon/shared/llm/processor.py | 68 +- kairon/shared/utils.py | 4 +- kairon/shared/vector_embeddings/db/base.py | 8 +- kairon/shared/vector_embeddings/db/qdrant.py | 24 +- kairon/train.py | 6 +- metadata/integrations.yml | 4 + requirements/prod.txt | 2 +- tests/integration_test/action_service_test.py | 1074 ++--------------- .../data_processor/data_processor_test.py | 188 +-- tests/unit_test/llm_test.py | 503 +++----- 15 files changed, 383 insertions(+), 1571 deletions(-) diff --git a/kairon/actions/definitions/database.py b/kairon/actions/definitions/database.py index 0d54abd49..ebdf83510 100644 --- a/kairon/actions/definitions/database.py +++ b/kairon/actions/definitions/database.py @@ -83,7 +83,7 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma request_body = ActionUtility.get_payload(payload, tracker) msg_logger.append(request_body) tracker_data = ActionUtility.build_context(tracker, True) - response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id) + response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id, bot=self.bot) logger.info("response: " + str(response)) response_context = self.__add_user_context_to_http_response(response, tracker_data) bot_response, bot_resp_log, _ = ActionUtility.compose_response(vector_action_config['response'], response_context) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 6441d607f..381e6f543 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -68,9 +68,10 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma user_msg = self.__get_user_msg(tracker, user_question) llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) - llm_processor = LLMProcessor(self.bot, llm_type) + llm_processor = LLMProcessor(self.bot) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, user=tracker.sender_id, + bot=self.bot, **llm_params) status = "FAILURE" if llm_response.get("is_failure", False) is True else status exception = llm_response.get("exception") diff --git a/kairon/api/models.py b/kairon/api/models.py index d61a98126..435566f34 100644 --- a/kairon/api/models.py +++ b/kairon/api/models.py @@ -15,7 +15,8 @@ ACTIVITY_STATUS, INTEGRATION_STATUS, FALLBACK_MESSAGE, - DEFAULT_NLU_FALLBACK_RESPONSE + DEFAULT_NLU_FALLBACK_RESPONSE, + DEFAULT_LLM ) from ..shared.actions.models import ( ActionParameterType, @@ -903,38 +904,16 @@ def validate_top_n(cls, v, values, **kwargs): return v -class CustomActionParameterModel(BaseModel): - value: Any = None - parameter_type: ActionParameterType = ActionParameterType.value - - @validator("parameter_type") - def validate_parameter_type(cls, v, values, **kwargs): - allowed_values = {ActionParameterType.value, ActionParameterType.slot} - if v not in allowed_values: - raise ValueError(f"Invalid parameter type. Allowed values: {allowed_values}") - return v - - @root_validator - def check(cls, values): - if values.get('parameter_type') == ActionParameterType.slot and not values.get('value'): - raise ValueError("Provide name of the slot as value") - - if values.get('parameter_type') == ActionParameterType.value and not isinstance(values.get('value'), list): - raise ValueError("Provide list of emails as value") - - return values - - class EmailActionRequest(BaseModel): action_name: constr(to_lower=True, strip_whitespace=True) smtp_url: str smtp_port: int smtp_userid: CustomActionParameter = None smtp_password: CustomActionParameter - from_email: CustomActionParameter + from_email: str subject: str custom_text: CustomActionParameter = None - to_email: CustomActionParameterModel + to_email: List[str] response: str tls: bool = False @@ -1058,8 +1037,8 @@ class PromptActionConfigRequest(BaseModel): num_bot_responses: int = 5 failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE user_question: UserQuestionModel = UserQuestionModel() - llm_type: str - hyperparameters: dict + llm_type: str = DEFAULT_LLM + hyperparameters: dict = None llm_prompts: List[LlmPromptRequest] instructions: List[str] = [] set_slots: List[SetSlotsUsingActionResponse] = [] @@ -1088,8 +1067,7 @@ def validate_llm_type(cls, v, values, **kwargs): @validator("hyperparameters") def validate_llm_hyperparameters(cls, v, values, **kwargs): - if values.get('llm_type'): - Utility.validate_llm_hyperparameters(v, values['llm_type'], ValueError) + Utility.validate_llm_hyperparameters(v, kwargs['llm_type'], ValueError) @root_validator def check(cls, values): diff --git a/kairon/shared/llm/base.py b/kairon/shared/llm/base.py index 006e38a3d..f07eceda0 100644 --- a/kairon/shared/llm/base.py +++ b/kairon/shared/llm/base.py @@ -8,9 +8,9 @@ def __init__(self, bot: Text): self.bot = bot @abstractmethod - async def train(self, user, *args, **kwargs) -> Dict: + async def train(self, user, bot, *args, **kwargs) -> Dict: pass @abstractmethod - async def predict(self, query, user, *args, **kwargs) -> Dict: + async def predict(self, query, user, bot, *args, **kwargs) -> Dict: pass diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py index 3af7a7870..06b720b48 100644 --- a/kairon/shared/llm/logger.py +++ b/kairon/shared/llm/logger.py @@ -1,10 +1,14 @@ from litellm.integrations.custom_logger import CustomLogger from .data_objects import LLMLogs import ujson as json -from loguru import logger class LiteLLMLogger(CustomLogger): + def log_pre_api_call(self, model, messages, kwargs): + pass + + def log_post_api_call(self, kwargs, response_obj, start_time, end_time): + pass def log_stream_event(self, kwargs, response_obj, start_time, end_time): self.__logs_litellm(**kwargs) @@ -25,17 +29,15 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti self.__logs_litellm(**kwargs) def __logs_litellm(self, **kwargs): - logger.info("logging llms call") - litellm_params = kwargs.get('litellm_params') - self.__save_logs(**{'response': json.loads(kwargs.get('original_response')) if kwargs.get('original_response') else None, - 'start_time': kwargs.get('start_time'), - 'end_time': kwargs.get('end_time'), - 'cost': kwargs.get("response_cost"), - 'llm_call_id': litellm_params.get('litellm_call_id'), - 'llm_provider': litellm_params.get('custom_llm_provider'), - 'model_params': kwargs.get("additional_args", {}).get("complete_input_dict"), - 'metadata': litellm_params.get('metadata')}) + litellm_params = kwargs['litellm_params'] + self.__save_logs(**{'response': json.loads(kwargs['original_response']), + 'start_time': kwargs['start_time'], + 'end_time': kwargs['end_time'], + 'cost': kwargs["response_cost"], + 'llm_call_id': litellm_params['litellm_call_id'], + 'llm_provider': litellm_params['custom_llm_provider'], + 'model_params': kwargs["additional_args"]["complete_input_dict"], + 'metadata': litellm_params['metadata']}) def __save_logs(self, **kwargs): - logs = LLMLogs(**kwargs).save().to_mongo().to_dict() - logger.info(f"llm logs: {logs}") + LLMLogs(**kwargs).save() diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index 7365e0aa7..ffc48e2eb 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -1,4 +1,4 @@ -from secrets import randbelow, choice +import random import time from typing import Text, Dict, List, Tuple from urllib.parse import urljoin @@ -16,7 +16,6 @@ from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_CONTEXT_PROMPT from kairon.shared.llm.base import LLMBase from kairon.shared.llm.logger import LiteLLMLogger -from kairon.shared.llm.data_objects import LLMLogs from kairon.shared.models import CognitionDataType from kairon.shared.rest_client import AioRestClient from kairon.shared.utils import Utility @@ -27,21 +26,20 @@ class LLMProcessor(LLMBase): __embedding__ = 1536 - def __init__(self, bot: Text, llm_type: str): + def __init__(self, bot: Text): super().__init__(bot) self.db_url = Utility.environment['vector']['db'] self.headers = {} if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.suffix = "_faq_embd" - self.llm_type = llm_type self.vector_config = {'size': self.__embedding__, 'distance': 'Cosine'} self.api_key = Sysadmin.get_bot_secret(bot, BotSecretType.gpt_key.value, raise_err=True) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 self.__logs = [] - async def train(self, user, *args, **kwargs) -> Dict: + async def train(self, user, bot, *args, **kwargs) -> Dict: await self.__delete_collections() count = 0 processor = CognitionDataProcessor() @@ -61,25 +59,26 @@ async def train(self, user, *args, **kwargs) -> Dict: content['data'], metadata) else: search_payload, embedding_payload = {'content': content["data"]}, content["data"] - embeddings = await self.get_embedding(embedding_payload, user) + #search_payload['collection_name'] = collection + embeddings = await self.get_embedding(embedding_payload, user, bot) points = [{'id': content['vector_id'], 'vector': embeddings, 'payload': search_payload}] await self.__collection_upsert__(collection, {'points': points}, err_msg="Unable to train FAQ! Contact support") count += 1 return {"faq": count} - async def predict(self, query: Text, user, *args, **kwargs) -> Tuple: + async def predict(self, query: Text, user, bot, *args, **kwargs) -> Tuple: start_time = time.time() embeddings_created = False try: - query_embedding = await self.get_embedding(query, user) + query_embedding = await self.get_embedding(query, user, bot) embeddings_created = True system_prompt = kwargs.pop('system_prompt', DEFAULT_SYSTEM_PROMPT) context_prompt = kwargs.pop('context_prompt', DEFAULT_CONTEXT_PROMPT) context = await self.__attach_similarity_prompt_if_enabled(query_embedding, context_prompt, **kwargs) - answer = await self.__get_answer(query, system_prompt, context, user, **kwargs) + answer = await self.__get_answer(query, system_prompt, context, user, bot, **kwargs) response = {"content": answer} except Exception as e: logging.exception(e) @@ -101,11 +100,11 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def get_embedding(self, text: Text, user) -> List[float]: + async def get_embedding(self, text: Text, user, bot) -> List[float]: truncated_text = self.truncate_text(text) result = await litellm.aembedding(model="text-embedding-3-small", input=[truncated_text], - metadata={'user': user, 'bot': self.bot}, + metadata={'user': user, 'bot': bot}, api_key=self.api_key, num_retries=3) return result["data"][0]["embedding"] @@ -113,25 +112,24 @@ async def get_embedding(self, text: Text, user) -> List[float]: async def __parse_completion_response(self, response, **kwargs): if kwargs.get("stream"): formatted_response = '' - msg_choice = randbelow(kwargs.get("n", 1)) + msg_choice = random.randint(0, kwargs.get("n", 1) - 1) if response["choices"][0].get("index") == msg_choice and response["choices"][0]['delta'].get('content'): formatted_response = f"{response['choices'][0]['delta']['content']}" else: - msg_choice = choice(response['choices']) + msg_choice = random.choice(response['choices']) formatted_response = msg_choice['message']['content'] return formatted_response - async def __get_completion(self, messages, hyperparameters, user, **kwargs): + async def __get_completion(self, messages, hyperparameters, user, bot, **kwargs): response = await litellm.acompletion(messages=messages, - metadata={'user': user, 'bot': self.bot}, + metadata={'user': user, 'bot': bot}, api_key=self.api_key, num_retries=3, **hyperparameters) - formatted_response = await self.__parse_completion_response(response, - **hyperparameters) + formatted_response = await self.__parse_completion_response(response, **kwargs) return formatted_response, response - async def __get_answer(self, query, system_prompt: Text, context: Text, user, **kwargs): + async def __get_answer(self, query, system_prompt: Text, context: Text, user, bot, **kwargs): use_query_prompt = False query_prompt = '' if kwargs.get('query_prompt', {}): @@ -146,7 +144,8 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, ** if use_query_prompt and query_prompt: query = await self.__rephrase_query(query, system_prompt, query_prompt, hyperparameters=hyperparameters, - user=user) + user=user, + bot=bot) messages = [ {"role": "system", "content": system_prompt}, ] @@ -157,12 +156,13 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, ** completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user) + user=user, + bot=bot) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'answer_query', 'hyperparameters': hyperparameters}) return completion - async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, **kwargs): + async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, bot, **kwargs): messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": f"{query_prompt}\n\n Q: {query}\n A:"} @@ -171,7 +171,8 @@ async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user) + user=user, + bot=bot) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'rephrase_query', 'hyperparameters': hyperparameters}) return completion @@ -261,26 +262,3 @@ async def __attach_similarity_prompt_if_enabled(self, query_embedding, context_p similarity_context = f"Instructions on how to use {similarity_prompt_name}:\n{extracted_values}\n{similarity_prompt_instructions}\n" context_prompt = f"{context_prompt}\n{similarity_context}" return context_prompt - - @staticmethod - def get_logs(bot: str, start_idx: int = 0, page_size: int = 10): - """ - Get all logs for data importer event. - @param bot: bot id. - @param start_idx: start index - @param page_size: page size - @return: list of logs. - """ - for log in LLMLogs.objects(metadata__bot=bot).order_by("-start_time").skip(start_idx).limit(page_size): - llm_log = log.to_mongo().to_dict() - llm_log.pop('_id') - yield llm_log - - @staticmethod - def get_row_count(bot: str): - """ - Gets the count of rows in a LLMLogs for a particular bot. - :param bot: bot id - :return: Count of rows - """ - return LLMLogs.objects(metadata__bot=bot).count() diff --git a/kairon/shared/utils.py b/kairon/shared/utils.py index acf3ff92a..2db890379 100644 --- a/kairon/shared/utils.py +++ b/kairon/shared/utils.py @@ -2069,12 +2069,12 @@ def get_llm_hyperparameters(llm_type): @staticmethod def validate_llm_hyperparameters(hyperparameters: dict, llm_type: str, exception_class): - from jsonschema_rs import JSONSchema, ValidationError as JValidationError + from jsonschema_rs import JSONSchema, ValidationError schema = Utility.system_metadata["llm"][llm_type] try: validator = JSONSchema(schema) validator.validate(hyperparameters) - except JValidationError as e: + except ValidationError as e: message = f"{e.instance_path}: {e.message}" raise exception_class(message) diff --git a/kairon/shared/vector_embeddings/db/base.py b/kairon/shared/vector_embeddings/db/base.py index 887be41bb..d1c2a1e97 100644 --- a/kairon/shared/vector_embeddings/db/base.py +++ b/kairon/shared/vector_embeddings/db/base.py @@ -8,16 +8,16 @@ class VectorEmbeddingsDbBase(ABC): @abstractmethod - async def embedding_search(self, request_body: Dict, user: str, **kwargs): + async def embedding_search(self, request_body: Dict, **kwargs): raise NotImplementedError("Provider not implemented") @abstractmethod - async def payload_search(self, request_body: Dict, user: str, **kwargs): + async def payload_search(self, request_body: Dict, **kwargs): raise NotImplementedError("Provider not implemented") - async def perform_operation(self, op_type: Text, request_body: Dict, user: str, **kwargs): + async def perform_operation(self, op_type: Text, request_body: Dict, **kwargs): supported_ops = {DbActionOperationType.payload_search.value: self.payload_search, DbActionOperationType.embedding_search.value: self.embedding_search} if op_type not in supported_ops.keys(): raise AppException("Operation type not supported") - return await supported_ops[op_type](request_body, user, **kwargs) + return await supported_ops[op_type](request_body, **kwargs) diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index 454eeb8fe..893a310ad 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -5,8 +5,10 @@ from kairon import Utility from kairon.shared.actions.utils import ActionUtility +from kairon.shared.admin.constants import BotSecretType +from kairon.shared.admin.processor import Sysadmin +from kairon.shared.constants import GPT3ResourceTypes from kairon.shared.llm.processor import LLMProcessor -from kairon.shared.data.constant import DEFAULT_LLM from kairon.shared.vector_embeddings.db.base import VectorEmbeddingsDbBase @@ -23,19 +25,27 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.llm_settings = llm_settings - self.llm = LLMProcessor(self.bot, DEFAULT_LLM) + self.llm = LLMProcessor(self.bot) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 - async def __get_embedding(self, text: Text, user: str, **kwargs) -> List[float]: - return await self.llm.get_embedding(text, user=user) + def truncate_text(self, text: Text) -> Text: + """ + Truncate text to 8191 tokens for openai + """ + tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] + return self.tokenizer.decode(tokens) - async def embedding_search(self, request_body: Dict, user: str, **kwargs): + async def __get_embedding(self, text: Text, **kwargs) -> List[float]: + result, _ = await self.llm.get_embedding(text, user=kwargs.get('user'), bot=kwargs.get('bot')) + return result + + async def embedding_search(self, request_body: Dict, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points") if request_body.get("text"): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/search") user_msg = request_body.get("text") - vector = await self.__get_embedding(user_msg, user, **kwargs) + vector = await self.__get_embedding(user_msg, **kwargs) request_body = {'vector': vector, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} embedding_search_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', @@ -43,7 +53,7 @@ async def embedding_search(self, request_body: Dict, user: str, **kwargs): request_body=request_body) return embedding_search_result - async def payload_search(self, request_body: Dict, user, **kwargs): + async def payload_search(self, request_body: Dict, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/scroll") payload_filter_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', diff --git a/kairon/train.py b/kairon/train.py index d8360ce67..0276f7bc5 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -6,7 +6,7 @@ from rasa.api import train from rasa.model import DEFAULT_MODELS_PATH from rasa.shared.constants import DEFAULT_CONFIG_PATH, DEFAULT_DATA_PATH, DEFAULT_DOMAIN_PATH -from kairon.shared.data.constant import DEFAULT_LLM + from kairon.chat.agent.agent import KaironAgent from kairon.exceptions import AppException from kairon.shared.account.processor import AccountProcessor @@ -101,8 +101,8 @@ def start_training(bot: str, user: str, token: str = None): settings = processor.get_bot_settings(bot, user) settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: - llm_processor = LLMProcessor(bot, DEFAULT_LLM) - faqs = asyncio.run(llm_processor.train(user=user)) + llm_processor = LLMProcessor(bot) + faqs = asyncio.run(llm_processor.train(user=user, bot=bot)) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) agent_url = Utility.environment['model']['agent'].get('url') diff --git a/metadata/integrations.yml b/metadata/integrations.yml index 6ba78b15f..227b4c413 100644 --- a/metadata/integrations.yml +++ b/metadata/integrations.yml @@ -129,6 +129,10 @@ llm: minimum: 1 maximum: 5 description: "The n hyperparameter controls the number of different response options that are generated by the model." + stream: + type: boolean + default: false + description: "the stream hyperparameter controls whether the model generates a continuous stream of text or a single response." stop: anyOf: - type: "string" diff --git a/requirements/prod.txt b/requirements/prod.txt index 13fcc40e7..e00d448a9 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -64,6 +64,6 @@ opentelemetry-instrumentation-requests==0.46b0 opentelemetry-instrumentation-sklearn==0.46b0 pykwalify==1.8.0 gunicorn==22.0.0 -litellm==1.39.5 +litellm==1.38.11 jsonschema_rs==0.18.0 mongoengine-jsonschema==0.1.3 \ No newline at end of file diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index e71a95494..1941a8fd6 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -3,7 +3,7 @@ from urllib.parse import urlencode, urljoin import litellm -from unittest import mock +import mock import numpy as np import pytest import responses @@ -24,8 +24,7 @@ EmailActionConfig, ActionServerLogs, GoogleSearchAction, JiraAction, ZendeskAction, PipedriveLeadsAction, SetSlots, \ HubspotFormsAction, HttpActionResponse, HttpActionRequestBody, SetSlotsFromResponse, CustomActionRequestParameters, \ KaironTwoStageFallbackAction, TwoStageFallbackTextualRecommendations, RazorpayAction, PromptAction, FormSlotSet, \ - DatabaseAction, DbQuery, PyscriptActionConfig, WebSearchAction, UserQuestion, LiveAgentActionConfig, \ - CustomActionParameters + DatabaseAction, DbQuery, PyscriptActionConfig, WebSearchAction, UserQuestion, LiveAgentActionConfig from kairon.shared.actions.exception import ActionFailure from kairon.shared.actions.models import ActionType, ActionParameterType, DispatchType, DbActionOperationType, \ DbQueryValueType @@ -5564,838 +5563,14 @@ def test_email_action_execution_script_evaluation(mock_smtp, mock_action_config, smtp_url="test.localhost", smtp_port=293, smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), - from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), - to_email=CustomActionParameters(value=["test@test.com"], parameter_type="value"), - subject="test", - response="Email Triggered", - custom_text=CustomActionRequestParameters(key="custom_text", value="custom_mail_text", - parameter_type=ActionParameterType.slot.value), - bot=bot, - user=user - ) - - def _get_action(*arge, **kwargs): - return action.to_mongo().to_dict() - - def _get_action_config(*arge, **kwargs): - return action_config.to_mongo().to_dict() - - request_object = json.load(open("tests/testing_data/actions/action-request.json")) - request_object["tracker"]["slots"]["bot"] = bot - request_object["tracker"]["slots"]["custom_mail_text"] = "The user has id udit.pandey" - request_object["next_action"] = action_name - request_object["tracker"]["sender_id"] = user - request_object["tracker"]["latest_message"]['text'] = 'hello' - - responses.add( - method=responses.POST, - url=Utility.environment['evaluator']['url'], - json={"success": True, "data": "The user has id udit.pandey"}, - status=200 - ) - - mock_action.side_effect = _get_action - mock_action_config.side_effect = _get_action_config - response = client.post("/webhook", json=request_object) - response_json = response.json() - assert response.status_code == 200 - assert len(response_json['events']) == 1 - assert len(response_json['responses']) == 1 - assert response_json['events'] == [ - {"event": "slot", "timestamp": None, "name": "kairon_action_response", - "value": "Email Triggered"}] - assert response_json['responses'][0]['text'] == "Email Triggered" - - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().connect' - assert {} == kwargs - - host, port = args - assert host == action_config.smtp_url - assert port == action_config.smtp_port - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().login' - assert {} == kwargs - - from_email, password = args - assert from_email == action_config.from_email.value - assert password == action_config.smtp_password.value - - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().sendmail' - assert {} == kwargs - - assert args[0] == action_config.from_email.value - assert args[1] == ["test@test.com"] - assert str(args[2]).__contains__(action_config.subject) - assert str(args[2]).__contains__("Content-Type: text/html") - - -@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") -@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@mock.patch("kairon.shared.utils.SMTP", autospec=True) -def test_email_action_execution(mock_smtp, mock_action_config, mock_action): - Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', - 'rb').read().decode() - Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( - 'template/emails/bot_msg_conversation.html', 'rb').read().decode() - Utility.email_conf['email']['templates']['user_msg_conversation'] = open( - 'template/emails/user_msg_conversation.html', 'rb').read().decode() - Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', - 'rb').read().decode() - - action_name = "test_run_email_action" - action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") - action_config = EmailActionConfig( - action_name=action_name, - smtp_url="test.localhost", - smtp_port=293, - smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), - from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), - to_email=CustomActionParameters(value=["test@test.com"], parameter_type="value"), - subject="test", - response="Email Triggered", - bot="bot", - user="user" - ) - - def _get_action(*arge, **kwargs): - return action.to_mongo().to_dict() - - def _get_action_config(*arge, **kwargs): - return action_config.to_mongo().to_dict() - - request_object = { - "next_action": action_name, - "tracker": { - "sender_id": "default", - "conversation_id": "default", - "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, - "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, - "latest_event_time": 1537645578.314389, - "followup_action": "action_listen", - "paused": False, - "events": [ - {"event": "action", "timestamp": 1594907100.12764, "name": "action_session_start", "policy": None, - "confidence": None}, {"event": "session_started", "timestamp": 1594907100.12765}, - {"event": "action", "timestamp": 1594907100.12767, "name": "action_listen", "policy": None, - "confidence": None}, {"event": "user", "timestamp": 1594907100.42744, "text": "can't", - "parse_data": { - "intent": {"name": "test intent", "confidence": 0.253578245639801}, - "entities": [], "intent_ranking": [ - {"name": "test intent", "confidence": 0.253578245639801}, - {"name": "goodbye", "confidence": 0.1504897326231}, - {"name": "greet", "confidence": 0.138640150427818}, - {"name": "affirm", "confidence": 0.0857767835259438}, - {"name": "smalltalk_human", "confidence": 0.0721133947372437}, - {"name": "deny", "confidence": 0.069614589214325}, - {"name": "bot_challenge", "confidence": 0.0664894133806229}, - {"name": "faq_vaccine", "confidence": 0.062177762389183}, - {"name": "faq_testing", "confidence": 0.0530692934989929}, - {"name": "out_of_scope", "confidence": 0.0480506233870983}], - "response_selector": { - "default": {"response": {"name": None, "confidence": 0}, - "ranking": [], "full_retrieval_intent": None}}, - "text": "can't"}, "input_channel": None, - "message_id": "bbd413bf5c834bf3b98e0da2373553b2", "metadata": {}}, - {"event": "action", "timestamp": 1594907100.4308, "name": "utter_test intent", - "policy": "policy_0_MemoizationPolicy", "confidence": 1}, - {"event": "bot", "timestamp": 1594907100.4308, "text": "will not = won\"t", - "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, - "image": None, "custom": None}, "metadata": {}}, - {"event": "action", "timestamp": 1594907100.43384, "name": "action_listen", - "policy": "policy_0_MemoizationPolicy", "confidence": 1}, - {"event": "user", "timestamp": 1594907117.04194, "text": "can\"t", - "parse_data": {"intent": {"name": "test intent", "confidence": 0.253578245639801}, "entities": [], - "intent_ranking": [{"name": "test intent", "confidence": 0.253578245639801}, - {"name": "goodbye", "confidence": 0.1504897326231}, - {"name": "greet", "confidence": 0.138640150427818}, - {"name": "affirm", "confidence": 0.0857767835259438}, - {"name": "smalltalk_human", "confidence": 0.0721133947372437}, - {"name": "deny", "confidence": 0.069614589214325}, - {"name": "bot_challenge", "confidence": 0.0664894133806229}, - {"name": "faq_vaccine", "confidence": 0.062177762389183}, - {"name": "faq_testing", "confidence": 0.0530692934989929}, - {"name": "out_of_scope", "confidence": 0.0480506233870983}], - "response_selector": { - "default": {"response": {"name": None, "confidence": 0}, "ranking": [], - "full_retrieval_intent": None}}, "text": "can\"t"}, - "input_channel": None, "message_id": "e96e2a85de0748798748385503c65fb3", "metadata": {}}, - {"event": "action", "timestamp": 1594907117.04547, "name": "utter_test intent", - "policy": "policy_1_TEDPolicy", "confidence": 0.978452920913696}, - {"event": "bot", "timestamp": 1594907117.04548, "text": "can not = can't", - "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, - "image": None, "custom": None}, "metadata": {}}], - "latest_input_channel": "rest", - "active_loop": {}, - "latest_action": {}, - }, - "domain": { - "config": {}, - "session_config": {}, - "intents": [], - "entities": [], - "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, - "responses": {}, - "actions": [], - "forms": {}, - "e2e_actions": [] - }, - "version": "version" - } - mock_action.side_effect = _get_action - mock_action_config.side_effect = _get_action_config - response = client.post("/webhook", json=request_object) - response_json = response.json() - assert response.status_code == 200 - assert len(response_json['events']) == 1 - assert len(response_json['responses']) == 1 - assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "Email Triggered"}] - assert response_json['responses'][0]['text'] == "Email Triggered" - logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() - assert logs.status == "SUCCESS" - - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().connect' - assert {} == kwargs - - host, port = args - assert host == action_config.smtp_url - assert port == action_config.smtp_port - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().login' - assert {} == kwargs - - from_email, password = args - assert from_email == action_config.from_email.value - assert password == action_config.smtp_password.value - - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().sendmail' - assert {} == kwargs - - assert args[0] == action_config.from_email.value - assert args[1] == ["test@test.com"] - assert str(args[2]).__contains__(action_config.subject) - assert str(args[2]).__contains__("Content-Type: text/html") - assert str(args[2]).__contains__("Subject: default test") - - -@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") -@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@mock.patch("kairon.shared.utils.SMTP", autospec=True) -def test_email_action_execution_with_sender_email_from_slot(mock_smtp, mock_action_config, mock_action): - Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', - 'rb').read().decode() - Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( - 'template/emails/bot_msg_conversation.html', 'rb').read().decode() - Utility.email_conf['email']['templates']['user_msg_conversation'] = open( - 'template/emails/user_msg_conversation.html', 'rb').read().decode() - Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', - 'rb').read().decode() - action_name = "test_email_action_execution_with_sender_email_from_slot" - action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") - action_config = EmailActionConfig( - action_name=action_name, - smtp_url="test.localhost", - smtp_port=293, - smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), - from_email=CustomActionRequestParameters(value="from_email", parameter_type="slot"), - to_email=CustomActionParameters(value=["test@test.com"], parameter_type="value"), - subject="test", - response="Email Triggered", - bot="bot", - user="user" - ) - - def _get_action(*arge, **kwargs): - return action.to_mongo().to_dict() - - def _get_action_config(*arge, **kwargs): - return action_config.to_mongo().to_dict() - - request_object = { - "next_action": action_name, - "tracker": { - "sender_id": "mahesh.sattala", - "conversation_id": "default", - "slots": {"bot": "5f50fd0a56b698ca10d35d2e", "requested_slot": "from_email", - "from_email": "mahesh@gmail.com"}, - "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, - "latest_event_time": 1537645578.314389, - "followup_action": "action_listen", - "paused": False, - "events": [ - {"event": "action", "timestamp": 1594907100.12764, "name": "action_session_start", "policy": None, - "confidence": None}, {"event": "session_started", "timestamp": 1594907100.12765}, - {"event": "action", "timestamp": 1594907100.12767, "name": "action_listen", "policy": None, - "confidence": None}, {"event": "user", "timestamp": 1594907100.42744, "text": "can't", - "parse_data": { - "intent": {"name": "test intent", "confidence": 0.253578245639801}, - "entities": [], "intent_ranking": [ - {"name": "test intent", "confidence": 0.253578245639801}, - {"name": "goodbye", "confidence": 0.1504897326231}, - {"name": "greet", "confidence": 0.138640150427818}, - {"name": "affirm", "confidence": 0.0857767835259438}, - {"name": "smalltalk_human", "confidence": 0.0721133947372437}, - {"name": "deny", "confidence": 0.069614589214325}, - {"name": "bot_challenge", "confidence": 0.0664894133806229}, - {"name": "faq_vaccine", "confidence": 0.062177762389183}, - {"name": "faq_testing", "confidence": 0.0530692934989929}, - {"name": "out_of_scope", "confidence": 0.0480506233870983}], - "response_selector": { - "default": {"response": {"name": None, "confidence": 0}, - "ranking": [], "full_retrieval_intent": None}}, - "text": "can't"}, "input_channel": None, - "message_id": "bbd413bf5c834bf3b98e0da2373553b2", "metadata": {}}, - {"event": "action", "timestamp": 1594907100.4308, "name": "utter_test intent", - "policy": "policy_0_MemoizationPolicy", "confidence": 1}, - {"event": "bot", "timestamp": 1594907100.4308, "text": "will not = won\"t", - "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, - "image": None, "custom": None}, "metadata": {}}, - {"event": "action", "timestamp": 1594907100.43384, "name": "action_listen", - "policy": "policy_0_MemoizationPolicy", "confidence": 1}, - {"event": "user", "timestamp": 1594907117.04194, "text": "can\"t", - "parse_data": {"intent": {"name": "test intent", "confidence": 0.253578245639801}, "entities": [], - "intent_ranking": [{"name": "test intent", "confidence": 0.253578245639801}, - {"name": "goodbye", "confidence": 0.1504897326231}, - {"name": "greet", "confidence": 0.138640150427818}, - {"name": "affirm", "confidence": 0.0857767835259438}, - {"name": "smalltalk_human", "confidence": 0.0721133947372437}, - {"name": "deny", "confidence": 0.069614589214325}, - {"name": "bot_challenge", "confidence": 0.0664894133806229}, - {"name": "faq_vaccine", "confidence": 0.062177762389183}, - {"name": "faq_testing", "confidence": 0.0530692934989929}, - {"name": "out_of_scope", "confidence": 0.0480506233870983}], - "response_selector": { - "default": {"response": {"name": None, "confidence": 0}, "ranking": [], - "full_retrieval_intent": None}}, "text": "can\"t"}, - "input_channel": None, "message_id": "e96e2a85de0748798748385503c65fb3", "metadata": {}}, - {"event": "action", "timestamp": 1594907117.04547, "name": "utter_test intent", - "policy": "policy_1_TEDPolicy", "confidence": 0.978452920913696}, - {"event": "bot", "timestamp": 1594907117.04548, "text": "can not = can't", - "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, - "image": None, "custom": None}, "metadata": {}}], - "latest_input_channel": "rest", - "active_loop": {}, - "latest_action": {}, - }, - "domain": { - "config": {}, - "session_config": {}, - "intents": [], - "entities": [], - "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, - "responses": {}, - "actions": [], - "forms": {}, - "e2e_actions": [] - }, - "version": "version" - } - mock_action.side_effect = _get_action - mock_action_config.side_effect = _get_action_config - response = client.post("/webhook", json=request_object) - response_json = response.json() - assert response.status_code == 200 - assert len(response_json['events']) == 1 - assert len(response_json['responses']) == 1 - assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "Email Triggered"}] - assert response_json['responses'][0]['text'] == "Email Triggered" - logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() - assert logs.status == "SUCCESS" - - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().connect' - assert {} == kwargs - - host, port = args - assert host == action_config.smtp_url - assert port == action_config.smtp_port - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().login' - assert {} == kwargs - - from_email, password = args - assert from_email == 'mahesh@gmail.com' - assert password == action_config.smtp_password.value - - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().sendmail' - assert {} == kwargs - - assert args[0] == 'mahesh@gmail.com' - assert args[1] == ["test@test.com"] - assert str(args[2]).__contains__(action_config.subject) - assert str(args[2]).__contains__("Content-Type: text/html") - assert str(args[2]).__contains__("Subject: mahesh.sattala test") - - -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) -def test_email_action_execution_with_receiver_email_list_from_slot(mock_smtp, mock_action_config, mock_action): - Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', - 'rb').read().decode() - Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( - 'template/emails/bot_msg_conversation.html', 'rb').read().decode() - Utility.email_conf['email']['templates']['user_msg_conversation'] = open( - 'template/emails/user_msg_conversation.html', 'rb').read().decode() - Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', - 'rb').read().decode() - - action_name = "test_email_action_execution_with_receiver_email_list_from_slot" - action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") - action_config = EmailActionConfig( - action_name=action_name, - smtp_url="test.localhost", - smtp_port=293, - smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), - from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), - to_email=CustomActionParameters(value="to_email", parameter_type="slot"), - subject="test", - response="Email Triggered", - bot="bot", - user="user" - ) - - def _get_action(*arge, **kwargs): - return action.to_mongo().to_dict() - - def _get_action_config(*arge, **kwargs): - return action_config.to_mongo().to_dict() - - request_object = { - "next_action": action_name, - "tracker": { - "sender_id": "mahesh.sattala", - "conversation_id": "default", - "slots": {"bot": "5f50fd0a56b698ca10d35d2e", "requested_slot": "to_email", - "to_email": ["test@gmail.com"]}, - "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, - "latest_event_time": 1537645578.314389, - "followup_action": "action_listen", - "paused": False, - "events": [ - {"event": "action", "timestamp": 1594907100.12764, "name": "action_session_start", "policy": None, - "confidence": None}, {"event": "session_started", "timestamp": 1594907100.12765}, - {"event": "action", "timestamp": 1594907100.12767, "name": "action_listen", "policy": None, - "confidence": None}, {"event": "user", "timestamp": 1594907100.42744, "text": "can't", - "parse_data": { - "intent": {"name": "test intent", "confidence": 0.253578245639801}, - "entities": [], "intent_ranking": [ - {"name": "test intent", "confidence": 0.253578245639801}, - {"name": "goodbye", "confidence": 0.1504897326231}, - {"name": "greet", "confidence": 0.138640150427818}, - {"name": "affirm", "confidence": 0.0857767835259438}, - {"name": "smalltalk_human", "confidence": 0.0721133947372437}, - {"name": "deny", "confidence": 0.069614589214325}, - {"name": "bot_challenge", "confidence": 0.0664894133806229}, - {"name": "faq_vaccine", "confidence": 0.062177762389183}, - {"name": "faq_testing", "confidence": 0.0530692934989929}, - {"name": "out_of_scope", "confidence": 0.0480506233870983}], - "response_selector": { - "default": {"response": {"name": None, "confidence": 0}, - "ranking": [], "full_retrieval_intent": None}}, - "text": "can't"}, "input_channel": None, - "message_id": "bbd413bf5c834bf3b98e0da2373553b2", "metadata": {}}, - {"event": "action", "timestamp": 1594907100.4308, "name": "utter_test intent", - "policy": "policy_0_MemoizationPolicy", "confidence": 1}, - {"event": "bot", "timestamp": 1594907100.4308, "text": "will not = won\"t", - "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, - "image": None, "custom": None}, "metadata": {}}, - {"event": "action", "timestamp": 1594907100.43384, "name": "action_listen", - "policy": "policy_0_MemoizationPolicy", "confidence": 1}, - {"event": "user", "timestamp": 1594907117.04194, "text": "can\"t", - "parse_data": {"intent": {"name": "test intent", "confidence": 0.253578245639801}, "entities": [], - "intent_ranking": [{"name": "test intent", "confidence": 0.253578245639801}, - {"name": "goodbye", "confidence": 0.1504897326231}, - {"name": "greet", "confidence": 0.138640150427818}, - {"name": "affirm", "confidence": 0.0857767835259438}, - {"name": "smalltalk_human", "confidence": 0.0721133947372437}, - {"name": "deny", "confidence": 0.069614589214325}, - {"name": "bot_challenge", "confidence": 0.0664894133806229}, - {"name": "faq_vaccine", "confidence": 0.062177762389183}, - {"name": "faq_testing", "confidence": 0.0530692934989929}, - {"name": "out_of_scope", "confidence": 0.0480506233870983}], - "response_selector": { - "default": {"response": {"name": None, "confidence": 0}, "ranking": [], - "full_retrieval_intent": None}}, "text": "can\"t"}, - "input_channel": None, "message_id": "e96e2a85de0748798748385503c65fb3", "metadata": {}}, - {"event": "action", "timestamp": 1594907117.04547, "name": "utter_test intent", - "policy": "policy_1_TEDPolicy", "confidence": 0.978452920913696}, - {"event": "bot", "timestamp": 1594907117.04548, "text": "can not = can't", - "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, - "image": None, "custom": None}, "metadata": {}}], - "latest_input_channel": "rest", - "active_loop": {}, - "latest_action": {}, - }, - "domain": { - "config": {}, - "session_config": {}, - "intents": [], - "entities": [], - "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, - "responses": {}, - "actions": [], - "forms": {}, - "e2e_actions": [] - }, - "version": "version" - } - mock_action.side_effect = _get_action - mock_action_config.side_effect = _get_action_config - response = client.post("/webhook", json=request_object) - response_json = response.json() - assert response.status_code == 200 - assert len(response_json['events']) == 1 - assert len(response_json['responses']) == 1 - assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "Email Triggered"}] - assert response_json['responses'][0]['text'] == "Email Triggered" - logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() - assert logs.status == "SUCCESS" - - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().connect' - assert {} == kwargs - - host, port = args - assert host == action_config.smtp_url - assert port == action_config.smtp_port - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().login' - assert {} == kwargs - - from_email, password = args - assert from_email == action_config.from_email.value - assert password == action_config.smtp_password.value - - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().sendmail' - assert {} == kwargs - - assert args[0] == action_config.from_email.value - assert args[1] == ["test@gmail.com"] - assert str(args[2]).__contains__(action_config.subject) - assert str(args[2]).__contains__("Content-Type: text/html") - assert str(args[2]).__contains__("Subject: mahesh.sattala test") - - -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) -def test_email_action_execution_with_single_receiver_email_from_slot(mock_smtp, mock_action_config, mock_action): - Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', - 'rb').read().decode() - Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( - 'template/emails/bot_msg_conversation.html', 'rb').read().decode() - Utility.email_conf['email']['templates']['user_msg_conversation'] = open( - 'template/emails/user_msg_conversation.html', 'rb').read().decode() - Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', - 'rb').read().decode() - - action_name = "test_email_action_execution_with_single_receiver_email_from_slot" - action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") - action_config = EmailActionConfig( - action_name=action_name, - smtp_url="test.localhost", - smtp_port=293, - smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), - from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), - to_email=CustomActionParameters(value="to_email", parameter_type="slot"), - subject="test", - response="Email Triggered", - bot="bot", - user="user" - ) - - def _get_action(*arge, **kwargs): - return action.to_mongo().to_dict() - - def _get_action_config(*arge, **kwargs): - return action_config.to_mongo().to_dict() - - request_object = { - "next_action": action_name, - "tracker": { - "sender_id": "mahesh.sattala", - "conversation_id": "default", - "slots": {"bot": "5f50fd0a56b698ca10d35d2e", "requested_slot": "to_email", - "to_email": "example@gmail.com"}, - "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, - "latest_event_time": 1537645578.314389, - "followup_action": "action_listen", - "paused": False, - "events": [ - {"event": "action", "timestamp": 1594907100.12764, "name": "action_session_start", "policy": None, - "confidence": None}, {"event": "session_started", "timestamp": 1594907100.12765}, - {"event": "action", "timestamp": 1594907100.12767, "name": "action_listen", "policy": None, - "confidence": None}, {"event": "user", "timestamp": 1594907100.42744, "text": "can't", - "parse_data": { - "intent": {"name": "test intent", "confidence": 0.253578245639801}, - "entities": [], "intent_ranking": [ - {"name": "test intent", "confidence": 0.253578245639801}, - {"name": "goodbye", "confidence": 0.1504897326231}, - {"name": "greet", "confidence": 0.138640150427818}, - {"name": "affirm", "confidence": 0.0857767835259438}, - {"name": "smalltalk_human", "confidence": 0.0721133947372437}, - {"name": "deny", "confidence": 0.069614589214325}, - {"name": "bot_challenge", "confidence": 0.0664894133806229}, - {"name": "faq_vaccine", "confidence": 0.062177762389183}, - {"name": "faq_testing", "confidence": 0.0530692934989929}, - {"name": "out_of_scope", "confidence": 0.0480506233870983}], - "response_selector": { - "default": {"response": {"name": None, "confidence": 0}, - "ranking": [], "full_retrieval_intent": None}}, - "text": "can't"}, "input_channel": None, - "message_id": "bbd413bf5c834bf3b98e0da2373553b2", "metadata": {}}, - {"event": "action", "timestamp": 1594907100.4308, "name": "utter_test intent", - "policy": "policy_0_MemoizationPolicy", "confidence": 1}, - {"event": "bot", "timestamp": 1594907100.4308, "text": "will not = won\"t", - "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, - "image": None, "custom": None}, "metadata": {}}, - {"event": "action", "timestamp": 1594907100.43384, "name": "action_listen", - "policy": "policy_0_MemoizationPolicy", "confidence": 1}, - {"event": "user", "timestamp": 1594907117.04194, "text": "can\"t", - "parse_data": {"intent": {"name": "test intent", "confidence": 0.253578245639801}, "entities": [], - "intent_ranking": [{"name": "test intent", "confidence": 0.253578245639801}, - {"name": "goodbye", "confidence": 0.1504897326231}, - {"name": "greet", "confidence": 0.138640150427818}, - {"name": "affirm", "confidence": 0.0857767835259438}, - {"name": "smalltalk_human", "confidence": 0.0721133947372437}, - {"name": "deny", "confidence": 0.069614589214325}, - {"name": "bot_challenge", "confidence": 0.0664894133806229}, - {"name": "faq_vaccine", "confidence": 0.062177762389183}, - {"name": "faq_testing", "confidence": 0.0530692934989929}, - {"name": "out_of_scope", "confidence": 0.0480506233870983}], - "response_selector": { - "default": {"response": {"name": None, "confidence": 0}, "ranking": [], - "full_retrieval_intent": None}}, "text": "can\"t"}, - "input_channel": None, "message_id": "e96e2a85de0748798748385503c65fb3", "metadata": {}}, - {"event": "action", "timestamp": 1594907117.04547, "name": "utter_test intent", - "policy": "policy_1_TEDPolicy", "confidence": 0.978452920913696}, - {"event": "bot", "timestamp": 1594907117.04548, "text": "can not = can't", - "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, - "image": None, "custom": None}, "metadata": {}}], - "latest_input_channel": "rest", - "active_loop": {}, - "latest_action": {}, - }, - "domain": { - "config": {}, - "session_config": {}, - "intents": [], - "entities": [], - "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, - "responses": {}, - "actions": [], - "forms": {}, - "e2e_actions": [] - }, - "version": "version" - } - mock_action.side_effect = _get_action - mock_action_config.side_effect = _get_action_config - response = client.post("/webhook", json=request_object) - response_json = response.json() - assert response.status_code == 200 - assert len(response_json['events']) == 1 - assert len(response_json['responses']) == 1 - assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "Email Triggered"}] - assert response_json['responses'][0]['text'] == "Email Triggered" - logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() - assert logs.status == "SUCCESS" - - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().connect' - assert {} == kwargs - - host, port = args - assert host == action_config.smtp_url - assert port == action_config.smtp_port - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().login' - assert {} == kwargs - - from_email, password = args - assert from_email == action_config.from_email.value - assert password == action_config.smtp_password.value - - name, args, kwargs = mock_smtp.method_calls.pop(0) - assert name == '().sendmail' - assert {} == kwargs - - assert args[0] == action_config.from_email.value - assert args[1] == ["example@gmail.com"] - assert str(args[2]).__contains__(action_config.subject) - assert str(args[2]).__contains__("Content-Type: text/html") - assert str(args[2]).__contains__("Subject: mahesh.sattala test") - - -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) -def test_email_action_execution_with_invalid_from_email(mock_smtp, mock_action_config, mock_action): - Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', - 'rb').read().decode() - Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( - 'template/emails/bot_msg_conversation.html', 'rb').read().decode() - Utility.email_conf['email']['templates']['user_msg_conversation'] = open( - 'template/emails/user_msg_conversation.html', 'rb').read().decode() - Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', - 'rb').read().decode() - - action_name = "test_email_action_execution_with_invalid_from_email" - action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") - action_config = EmailActionConfig( - action_name=action_name, - smtp_url="test.localhost", - smtp_port=293, - smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), - from_email=CustomActionRequestParameters(value=["test@demo.com"], parameter_type="value"), - to_email=CustomActionParameters(value="to_email", parameter_type="slot"), - subject="test", - response="Email Triggered", - bot="bot", - user="user" - ) - - def _get_action(*arge, **kwargs): - return action.to_mongo().to_dict() - - def _get_action_config(*arge, **kwargs): - return action_config.to_mongo().to_dict() - - request_object = { - "next_action": action_name, - "tracker": { - "sender_id": "mahesh.sattala", - "conversation_id": "default", - "slots": {"bot": "5f50fd0a56b698ca10d35d2e", "requested_slot": "to_email", - "to_email": "example@gmail.com"}, - "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, - "latest_event_time": 1537645578.314389, - "followup_action": "action_listen", - "paused": False, - "events": [ - {"event": "action", "timestamp": 1594907100.12764, "name": "action_session_start", "policy": None, - "confidence": None}, {"event": "session_started", "timestamp": 1594907100.12765}, - {"event": "action", "timestamp": 1594907100.12767, "name": "action_listen", "policy": None, - "confidence": None}, {"event": "user", "timestamp": 1594907100.42744, "text": "can't", - "parse_data": { - "intent": {"name": "test intent", "confidence": 0.253578245639801}, - "entities": [], "intent_ranking": [ - {"name": "test intent", "confidence": 0.253578245639801}, - {"name": "goodbye", "confidence": 0.1504897326231}, - {"name": "greet", "confidence": 0.138640150427818}, - {"name": "affirm", "confidence": 0.0857767835259438}, - {"name": "smalltalk_human", "confidence": 0.0721133947372437}, - {"name": "deny", "confidence": 0.069614589214325}, - {"name": "bot_challenge", "confidence": 0.0664894133806229}, - {"name": "faq_vaccine", "confidence": 0.062177762389183}, - {"name": "faq_testing", "confidence": 0.0530692934989929}, - {"name": "out_of_scope", "confidence": 0.0480506233870983}], - "response_selector": { - "default": {"response": {"name": None, "confidence": 0}, - "ranking": [], "full_retrieval_intent": None}}, - "text": "can't"}, "input_channel": None, - "message_id": "bbd413bf5c834bf3b98e0da2373553b2", "metadata": {}}, - {"event": "action", "timestamp": 1594907100.4308, "name": "utter_test intent", - "policy": "policy_0_MemoizationPolicy", "confidence": 1}, - {"event": "bot", "timestamp": 1594907100.4308, "text": "will not = won\"t", - "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, - "image": None, "custom": None}, "metadata": {}}, - {"event": "action", "timestamp": 1594907100.43384, "name": "action_listen", - "policy": "policy_0_MemoizationPolicy", "confidence": 1}, - {"event": "user", "timestamp": 1594907117.04194, "text": "can\"t", - "parse_data": {"intent": {"name": "test intent", "confidence": 0.253578245639801}, "entities": [], - "intent_ranking": [{"name": "test intent", "confidence": 0.253578245639801}, - {"name": "goodbye", "confidence": 0.1504897326231}, - {"name": "greet", "confidence": 0.138640150427818}, - {"name": "affirm", "confidence": 0.0857767835259438}, - {"name": "smalltalk_human", "confidence": 0.0721133947372437}, - {"name": "deny", "confidence": 0.069614589214325}, - {"name": "bot_challenge", "confidence": 0.0664894133806229}, - {"name": "faq_vaccine", "confidence": 0.062177762389183}, - {"name": "faq_testing", "confidence": 0.0530692934989929}, - {"name": "out_of_scope", "confidence": 0.0480506233870983}], - "response_selector": { - "default": {"response": {"name": None, "confidence": 0}, "ranking": [], - "full_retrieval_intent": None}}, "text": "can\"t"}, - "input_channel": None, "message_id": "e96e2a85de0748798748385503c65fb3", "metadata": {}}, - {"event": "action", "timestamp": 1594907117.04547, "name": "utter_test intent", - "policy": "policy_1_TEDPolicy", "confidence": 0.978452920913696}, - {"event": "bot", "timestamp": 1594907117.04548, "text": "can not = can't", - "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, - "image": None, "custom": None}, "metadata": {}}], - "latest_input_channel": "rest", - "active_loop": {}, - "latest_action": {}, - }, - "domain": { - "config": {}, - "session_config": {}, - "intents": [], - "entities": [], - "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, - "responses": {}, - "actions": [], - "forms": {}, - "e2e_actions": [] - }, - "version": "version" - } - mock_action.side_effect = _get_action - mock_action_config.side_effect = _get_action_config - response = client.post("/webhook", json=request_object) - response_json = response.json() - assert response.status_code == 200 - assert len(response_json['events']) == 1 - assert len(response_json['responses']) == 1 - assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "I have failed to process your request"}] - assert response_json['responses'][0]['text'] == "I have failed to process your request" - logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() - assert logs.status == "FAILURE" - assert logs.exception == "Invalid 'from_email' type. It must be of type str." - - -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) -def test_email_action_execution_with_invalid_to_email(mock_smtp, mock_action_config, mock_action): - Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', - 'rb').read().decode() - Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( - 'template/emails/bot_msg_conversation.html', 'rb').read().decode() - Utility.email_conf['email']['templates']['user_msg_conversation'] = open( - 'template/emails/user_msg_conversation.html', 'rb').read().decode() - Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', - 'rb').read().decode() - - action_name = "test_email_action_execution_with_invalid_to_email" - action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") - action_config = EmailActionConfig( - action_name=action_name, - smtp_url="test.localhost", - smtp_port=293, - smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), - from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), - to_email=CustomActionParameters(value={"to_email": "test@test.com"}, parameter_type="value"), - subject="test", + from_email="test@demo.com", + subject="test script", + to_email=["test@test.com"], response="Email Triggered", - bot="bot", - user="user" + custom_text=CustomActionRequestParameters(key="custom_text", value="custom_mail_text", + parameter_type=ActionParameterType.slot.value), + bot=bot, + user=user ) def _get_action(*arge, **kwargs): @@ -6404,85 +5579,20 @@ def _get_action(*arge, **kwargs): def _get_action_config(*arge, **kwargs): return action_config.to_mongo().to_dict() - request_object = { - "next_action": action_name, - "tracker": { - "sender_id": "mahesh.sattala", - "conversation_id": "default", - "slots": {"bot": "5f50fd0a56b698ca10d35d2e", "requested_slot": "to_email", - "to_email": "example@gmail.com"}, - "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, - "latest_event_time": 1537645578.314389, - "followup_action": "action_listen", - "paused": False, - "events": [ - {"event": "action", "timestamp": 1594907100.12764, "name": "action_session_start", "policy": None, - "confidence": None}, {"event": "session_started", "timestamp": 1594907100.12765}, - {"event": "action", "timestamp": 1594907100.12767, "name": "action_listen", "policy": None, - "confidence": None}, {"event": "user", "timestamp": 1594907100.42744, "text": "can't", - "parse_data": { - "intent": {"name": "test intent", "confidence": 0.253578245639801}, - "entities": [], "intent_ranking": [ - {"name": "test intent", "confidence": 0.253578245639801}, - {"name": "goodbye", "confidence": 0.1504897326231}, - {"name": "greet", "confidence": 0.138640150427818}, - {"name": "affirm", "confidence": 0.0857767835259438}, - {"name": "smalltalk_human", "confidence": 0.0721133947372437}, - {"name": "deny", "confidence": 0.069614589214325}, - {"name": "bot_challenge", "confidence": 0.0664894133806229}, - {"name": "faq_vaccine", "confidence": 0.062177762389183}, - {"name": "faq_testing", "confidence": 0.0530692934989929}, - {"name": "out_of_scope", "confidence": 0.0480506233870983}], - "response_selector": { - "default": {"response": {"name": None, "confidence": 0}, - "ranking": [], "full_retrieval_intent": None}}, - "text": "can't"}, "input_channel": None, - "message_id": "bbd413bf5c834bf3b98e0da2373553b2", "metadata": {}}, - {"event": "action", "timestamp": 1594907100.4308, "name": "utter_test intent", - "policy": "policy_0_MemoizationPolicy", "confidence": 1}, - {"event": "bot", "timestamp": 1594907100.4308, "text": "will not = won\"t", - "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, - "image": None, "custom": None}, "metadata": {}}, - {"event": "action", "timestamp": 1594907100.43384, "name": "action_listen", - "policy": "policy_0_MemoizationPolicy", "confidence": 1}, - {"event": "user", "timestamp": 1594907117.04194, "text": "can\"t", - "parse_data": {"intent": {"name": "test intent", "confidence": 0.253578245639801}, "entities": [], - "intent_ranking": [{"name": "test intent", "confidence": 0.253578245639801}, - {"name": "goodbye", "confidence": 0.1504897326231}, - {"name": "greet", "confidence": 0.138640150427818}, - {"name": "affirm", "confidence": 0.0857767835259438}, - {"name": "smalltalk_human", "confidence": 0.0721133947372437}, - {"name": "deny", "confidence": 0.069614589214325}, - {"name": "bot_challenge", "confidence": 0.0664894133806229}, - {"name": "faq_vaccine", "confidence": 0.062177762389183}, - {"name": "faq_testing", "confidence": 0.0530692934989929}, - {"name": "out_of_scope", "confidence": 0.0480506233870983}], - "response_selector": { - "default": {"response": {"name": None, "confidence": 0}, "ranking": [], - "full_retrieval_intent": None}}, "text": "can\"t"}, - "input_channel": None, "message_id": "e96e2a85de0748798748385503c65fb3", "metadata": {}}, - {"event": "action", "timestamp": 1594907117.04547, "name": "utter_test intent", - "policy": "policy_1_TEDPolicy", "confidence": 0.978452920913696}, - {"event": "bot", "timestamp": 1594907117.04548, "text": "can not = can't", - "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, - "image": None, "custom": None}, "metadata": {}}], - "latest_input_channel": "rest", - "active_loop": {}, - "latest_action": {}, - }, - "domain": { - "config": {}, - "session_config": {}, - "intents": [], - "entities": [], - "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, - "responses": {}, - "actions": [], - "forms": {}, - "e2e_actions": [] - }, - "version": "version" - } + request_object = json.load(open("tests/testing_data/actions/action-request.json")) + request_object["tracker"]["slots"]["bot"] = bot + request_object["tracker"]["slots"]["custom_mail_text"] = "The user has id udit.pandey" + request_object["next_action"] = action_name + request_object["tracker"]["sender_id"] = user + request_object["tracker"]["latest_message"]['text'] = 'hello' + + responses.add( + method=responses.POST, + url=Utility.environment['evaluator']['url'], + json={"success": True, "data": "The user has id udit.pandey"}, + status=200 + ) + mock_action.side_effect = _get_action mock_action_config.side_effect = _get_action_config response = client.post("/webhook", json=request_object) @@ -6491,18 +5601,39 @@ def _get_action_config(*arge, **kwargs): assert len(response_json['events']) == 1 assert len(response_json['responses']) == 1 assert response_json['events'] == [ - {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "I have failed to process your request"}] - assert response_json['responses'][0]['text'] == "I have failed to process your request" - logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() - assert logs.status == "FAILURE" - assert logs.exception == "Invalid 'from_email' type. It must be of type str." + {"event": "slot", "timestamp": None, "name": "kairon_action_response", + "value": "Email Triggered"}] + assert response_json['responses'][0]['text'] == "Email Triggered" + + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().connect' + assert {} == kwargs + + host, port = args + assert host == action_config.smtp_url + assert port == action_config.smtp_port + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().login' + assert {} == kwargs + + from_email, password = args + assert from_email == action_config.from_email + assert password == action_config.smtp_password.value + + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().sendmail' + assert {} == kwargs + + assert args[0] == action_config.from_email + assert args[1] == ["test@test.com"] + assert str(args[2]).__contains__(action_config.subject) + assert str(args[2]).__contains__("Content-Type: text/html") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) -def test_email_action_execution_with_invalid_to_email(mock_smtp, mock_action_config, mock_action): +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) +def test_email_action_execution(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( @@ -6512,16 +5643,16 @@ def test_email_action_execution_with_invalid_to_email(mock_smtp, mock_action_con Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', 'rb').read().decode() - action_name = "test_email_action_execution_with_invalid_to_email" + action_name = "test_run_email_action" action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") action_config = EmailActionConfig( action_name=action_name, smtp_url="test.localhost", smtp_port=293, smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), - from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), - to_email=CustomActionParameters(value={"to_email": "test@test.com"}, parameter_type="value"), + from_email="test@demo.com", subject="test", + to_email=["test@test.com"], response="Email Triggered", bot="bot", user="user" @@ -6536,10 +5667,9 @@ def _get_action_config(*arge, **kwargs): request_object = { "next_action": action_name, "tracker": { - "sender_id": "mahesh.sattala", + "sender_id": "default", "conversation_id": "default", - "slots": {"bot": "5f50fd0a56b698ca10d35d2e", "requested_slot": "to_email", - "to_email": "example@gmail.com"}, + "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, "latest_event_time": 1537645578.314389, "followup_action": "action_listen", @@ -6621,17 +5751,40 @@ def _get_action_config(*arge, **kwargs): assert len(response_json['responses']) == 1 assert response_json['events'] == [ {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', - 'value': "I have failed to process your request"}] - assert response_json['responses'][0]['text'] == "I have failed to process your request" + 'value': "Email Triggered"}] + assert response_json['responses'][0]['text'] == "Email Triggered" logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() - print(logs.to_mongo().to_dict()) - assert logs.status == "FAILURE" - assert logs.exception == "Invalid 'to_email' type. It must be of type str or list." + assert logs.status == "SUCCESS" + + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().connect' + assert {} == kwargs + + host, port = args + assert host == action_config.smtp_url + assert port == action_config.smtp_port + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().login' + assert {} == kwargs + + from_email, password = args + assert from_email == action_config.from_email + assert password == action_config.smtp_password.value + + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().sendmail' + assert {} == kwargs + + assert args[0] == action_config.from_email + assert args[1] == ["test@test.com"] + assert str(args[2]).__contains__(action_config.subject) + assert str(args[2]).__contains__("Content-Type: text/html") + assert str(args[2]).__contains__("Subject: default test") -@patch("kairon.shared.actions.utils.ActionUtility.get_action") -@patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") -@patch("kairon.shared.utils.SMTP", autospec=True) +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) def test_email_action_execution_varied_utterances(mock_smtp, mock_action_config, mock_action): Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', 'rb').read().decode() @@ -6649,9 +5802,9 @@ def test_email_action_execution_varied_utterances(mock_smtp, mock_action_config, smtp_url="test.localhost", smtp_port=293, smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), - from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), - to_email=CustomActionParameters(value=["test@test.com"], parameter_type="value"), + from_email="test@demo.com", subject="test", + to_email=["test@test.com"], response="Email Triggered", bot="bot", user="user" @@ -6973,14 +6126,14 @@ def _get_action_config(*arge, **kwargs): assert {} == kwargs from_email, password = args - assert from_email == action_config.from_email.value + assert from_email == action_config.from_email assert password == action_config.smtp_password.value name, args, kwargs = mock_smtp.method_calls.pop(0) assert name == '().sendmail' assert {} == kwargs - assert args[0] == action_config.from_email.value + assert args[0] == action_config.from_email assert args[1] == ["test@test.com"] assert str(args[2]).__contains__(action_config.subject) assert str(args[2]).__contains__("Content-Type: text/html") @@ -7095,9 +6248,9 @@ def test_email_action_failed_execution(mock_action_config, mock_action): smtp_url="test.localhost", smtp_port=293, smtp_password=CustomActionRequestParameters(value="test"), - from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), - to_email=CustomActionParameters(value="test@test.com", parameter_type="value"), + from_email="test@demo.com", subject="test", + to_email="test@test.com", response="Email Triggered", bot="bot", user="user" @@ -11344,7 +10497,7 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11411,7 +10564,7 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11481,7 +10634,7 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11706,7 +10859,7 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.return_value = {'choices': [{'delta': {'role': 'assistant', 'content': generated_text}, 'finish_reason': None, 'index': 0}]} + mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -11722,7 +10875,6 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m response = client.post("/webhook", json=request_object) response_json = response.json() - print(response_json['events']) assert response_json['events'] == [ {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': generated_text}] assert response_json['responses'] == [ @@ -12009,14 +11161,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12093,7 +11245,7 @@ def mock_completion_for_answer(*args, **kwargs): 'content': 'Kanban is a workflow management tool which visualizes both the process (the workflow) and the actual work passing through that process.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) @@ -12101,7 +11253,7 @@ def mock_completion_for_answer(*args, **kwargs): 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12218,14 +11370,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12325,14 +11477,14 @@ def mock_completion_for_answer(*args, **kwargs): 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12423,14 +11575,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12494,7 +11646,7 @@ def __mock_fetch_similar(*args, **kwargs): 'content': "\nInstructions on how to use Similarity Prompt:\n['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Kanban And Scrum Together? \nA:"}], 'metadata': {'user': user, 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12558,7 +11710,7 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], 'metadata': {'user': 'udit.pandey', 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -12625,5 +11777,5 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], 'metadata': {'user': user, 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 3c93e1044..b829acf05 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -14,7 +14,7 @@ Utility.load_system_metadata() -from unittest.mock import patch +from mock import patch import numpy as np import pandas as pd import pytest @@ -60,7 +60,7 @@ from kairon.shared.data.constant import UTTERANCE_TYPE, EVENT_STATUS, STORY_EVENT, ALLOWED_DOMAIN_FORMATS, \ ALLOWED_CONFIG_FORMATS, ALLOWED_NLU_FORMATS, ALLOWED_STORIES_FORMATS, ALLOWED_RULES_FORMATS, REQUIREMENTS, \ DEFAULT_NLU_FALLBACK_RULE, SLOT_TYPE, KAIRON_TWO_STAGE_FALLBACK, AuditlogActions, TOKEN_TYPE, GPT_LLM_FAQ, \ - DEFAULT_CONTEXT_PROMPT, DEFAULT_NLU_FALLBACK_RESPONSE, DEFAULT_SYSTEM_PROMPT, DEFAULT_LLM + DEFAULT_CONTEXT_PROMPT, DEFAULT_NLU_FALLBACK_RESPONSE, DEFAULT_SYSTEM_PROMPT from kairon.shared.data.data_objects import (TrainingExamples, Slots, Entities, EntitySynonyms, RegexFeatures, @@ -72,8 +72,7 @@ Utterances, BotSettings, ChatClientConfig, LookupTables, Forms, SlotMapping, KeyVault, MultiflowStories, LLMSettings, MultiflowStoryEvents, Synonyms, - Lookup, - DemoRequestLogs + Lookup ) from kairon.shared.data.history_log_processor import HistoryDeletionLogProcessor from kairon.shared.data.model_processor import ModelProcessor @@ -81,7 +80,6 @@ from kairon.shared.data.training_data_generation_processor import TrainingDataGenerationProcessor from kairon.shared.data.utils import DataUtility from kairon.shared.importer.processor import DataImporterLogProcessor -from kairon.shared.live_agent.live_agent import LiveAgentHandler from kairon.shared.metering.constants import MetricType from kairon.shared.metering.data_object import Metering from kairon.shared.models import StoryEventType, HttpContentType, CognitionDataType @@ -153,50 +151,6 @@ def test_add_complex_story_with_slot(self): {'name': 'persona', 'type': 'SLOT', 'value': 'positive'}, {'name': 'utter_welcome_user', 'type': 'BOT'}] - def test_add_demo_request_with_empty_first_name(self): - processor = MongoProcessor() - processor.add_demo_request( - first_name="", last_name="Sattala", email="mahesh.sattala@digite.com", phone="+919876543210", - message="This is test message", recaptcha_response="Svw2mPVxM0SkO4_2yxTcDQQ7iKNUDeDhGf4l6C2i" - ) - - def test_add_demo_request_with_empty_last_name(self): - processor = MongoProcessor() - processor.add_demo_request( - first_name="Mahesh", last_name="", email="mahesh.sattala@digite.com", phone="+919876543210", - message="This is test message", recaptcha_response="Svw2mPVxM0SkO4_2yxTcDQQ7iKNUDeDhGf4l6C2i" - ) - - def test_add_demo_request_with_invalid_email(self): - processor = MongoProcessor() - processor.add_demo_request( - first_name="Mahesh", last_name="Sattala", email="mahesh.sattala", phone="+919876543210", - message="This is test message", recaptcha_response="Svw2mPVxM0SkO4_2yxTcDQQ7iKNUDeDhGf4l6C2i" - ) - - def test_add_demo_request_with_invalid_status(self): - processor = MongoProcessor() - processor.add_demo_request( - first_name="Mahesh", last_name="Sattala", email="mahesh.sattala@digite.com", - phone="+919876543210", message="This is test message", status="Invalid_status", - recaptcha_response="Svw2mPVxM0SkO4_2yxTcDQQ7iKNUDeDhGf4l6C2i" - ) - - def test_add_demo_request(self): - processor = MongoProcessor() - processor.add_demo_request(first_name="Mahesh", last_name="Sattala", email="mahesh.sattala@nimblework.com", - phone="+919876543210", message="This is test message", status="demo_given", - recaptcha_response="Svw2mPVxM0SkO4_2yxTcDQQ7iKNUDeDhGf4l6C2i") - demo_request_logs = DemoRequestLogs.objects(first_name="Mahesh", last_name="Sattala", - email="mahesh.sattala@nimblework.com").get().to_mongo().to_dict() - assert demo_request_logs['first_name'] == "Mahesh" - assert demo_request_logs['last_name'] == "Sattala" - assert demo_request_logs['email'] == "mahesh.sattala@nimblework.com" - assert demo_request_logs['phone'] == "+919876543210" - assert demo_request_logs['status'] == "demo_given" - assert demo_request_logs['message'] == "This is test message" - assert demo_request_logs['recaptcha_response'] == "Svw2mPVxM0SkO4_2yxTcDQQ7iKNUDeDhGf4l6C2i" - def test_add_prompt_action_with_gpt_feature_disabled(self): processor = MongoProcessor() bot = 'test' @@ -216,7 +170,7 @@ def test_add_prompt_action_with_invalid_slots(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', @@ -244,7 +198,7 @@ def test_add_prompt_action_with_invalid_http_action(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', @@ -273,7 +227,7 @@ def test_add_prompt_action_with_invalid_similarity_threshold(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -303,7 +257,7 @@ def test_add_prompt_action_with_invalid_top_results(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -355,7 +309,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', @@ -400,7 +354,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', @@ -600,7 +554,7 @@ def test_add_prompt_action_with_empty_llm_prompts(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': []} with pytest.raises(ValidationError, match="llm_prompts are required!"): @@ -627,7 +581,7 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -648,7 +602,7 @@ def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -665,7 +619,7 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': ["\n", ".", "?", "!", ";"], + 'n': 1, 'stream': False, 'stop': ["\n", ".", "?", "!", ";"], 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -685,7 +639,7 @@ def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': '?', 'presence_penalty': -3.0, + 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': -3.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -703,7 +657,7 @@ def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 3.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -720,7 +674,7 @@ def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -737,7 +691,7 @@ def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -754,7 +708,7 @@ def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt-3.5-turbo', 'top_p': 3.0, - 'n': 1, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -771,7 +725,7 @@ def test_add_prompt_action_with_invalid_n_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 7, 'stop': '?', 'presence_penalty': 0.0, + 'n': 7, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -788,7 +742,7 @@ def test_add_prompt_action_with_zero_n_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 0, 'stop': '?', 'presence_penalty': 0.0, + 'n': 0, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -805,7 +759,7 @@ def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 2, 'stop': '?', 'presence_penalty': 0.0, + 'n': 2, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': 'a'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -880,7 +834,7 @@ def test_edit_prompt_action_faq_action(self): assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -915,7 +869,7 @@ def test_edit_prompt_action_faq_action(self): 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -958,7 +912,7 @@ def test_edit_prompt_action_with_less_hyperparameters(self): 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -993,7 +947,7 @@ def test_get_prompt_faq_action(self): 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -6538,7 +6492,6 @@ def _mock_bot_info(*args, **kwargs): actual_config.config.pop('live_agent_socket_url') headers = actual_config.config.pop('headers') expected_config['multilingual'] = {'enable': False, 'bots': []} - expected_config['live_agent_enabled'] = True assert expected_config == actual_config.config primary_token_claims = Utility.decode_limited_access_token(headers['authorization']['access_token']) @@ -6557,22 +6510,6 @@ def _mock_bot_info(*args, **kwargs): '/api/bot/.+/metric/user/logs/user_metrics' ], 'access-limit': ['/api/auth/.+/token/refresh']} - def test_get_chat_client_config_live_agent_enabled_false(self, monkeypatch): - def _mock_bot_info(*args, **kwargs): - return { - "_id": "9876543210", 'name': 'test_bot', 'account': 2, 'user': 'user@integration.com', - 'status': True, - "metadata": {"source_bot_id": None} - } - def _mock_is_live_agent_service_available(*args, **kwargs): - return False - monkeypatch.setattr(AccountProcessor, 'get_bot', _mock_bot_info) - monkeypatch.setattr(LiveAgentHandler, 'is_live_agent_service_available', _mock_is_live_agent_service_available) - processor = MongoProcessor() - actual_config = processor.get_chat_client_config('test_bot', 'user@integration.com') - assert actual_config.config['live_agent_enabled'] == False - - def test_save_chat_client_config_without_whitelisted_domain(self, monkeypatch): def _mock_bot_info(*args, **kwargs): return {'name': 'test', 'account': 1, 'user': 'user@integration.com', 'status': True} @@ -8551,9 +8488,7 @@ def test_delete_action_with_attached_http_action(self): 'data': 'tester_action', 'instructions': 'Answer according to the context', 'type': 'user', 'source': 'action', - 'is_enabled': True}], - llm_type=DEFAULT_LLM, - hyperparameters=Utility.get_default_llm_hyperparameters() + 'is_enabled': True}] ) processor.add_http_action_config(http_action_config.dict(), user, bot) processor.add_prompt_action(prompt_action_config.dict(), bot, user) @@ -13262,8 +13197,8 @@ def test_add_email_action(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": {"value": "from_email", "parameter_type": "slot"}, - "to_email": {"value": ["test@test.com", "test1@test.com"], "parameter_type": "value"}, + "from_email": "test@demo.com", + "to_email": ["test@test.com", "test1@test.com"], "subject": "Test Subject", "response": "Test Response", "tls": False @@ -13278,8 +13213,8 @@ def test_add_email_action_with_custom_text(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": {"value": "from_email", "parameter_type": "slot"}, - "to_email": {"value": ["test@test.com", "test1@test.com"], "parameter_type": "value"}, + "from_email": "test@demo.com", + "to_email": ["test@test.com", "test1@test.com"], "subject": "Test Subject", "response": "Test Response", "tls": False, @@ -13316,8 +13251,8 @@ def test_add_email_action_validation_error(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": {"value": "from_email", "parameter_type": "slot"}, - "to_email": {"value": ["test@test.com", "test1@test.com"], "parameter_type": "value"}, + "from_email": "test@demo.com", + "to_email": "test@test.com", "subject": "Test Subject", "response": "Test Response", "tls": False @@ -13341,29 +13276,16 @@ def test_add_email_action_validation_error(self): email_config['smtp_url'] = temp temp = email_config['from_email'] - email_config['from_email'] = {"value": "test@test", "parameter_type": "value"} + email_config['from_email'] = "test@test" with pytest.raises(ValidationError, match="Invalid From or To email address"): processor.add_email_action(email_config, "TEST", "tests") - - email_config['from_email'] = {"value": "", "parameter_type": "slot"} - with pytest.raises(ValidationError, match="Provide name of the slot as value"): - processor.add_email_action(email_config, "TEST", "tests") email_config['from_email'] = temp temp = email_config['to_email'] - email_config['to_email'] = {"value": "test@test", "parameter_type": "value"} - with pytest.raises(ValidationError, match="Provide list of emails as value"): - processor.add_email_action(email_config, "TEST", "tests") - - email_config['to_email'] = {"value": ["test@test"], "parameter_type": "value"} + email_config['to_email'] = "test@test" with pytest.raises(ValidationError, match="Invalid From or To email address"): processor.add_email_action(email_config, "TEST", "tests") - - email_config['to_email'] = {"value": "", "parameter_type": "slot"} - with pytest.raises(ValidationError, match="Provide name of the slot as value"): - processor.add_email_action(email_config, "TEST", "tests") - email_config['to_email'] = temp - + email_config['to_email'] = ["test@demo.com"] email_config["custom_text"] = {"value": "custom_text_slot", "parameter_type": "sender_id"} with pytest.raises(ValidationError, match="custom_text can only be of type value or slot!"): processor.add_email_action(email_config, "TEST", "tests") @@ -13375,8 +13297,8 @@ def test_add_email_action_duplicate(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": {"value": "from_email", "parameter_type": "slot"}, - "to_email": {"value": ["test@test.com", "test1@test.com"], "parameter_type": "value"}, + "from_email": "test@demo.com", + "to_email": ["test@test.com"], "subject": "Test Subject", "response": "Test Response", "tls": False @@ -13392,8 +13314,8 @@ def test_add_email_action_existing_name(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": {"value": "test@demo.com", "parameter_type": "value"}, - "to_email": {"value": "to_email", "parameter_type": "slot"}, + "from_email": "test@demo.com", + "to_email": ["test@test.com"], "subject": "Test Subject", "response": "Test Response", "tls": False @@ -13409,8 +13331,8 @@ def test_edit_email_action(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": {"value": "test@demo.com", "parameter_type": "value"}, - "to_email": {"value": "to_email", "parameter_type": "slot"}, + "from_email": "test@demo.com", + "to_email": ["test@test.com", "test1@test.com"], "subject": "Test Subject", "response": "Test Response", "tls": False @@ -13429,8 +13351,8 @@ def test_edit_email_action_validation_error(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": {"value": "test@demo.com", "parameter_type": "value"}, - "to_email": {"value": "to_email", "parameter_type": "slot"}, + "from_email": "test@demo.com", + "to_email": "test@test.com", "subject": "Test Subject", "response": "Test Response", "tls": False @@ -13454,27 +13376,15 @@ def test_edit_email_action_validation_error(self): email_config['smtp_url'] = temp temp = email_config['from_email'] - email_config['from_email'] = {"value": "test@demo", "parameter_type": "value"} + email_config['from_email'] = "test@test" with pytest.raises(ValidationError, match="Invalid From or To email address"): processor.edit_email_action(email_config, "TEST", "tests") - - email_config['from_email'] = {"value": "", "parameter_type": "slot"} - with pytest.raises(ValidationError, match="Provide name of the slot as value"): - processor.edit_email_action(email_config, "TEST", "tests") email_config['from_email'] = temp temp = email_config['to_email'] - email_config['to_email'] = {"value": "test@test", "parameter_type": "value"} - with pytest.raises(ValidationError, match="Provide list of emails as value"): - processor.edit_email_action(email_config, "TEST", "tests") - - email_config['to_email'] = {"value": ["test@test"], "parameter_type": "value"} + email_config['to_email'] = "test@test" with pytest.raises(ValidationError, match="Invalid From or To email address"): processor.edit_email_action(email_config, "TEST", "tests") - - email_config['to_email'] = {"value": "", "parameter_type": "slot"} - with pytest.raises(ValidationError, match="Provide name of the slot as value"): - processor.edit_email_action(email_config, "TEST", "tests") email_config['to_email'] = temp def test_edit_email_action_does_not_exist(self): @@ -13484,8 +13394,8 @@ def test_edit_email_action_does_not_exist(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": {"value": "test@demo.com", "parameter_type": "value"}, - "to_email": {"value": "to_email", "parameter_type": "slot"}, + "from_email": "test@demo.com", + "to_email": "test@test.com", "subject": "Test Subject", "response": "Test Response", "tls": False @@ -14339,8 +14249,8 @@ def test_delete_secret_attached_to_email_action(self): "smtp_port": 25, "smtp_userid": smtp_userid_list, "smtp_password": {'value': "test"}, - "from_email": {"value": "test@demo.com", "parameter_type": "value"}, - "to_email": {"value": "to_email", "parameter_type": "slot"}, + "from_email": "test@demo.com", + "to_email": ["test@test.com", "test1@test.com"], "subject": "Test Subject", "response": "Test Response", "tls": False diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 53c31bace..c8f727db3 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1,22 +1,20 @@ import os -from unittest import mock from urllib.parse import urljoin +import mock import numpy as np import pytest import ujson as json from aiohttp import ClientConnectionError from mongoengine import connect - from kairon.shared.utils import Utility - Utility.load_system_metadata() from kairon.exceptions import AppException from kairon.shared.admin.constants import BotSecretType from kairon.shared.admin.data_objects import BotSecrets from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema -from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_LLM +from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT from kairon.shared.llm.processor import LLMProcessor import litellm from deepdiff import DeepDiff @@ -44,7 +42,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -68,7 +66,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user) + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, @@ -121,16 +119,14 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() embedding = list(np.random.random(LLMProcessor.__embedding__)) - mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, { - 'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(bot, DEFAULT_LLM) + mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]} + gpt3 = LLMProcessor(bot) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), method="GET", - payload={"time": 0, "status": "ok", - "result": {"collections": [{"name": "test_embed_faq_text_swift_faq_embd"}, - {"name": "example_bot_swift_faq_embd"}]}} + payload={"time": 0, "status": "ok", "result": {"collections": [{"name": "test_embed_faq_text_swift_faq_embd"}, + {"name": "example_bot_swift_faq_embd"}]}} ) aioresponses.add( @@ -145,22 +141,19 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/{gpt3.bot}_user_details{gpt3.suffix}/points"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_user_details{gpt3.suffix}/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/{gpt3.bot}_country_details{gpt3.suffix}"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_country_details{gpt3.suffix}"), method="PUT", status=200 ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/{gpt3.bot}_country_details{gpt3.suffix}/points"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_country_details{gpt3.suffix}/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) @@ -172,29 +165,24 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user) + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 3 - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { - 'name': f"{gpt3.bot}_country_details{gpt3.suffix}", - 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'name': f"{gpt3.bot}_country_details{gpt3.suffix}", + 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { - 'points': [{'id': test_content_two.vector_id, - 'vector': embedding, - 'payload': {'country': 'Spain'}}]} - assert list(aioresponses.requests.values())[3][1].kwargs['json'] == { - 'points': [{'id': test_content_three.vector_id, - 'vector': embedding, - 'payload': {'role': 'ds'}}]} + assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content_two.vector_id, + 'vector': embedding, + 'payload': {'country': 'Spain'}}]} + assert list(aioresponses.requests.values())[3][1].kwargs['json'] == {'points': [{'id': test_content_three.vector_id, + 'vector': embedding, + 'payload': {'role': 'ds'}}]} - assert list(aioresponses.requests.values())[4][0].kwargs['json'] == { - 'name': f"{gpt3.bot}_user_details{gpt3.suffix}", - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[5][0].kwargs['json'] == { - 'points': [{'id': test_content.vector_id, - 'vector': embedding, - 'payload': {'name': 'Nupur'}}]} + assert list(aioresponses.requests.values())[4][0].kwargs['json'] == {'name': f"{gpt3.bot}_user_details{gpt3.suffix}", + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[5][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + 'vector': embedding, + 'payload': {'name': 'Nupur'}}]} assert response['faq'] == 3 expected = {"model": "text-embedding-3-small", @@ -227,10 +215,9 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a input = {"name": "Ram", "color": "red"} mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(bot, DEFAULT_LLM) + gpt3 = LLMProcessor(bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), method="PUT", status=200 ) @@ -240,21 +227,18 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a payload={"time": 0, "status": "ok", "result": {"collections": []}}) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/test_embed_faq_json_payload_with_int_faq_embd/points"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - response = await gpt3.train(user=user) + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 1 - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == { - 'name': 'test_embed_faq_json_payload_with_int_faq_embd', - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { - 'points': [{'id': test_content.vector_id, + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_embed_faq_json_payload_with_int_faq_embd', + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, 'vector': embedding, 'payload': {'name': 'Ram', 'age': 23, 'color': 'red'} }]} @@ -288,7 +272,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): input = {"name": "Ram", "color": "red"} mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(bot, DEFAULT_LLM) + gpt3 = LLMProcessor(bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -318,7 +302,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user) + response = await gpt3.train(user=user, bot=bot) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_int_embd_int_faq_embd', @@ -339,7 +323,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): def test_gpt3_faq_embedding_train_failure(self): with pytest.raises(AppException, match=f"Bot secret '{BotSecretType.gpt_key.value}' not configured!"): - LLMProcessor('test_failure', DEFAULT_LLM) + LLMProcessor('test_failure') @pytest.mark.asyncio @mock.patch.object(litellm, "aembedding", autospec=True) @@ -357,7 +341,7 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aiore mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -379,18 +363,16 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aiore url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}{gpt3.suffix}/points"), method="PUT", payload={"result": None, - 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, - "time": 0.003612634} + 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, + "time": 0.003612634} ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train(user=user) + await gpt3.train(user=user, bot=bot) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { - 'points': [{'id': test_content.vector_id, - 'vector': embedding, 'payload': {'content': test_content.data}}]} + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + 'vector': embedding, 'payload': {'content': test_content.data}}]} expected = {"model": "text-embedding-3-small", "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, @@ -421,7 +403,7 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -430,38 +412,33 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb aioresponses.add( method="DELETE", - url=urljoin(Utility.environment['vector']['db'], - f"/collections/payload_upsert_error_error_json_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd"), ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/payload_upsert_error_error_json_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd"), method="PUT", status=200 ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/payload_upsert_error_error_json_faq_embd/points"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd/points"), method="PUT", payload={"result": None, - 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, - "time": 0.003612634} + 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, + "time": 0.003612634} ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train(user=user) + await gpt3.train(user=user, bot=bot) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == { - 'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} expected_payload = test_content.data #expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { - 'points': [{'id': test_content.vector_id, - 'vector': embedding, - 'payload': expected_payload - }]} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + 'vector': embedding, + 'payload': expected_payload + }]} expected = {"model": "text-embedding-3-small", "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, @@ -472,7 +449,7 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict" @@ -492,9 +469,9 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], "hyperparameters": hyperparameters } mock_completion_request = {"messages": [ @@ -507,17 +484,16 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, @@ -538,8 +514,7 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, - aioresponses): + async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict_with_default_collection" @@ -574,7 +549,7 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_emb mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -584,7 +559,7 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_emb {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, @@ -624,11 +599,11 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], "hyperparameters": hyperparameters - } + } mock_completion_request = {"messages": [ {"role": "system", @@ -640,18 +615,17 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - with mock.patch.dict(Utility.environment, {'vector': {"key": "test", 'db': "http://localhost:6333"}}): - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) + with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=gpt3.bot, **k_faq_action_config) assert response['content'] == generated_text assert gpt3.logs == [ {'messages': [{'role': 'system', @@ -664,12 +638,10 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, - 'with_payload': True, - 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 @@ -687,137 +659,25 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, - aioresponses): + async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) - + user = "test" + bot = "test_gpt3_faq_embedding_predict_with_values_with_instructions" + key = 'test' test_content = CognitionData( - data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", - collection='python', bot="test_gpt3_faq_embedding_predict_with_values_and_stream", user="test").save() - + data="Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ", + collection='java', bot=bot, user=user).save() + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" hyperparameters = Utility.get_default_llm_hyperparameters() - hyperparameters['stream'] = True - key = 'test' - user = "tests" - BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}], - "hyperparameters": hyperparameters - } - - mock_completion_request = {"messages": [ - {"role": "system", - "content": "You are a personal assistant. Answer the question according to the below context"}, - {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} - ]} - mock_completion_request.update(hyperparameters) - mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.side_effect = [{'choices': [ - {'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, - {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[29:60]}, - 'finish_reason': None, 'index': 0}]}, - {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[60:]}, - 'finish_reason': 'stop', 'index': 0}]} - ] - - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) - - aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), - method="POST", - payload={'result': [ - {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} - ) - - response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) - assert response['content'] == "Python is dynamically typed, " - assert gpt3.logs == [ - {'messages': [{'role': 'system', - 'content': 'You are a personal assistant. Answer the question according to the below context'}, - {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': {'choices': [ - {'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, - 'index': 0}]}, - 'type': 'answer_query', - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': True, 'stop': None, 'presence_penalty': 0.0, - 'frequency_penalty': 0.0, 'logit_bias': {}}}] - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, - 'with_payload': True, - 'score_threshold': 0.70} - - assert isinstance(time_elapsed, float) and time_elapsed > 0.0 - - expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot}, - "api_key": key, - "num_retries": 3} - assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) - expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': gpt3.bot} - expected['api_key'] = key - expected['num_retries'] = 3 - assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) - - @pytest.mark.asyncio - @mock.patch.object(litellm, "acompletion", autospec=True) - @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, - mock_embedding, - mock_completion, - aioresponses): - embedding = list(np.random.random(LLMProcessor.__embedding__)) - user = "test" - bot = "payload_with_instruction" - key = 'test' - CognitionSchema( - metadata=[{"column_name": "name", "data_type": "str", "enable_search": True, "create_embeddings": True}, - {"column_name": "city", "data_type": "str", "enable_search": True, "create_embeddings": True}], - collection_name="User_details", - bot=bot, user=user - ).save() - test_content1 = CognitionData( - data={"name": "Nupur", "city": "Pune"}, - content_type="json", - collection="User_details", - bot=bot, user=user).save() - test_content2 = CognitionData( - data={"name": "Fahad", "city": "Mumbai"}, - content_type="json", - collection="User_details", - bot=bot, user=user).save() - test_content3 = CognitionData( - data={"name": "Hitesh", "city": "Mumbai"}, - content_type="json", - collection="User_details", - bot=bot, user=user).save() - - BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=bot, user=user).save() - - generated_text = "Hitesh and Fahad lives in mumbai city." - query = "List all the user lives in mumbai city" - hyperparameters = Utility.get_default_llm_hyperparameters() - k_faq_action_config = { - "system_prompt": "You are a personal assistant. Answer the question according to the below context", - "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", - "similarity_prompt": [{"top_results": 10, - "similarity_threshold": 0.70, - 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": "user_details"}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": "java"}], 'instructions': ['Answer in a short way.', 'Keep it simple.'], "hyperparameters": hyperparameters } @@ -826,39 +686,39 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n[{'name': 'Fahad', 'city': 'Mumbai'}, {'name': 'Hitesh', 'city': 'Mumbai'}]\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: List all the user lives in mumbai city \nA:"} + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"} ]} mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(bot, DEFAULT_LLM) + gpt3 = LLMProcessor(test_content.bot) + aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/{gpt3.bot}_{test_content1.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ - {'id': test_content2.vector_id, 'score': 0.80, "payload": test_content2.data}, - {'id': test_content3.vector_id, 'score': 0.80, "payload": test_content3.data} - ]} + {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) assert response['content'] == generated_text - assert gpt3.logs == [{'messages': [{'role': 'system', - 'content': 'You are a personal assistant. Answer the question according to the below context'}, - {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n[{'name': 'Fahad', 'city': 'Mumbai'}, {'name': 'Hitesh', 'city': 'Mumbai'}]\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: List all the user lives in mumbai city \nA:"}], - 'raw_completion_response': {'choices': [{'message': { - 'content': 'Hitesh and Fahad lives in mumbai city.', 'role': 'assistant'}}]}, - 'type': 'answer_query', - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stop': None, - 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, - 'with_payload': True, - 'score_threshold': 0.70} + assert gpt3.logs == [ + {'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': { + 'choices': [{'message': {'content': 'Python is dynamically typed, garbage-collected, ' + 'high level, general purpose programming.', + 'role': 'assistant'}}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, + 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, + 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 @@ -876,8 +736,7 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, - aioresponses): + async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_gpt3_faq_embedding_predict_completion_connection_error" user = 'test' @@ -896,11 +755,11 @@ async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters - } + } def __mock_connection_error(*args, **kwargs): raise Exception("Connection reset by peer!") @@ -908,24 +767,21 @@ def __mock_connection_error(*args, **kwargs): mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.side_effect = __mock_connection_error - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, - 'with_payload': True, - 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", @@ -939,7 +795,7 @@ def __mock_connection_error(*args, **kwargs): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error'}, 'api_key': 'test', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, - 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -948,7 +804,7 @@ def __mock_connection_error(*args, **kwargs): @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock_llm_request): embedding = list(np.random.random(LLMProcessor.__embedding__)) - user = "test" + user ="test" bot = "test_gpt3_faq_embedding_predict_exact_match" key = 'test' test_content = CognitionData( @@ -962,28 +818,27 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters } mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_llm_request.side_effect = ClientConnectionError() - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) + gpt3 = LLMProcessor(test_content.bot) - response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_exact_match", **k_faq_action_config) assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} - assert gpt3.logs == [ - {'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] + assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, - "api_key": key, - "num_retries": 3} + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio @@ -1009,10 +864,10 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ } mock_embedding.side_effect = [Exception("Connection reset by peer!"), {'data': [{'embedding': embedding}]}] - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) + gpt3 = LLMProcessor(test_content.bot) mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] - response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_embedding_connection_error", **k_faq_action_config) assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] @@ -1027,8 +882,7 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, - aioresponses): + async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_gpt3_faq_embedding_predict_with_previous_bot_responses" @@ -1064,17 +918,16 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, @@ -1117,11 +970,11 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding "query_prompt": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", "use_query_prompt": True}, "similarity_prompt": [ - {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters - } + } mock_rephrase_request = {"messages": [ {"role": "system", @@ -1140,20 +993,18 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, { - 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) + gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], - f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, @@ -1171,77 +1022,3 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) - - @pytest.mark.asyncio - async def test_llm_logging(self): - from kairon.shared.llm.logger import LiteLLMLogger - bot = "test_llm_logging" - user = "test" - litellm.callbacks = [LiteLLMLogger()] - - messages = [{"role":"user", "content":"Hi"}] - expected = "Hi, How may i help you?" - - result = await litellm.acompletion(messages=messages, - model="gpt-3.5-turbo", - mock_response=expected, - metadata={'user': user, 'bot': bot}) - assert result['choices'][0]['message']['content'] == expected - - result = litellm.completion(messages=messages, - model="gpt-3.5-turbo", - mock_response=expected, - metadata={'user': user, 'bot': bot}) - assert result['choices'][0]['message']['content'] == expected - - result = litellm.completion(messages=messages, - model="gpt-3.5-turbo", - mock_response=expected, - stream=True, - metadata={'user': user, 'bot': bot}) - response = '' - for chunk in result: - content = chunk["choices"][0]["delta"]["content"] - if content: - response = response + content - - assert response == expected - - result = await litellm.acompletion(messages=messages, - model="gpt-3.5-turbo", - mock_response=expected, - stream=True, - metadata={'user': user, 'bot': bot}) - response = '' - async for chunk in result: - content = chunk["choices"][0]["delta"]["content"] - print(chunk) - if content: - response += content - - assert response.__contains__(expected) - - with pytest.raises(Exception) as e: - await litellm.acompletion(messages=messages, - model="gpt-3.5-turbo", - mock_response=Exception("Authentication error"), - metadata={'user': user, 'bot': bot}) - - assert str(e) == "Authentication error" - - with pytest.raises(Exception) as e: - litellm.completion(messages=messages, - model="gpt-3.5-turbo", - mock_response=Exception("Authentication error"), - metadata={'user': user, 'bot': bot}) - - assert str(e) == "Authentication error" - - with pytest.raises(Exception) as e: - await litellm.acompletion(messages=messages, - model="gpt-3.5-turbo", - mock_response=Exception("Authentication error"), - stream=True, - metadata={'user': user, 'bot': bot}) - - assert str(e) == "Authentication error" \ No newline at end of file From 06693c03032f7f7aa8ed02ce93fea39b4e841a46 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Mon, 24 Jun 2024 19:00:46 +0530 Subject: [PATCH 43/57] 1. added missing test cases 2. removed stream from hyperparameters from prompt action --- kairon/actions/definitions/database.py | 2 +- kairon/actions/definitions/prompt.py | 1 - kairon/api/models.py | 10 +- kairon/shared/llm/base.py | 4 +- kairon/shared/llm/processor.py | 41 +- kairon/shared/utils.py | 4 +- kairon/shared/vector_embeddings/db/base.py | 8 +- kairon/shared/vector_embeddings/db/qdrant.py | 18 +- kairon/train.py | 2 +- metadata/integrations.yml | 4 - tests/integration_test/action_service_test.py | 35 +- .../data_processor/data_processor_test.py | 44 +- tests/unit_test/llm_test.py | 390 ++++++++++++------ 13 files changed, 346 insertions(+), 217 deletions(-) diff --git a/kairon/actions/definitions/database.py b/kairon/actions/definitions/database.py index ebdf83510..0d54abd49 100644 --- a/kairon/actions/definitions/database.py +++ b/kairon/actions/definitions/database.py @@ -83,7 +83,7 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma request_body = ActionUtility.get_payload(payload, tracker) msg_logger.append(request_body) tracker_data = ActionUtility.build_context(tracker, True) - response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id, bot=self.bot) + response = await vector_db.perform_operation(operation_type, request_body, user=tracker.sender_id) logger.info("response: " + str(response)) response_context = self.__add_user_context_to_http_response(response, tracker_data) bot_response, bot_resp_log, _ = ActionUtility.compose_response(vector_action_config['response'], response_context) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 381e6f543..4c7bf6bc4 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -71,7 +71,6 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma llm_processor = LLMProcessor(self.bot) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, user=tracker.sender_id, - bot=self.bot, **llm_params) status = "FAILURE" if llm_response.get("is_failure", False) is True else status exception = llm_response.get("exception") diff --git a/kairon/api/models.py b/kairon/api/models.py index 435566f34..789873fff 100644 --- a/kairon/api/models.py +++ b/kairon/api/models.py @@ -15,8 +15,7 @@ ACTIVITY_STATUS, INTEGRATION_STATUS, FALLBACK_MESSAGE, - DEFAULT_NLU_FALLBACK_RESPONSE, - DEFAULT_LLM + DEFAULT_NLU_FALLBACK_RESPONSE ) from ..shared.actions.models import ( ActionParameterType, @@ -1037,8 +1036,8 @@ class PromptActionConfigRequest(BaseModel): num_bot_responses: int = 5 failure_message: str = DEFAULT_NLU_FALLBACK_RESPONSE user_question: UserQuestionModel = UserQuestionModel() - llm_type: str = DEFAULT_LLM - hyperparameters: dict = None + llm_type: str + hyperparameters: dict llm_prompts: List[LlmPromptRequest] instructions: List[str] = [] set_slots: List[SetSlotsUsingActionResponse] = [] @@ -1067,7 +1066,8 @@ def validate_llm_type(cls, v, values, **kwargs): @validator("hyperparameters") def validate_llm_hyperparameters(cls, v, values, **kwargs): - Utility.validate_llm_hyperparameters(v, kwargs['llm_type'], ValueError) + if values.get('llm_type'): + Utility.validate_llm_hyperparameters(v, values['llm_type'], ValueError) @root_validator def check(cls, values): diff --git a/kairon/shared/llm/base.py b/kairon/shared/llm/base.py index f07eceda0..006e38a3d 100644 --- a/kairon/shared/llm/base.py +++ b/kairon/shared/llm/base.py @@ -8,9 +8,9 @@ def __init__(self, bot: Text): self.bot = bot @abstractmethod - async def train(self, user, bot, *args, **kwargs) -> Dict: + async def train(self, user, *args, **kwargs) -> Dict: pass @abstractmethod - async def predict(self, query, user, bot, *args, **kwargs) -> Dict: + async def predict(self, query, user, *args, **kwargs) -> Dict: pass diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index ffc48e2eb..c6e7fa8af 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -1,4 +1,4 @@ -import random +from secrets import randbelow, choice import time from typing import Text, Dict, List, Tuple from urllib.parse import urljoin @@ -39,7 +39,7 @@ def __init__(self, bot: Text): self.EMBEDDING_CTX_LENGTH = 8191 self.__logs = [] - async def train(self, user, bot, *args, **kwargs) -> Dict: + async def train(self, user, *args, **kwargs) -> Dict: await self.__delete_collections() count = 0 processor = CognitionDataProcessor() @@ -59,26 +59,25 @@ async def train(self, user, bot, *args, **kwargs) -> Dict: content['data'], metadata) else: search_payload, embedding_payload = {'content': content["data"]}, content["data"] - #search_payload['collection_name'] = collection - embeddings = await self.get_embedding(embedding_payload, user, bot) + embeddings = await self.get_embedding(embedding_payload, user) points = [{'id': content['vector_id'], 'vector': embeddings, 'payload': search_payload}] await self.__collection_upsert__(collection, {'points': points}, err_msg="Unable to train FAQ! Contact support") count += 1 return {"faq": count} - async def predict(self, query: Text, user, bot, *args, **kwargs) -> Tuple: + async def predict(self, query: Text, user, *args, **kwargs) -> Tuple: start_time = time.time() embeddings_created = False try: - query_embedding = await self.get_embedding(query, user, bot) + query_embedding = await self.get_embedding(query, user) embeddings_created = True system_prompt = kwargs.pop('system_prompt', DEFAULT_SYSTEM_PROMPT) context_prompt = kwargs.pop('context_prompt', DEFAULT_CONTEXT_PROMPT) context = await self.__attach_similarity_prompt_if_enabled(query_embedding, context_prompt, **kwargs) - answer = await self.__get_answer(query, system_prompt, context, user, bot, **kwargs) + answer = await self.__get_answer(query, system_prompt, context, user, **kwargs) response = {"content": answer} except Exception as e: logging.exception(e) @@ -100,11 +99,11 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def get_embedding(self, text: Text, user, bot) -> List[float]: + async def get_embedding(self, text: Text, user) -> List[float]: truncated_text = self.truncate_text(text) result = await litellm.aembedding(model="text-embedding-3-small", input=[truncated_text], - metadata={'user': user, 'bot': bot}, + metadata={'user': user, 'bot': self.bot}, api_key=self.api_key, num_retries=3) return result["data"][0]["embedding"] @@ -112,24 +111,25 @@ async def get_embedding(self, text: Text, user, bot) -> List[float]: async def __parse_completion_response(self, response, **kwargs): if kwargs.get("stream"): formatted_response = '' - msg_choice = random.randint(0, kwargs.get("n", 1) - 1) + msg_choice = randbelow(kwargs.get("n", 1)) if response["choices"][0].get("index") == msg_choice and response["choices"][0]['delta'].get('content'): formatted_response = f"{response['choices'][0]['delta']['content']}" else: - msg_choice = random.choice(response['choices']) + msg_choice = choice(response['choices']) formatted_response = msg_choice['message']['content'] return formatted_response - async def __get_completion(self, messages, hyperparameters, user, bot, **kwargs): + async def __get_completion(self, messages, hyperparameters, user, **kwargs): response = await litellm.acompletion(messages=messages, - metadata={'user': user, 'bot': bot}, + metadata={'user': user, 'bot': self.bot}, api_key=self.api_key, num_retries=3, **hyperparameters) - formatted_response = await self.__parse_completion_response(response, **kwargs) + formatted_response = await self.__parse_completion_response(response, + **hyperparameters) return formatted_response, response - async def __get_answer(self, query, system_prompt: Text, context: Text, user, bot, **kwargs): + async def __get_answer(self, query, system_prompt: Text, context: Text, user, **kwargs): use_query_prompt = False query_prompt = '' if kwargs.get('query_prompt', {}): @@ -144,8 +144,7 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, bo if use_query_prompt and query_prompt: query = await self.__rephrase_query(query, system_prompt, query_prompt, hyperparameters=hyperparameters, - user=user, - bot=bot) + user=user) messages = [ {"role": "system", "content": system_prompt}, ] @@ -156,13 +155,12 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, bo completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user, - bot=bot) + user=user) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'answer_query', 'hyperparameters': hyperparameters}) return completion - async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, bot, **kwargs): + async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, **kwargs): messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": f"{query_prompt}\n\n Q: {query}\n A:"} @@ -171,8 +169,7 @@ async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user, - bot=bot) + user=user) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'rephrase_query', 'hyperparameters': hyperparameters}) return completion diff --git a/kairon/shared/utils.py b/kairon/shared/utils.py index 2db890379..acf3ff92a 100644 --- a/kairon/shared/utils.py +++ b/kairon/shared/utils.py @@ -2069,12 +2069,12 @@ def get_llm_hyperparameters(llm_type): @staticmethod def validate_llm_hyperparameters(hyperparameters: dict, llm_type: str, exception_class): - from jsonschema_rs import JSONSchema, ValidationError + from jsonschema_rs import JSONSchema, ValidationError as JValidationError schema = Utility.system_metadata["llm"][llm_type] try: validator = JSONSchema(schema) validator.validate(hyperparameters) - except ValidationError as e: + except JValidationError as e: message = f"{e.instance_path}: {e.message}" raise exception_class(message) diff --git a/kairon/shared/vector_embeddings/db/base.py b/kairon/shared/vector_embeddings/db/base.py index d1c2a1e97..887be41bb 100644 --- a/kairon/shared/vector_embeddings/db/base.py +++ b/kairon/shared/vector_embeddings/db/base.py @@ -8,16 +8,16 @@ class VectorEmbeddingsDbBase(ABC): @abstractmethod - async def embedding_search(self, request_body: Dict, **kwargs): + async def embedding_search(self, request_body: Dict, user: str, **kwargs): raise NotImplementedError("Provider not implemented") @abstractmethod - async def payload_search(self, request_body: Dict, **kwargs): + async def payload_search(self, request_body: Dict, user: str, **kwargs): raise NotImplementedError("Provider not implemented") - async def perform_operation(self, op_type: Text, request_body: Dict, **kwargs): + async def perform_operation(self, op_type: Text, request_body: Dict, user: str, **kwargs): supported_ops = {DbActionOperationType.payload_search.value: self.payload_search, DbActionOperationType.embedding_search.value: self.embedding_search} if op_type not in supported_ops.keys(): raise AppException("Operation type not supported") - return await supported_ops[op_type](request_body, **kwargs) + return await supported_ops[op_type](request_body, user, **kwargs) diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index 893a310ad..12b5268e2 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -29,23 +29,15 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 - def truncate_text(self, text: Text) -> Text: - """ - Truncate text to 8191 tokens for openai - """ - tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] - return self.tokenizer.decode(tokens) + async def __get_embedding(self, text: Text, user: str, **kwargs) -> List[float]: + return await self.llm.get_embedding(text, user=user) - async def __get_embedding(self, text: Text, **kwargs) -> List[float]: - result, _ = await self.llm.get_embedding(text, user=kwargs.get('user'), bot=kwargs.get('bot')) - return result - - async def embedding_search(self, request_body: Dict, **kwargs): + async def embedding_search(self, request_body: Dict, user: str, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points") if request_body.get("text"): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/search") user_msg = request_body.get("text") - vector = await self.__get_embedding(user_msg, **kwargs) + vector = await self.__get_embedding(user_msg, user, **kwargs) request_body = {'vector': vector, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} embedding_search_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', @@ -53,7 +45,7 @@ async def embedding_search(self, request_body: Dict, **kwargs): request_body=request_body) return embedding_search_result - async def payload_search(self, request_body: Dict, **kwargs): + async def payload_search(self, request_body: Dict, user, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points/scroll") payload_filter_result = ActionUtility.execute_http_request(http_url=url, request_method='POST', diff --git a/kairon/train.py b/kairon/train.py index 0276f7bc5..3ddcf9eb0 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -102,7 +102,7 @@ def start_training(bot: str, user: str, token: str = None): settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: llm_processor = LLMProcessor(bot) - faqs = asyncio.run(llm_processor.train(user=user, bot=bot)) + faqs = asyncio.run(llm_processor.train(user=user)) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) agent_url = Utility.environment['model']['agent'].get('url') diff --git a/metadata/integrations.yml b/metadata/integrations.yml index 227b4c413..6ba78b15f 100644 --- a/metadata/integrations.yml +++ b/metadata/integrations.yml @@ -129,10 +129,6 @@ llm: minimum: 1 maximum: 5 description: "The n hyperparameter controls the number of different response options that are generated by the model." - stream: - type: boolean - default: false - description: "the stream hyperparameter controls whether the model generates a continuous stream of text or a single response." stop: anyOf: - type: "string" diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index 1941a8fd6..e54daa27e 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -10497,7 +10497,7 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10564,7 +10564,7 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10634,7 +10634,7 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10859,7 +10859,7 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m embedding = list(np.random.random(OPENAI_EMBEDDING_OUTPUT)) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion.return_value = {'choices': [{'delta': {'role': 'assistant', 'content': generated_text}, 'finish_reason': None, 'index': 0}]} mock_search.return_value = { 'result': [{'id': uuid7().__str__(), 'score': 0.80, 'payload': {'content': bot_content}}]} Actions(name=action_name, type=ActionType.prompt_action.value, bot=bot, user=user).save() @@ -10875,6 +10875,7 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m response = client.post("/webhook", json=request_object) response_json = response.json() + print(response_json['events']) assert response_json['events'] == [ {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', 'value': generated_text}] assert response_json['responses'] == [ @@ -11161,14 +11162,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11245,7 +11246,7 @@ def mock_completion_for_answer(*args, **kwargs): 'content': 'Kanban is a workflow management tool which visualizes both the process (the workflow) and the actual work passing through that process.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) @@ -11253,7 +11254,7 @@ def mock_completion_for_answer(*args, **kwargs): 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11370,14 +11371,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11477,14 +11478,14 @@ def mock_completion_for_answer(*args, **kwargs): 'content': '{"api_type": "filter", {"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}}', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11575,14 +11576,14 @@ def __mock_fetch_similar(*args, **kwargs): 'content': 'Python is dynamically typed, garbage-collected, high level, general purpose programming.', 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11646,7 +11647,7 @@ def __mock_fetch_similar(*args, **kwargs): 'content': "\nInstructions on how to use Similarity Prompt:\n['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Kanban And Scrum Together? \nA:"}], 'metadata': {'user': user, 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11710,7 +11711,7 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], 'metadata': {'user': 'udit.pandey', 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11777,5 +11778,5 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], 'metadata': {'user': user, 'bot': bot}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} + 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index b829acf05..5a9a0bd30 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -170,7 +170,7 @@ def test_add_prompt_action_with_invalid_slots(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', @@ -198,7 +198,7 @@ def test_add_prompt_action_with_invalid_http_action(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'instructions': 'Answer question based on the context below.', 'type': 'system', @@ -227,7 +227,7 @@ def test_add_prompt_action_with_invalid_similarity_threshold(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -257,7 +257,7 @@ def test_add_prompt_action_with_invalid_top_results(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -309,7 +309,7 @@ def test_add_prompt_action_with_empty_collection_for_bot_content_prompt(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', @@ -354,7 +354,7 @@ def test_add_prompt_action_with_bot_content_prompt(self): 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', @@ -554,7 +554,7 @@ def test_add_prompt_action_with_empty_llm_prompts(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': []} with pytest.raises(ValidationError, match="llm_prompts are required!"): @@ -581,7 +581,7 @@ def test_add_prompt_action_faq_action_with_default_values_and_instructions(self) 'failure_message': "I'm sorry, I didn't quite understand that. Could you rephrase?", 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -602,7 +602,7 @@ def test_add_prompt_action_with_invalid_temperature_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 3.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -619,7 +619,7 @@ def test_add_prompt_action_with_invalid_stop_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': ["\n", ".", "?", "!", ";"], + 'n': 1, 'stop': ["\n", ".", "?", "!", ";"], 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -639,7 +639,7 @@ def test_add_prompt_action_with_invalid_presence_penalty_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': -3.0, + 'n': 1, 'stop': '?', 'presence_penalty': -3.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -657,7 +657,7 @@ def test_add_prompt_action_with_invalid_frequency_penalty_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 3.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -674,7 +674,7 @@ def test_add_prompt_action_with_invalid_max_tokens_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 2, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -691,7 +691,7 @@ def test_add_prompt_action_with_zero_max_tokens_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 0, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -708,7 +708,7 @@ def test_add_prompt_action_with_invalid_top_p_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 256, 'model': 'gpt-3.5-turbo', 'top_p': 3.0, - 'n': 1, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 1, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -725,7 +725,7 @@ def test_add_prompt_action_with_invalid_n_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 7, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 7, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -742,7 +742,7 @@ def test_add_prompt_action_with_zero_n_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 0, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 0, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -759,7 +759,7 @@ def test_add_prompt_action_with_invalid_logit_bias_hyperparameter(self): 'failure_message': DEFAULT_NLU_FALLBACK_RESPONSE, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 200, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 2, 'stream': False, 'stop': '?', 'presence_penalty': 0.0, + 'n': 2, 'stop': '?', 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': 'a'}, 'llm_prompts': [{'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', 'source': 'static', 'is_enabled': True}, @@ -834,7 +834,7 @@ def test_edit_prompt_action_faq_action(self): assert not DeepDiff(action, [{'name': 'test_edit_prompt_action_faq_action', 'num_bot_responses': 5, 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_user_message'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -869,7 +869,7 @@ def test_edit_prompt_action_faq_action(self): 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'llm_type': 'openai', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_prompts': [ {'name': 'System Prompt', 'data': 'You are a personal assistant.', 'type': 'system', @@ -912,7 +912,7 @@ def test_edit_prompt_action_with_less_hyperparameters(self): 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ @@ -947,7 +947,7 @@ def test_get_prompt_faq_action(self): 'failure_message': 'updated_failure_message', 'user_question': {'type': 'from_slot', 'value': 'prompt_question'}, 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', - 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}, 'llm_type': 'openai', 'llm_prompts': [ diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index c8f727db3..7738dfeb0 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -8,6 +8,7 @@ from aiohttp import ClientConnectionError from mongoengine import connect from kairon.shared.utils import Utility + Utility.load_system_metadata() from kairon.exceptions import AppException @@ -66,7 +67,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, @@ -119,14 +120,16 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore secret = BotSecrets(secret_type=BotSecretType.gpt_key.value, value=value, bot=bot, user=user).save() embedding = list(np.random.random(LLMProcessor.__embedding__)) - mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]} + mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, { + 'data': [{'embedding': embedding}]} gpt3 = LLMProcessor(bot) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), method="GET", - payload={"time": 0, "status": "ok", "result": {"collections": [{"name": "test_embed_faq_text_swift_faq_embd"}, - {"name": "example_bot_swift_faq_embd"}]}} + payload={"time": 0, "status": "ok", + "result": {"collections": [{"name": "test_embed_faq_text_swift_faq_embd"}, + {"name": "example_bot_swift_faq_embd"}]}} ) aioresponses.add( @@ -141,19 +144,22 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_user_details{gpt3.suffix}/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_user_details{gpt3.suffix}/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_country_details{gpt3.suffix}"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_country_details{gpt3.suffix}"), method="PUT", status=200 ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_country_details{gpt3.suffix}/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_country_details{gpt3.suffix}/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) @@ -165,24 +171,29 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 3 - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'name': f"{gpt3.bot}_country_details{gpt3.suffix}", - 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'name': f"{gpt3.bot}_country_details{gpt3.suffix}", + 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[3][0].kwargs['json'] == {'points': [{'id': test_content_two.vector_id, - 'vector': embedding, - 'payload': {'country': 'Spain'}}]} - assert list(aioresponses.requests.values())[3][1].kwargs['json'] == {'points': [{'id': test_content_three.vector_id, - 'vector': embedding, - 'payload': {'role': 'ds'}}]} + assert list(aioresponses.requests.values())[3][0].kwargs['json'] == { + 'points': [{'id': test_content_two.vector_id, + 'vector': embedding, + 'payload': {'country': 'Spain'}}]} + assert list(aioresponses.requests.values())[3][1].kwargs['json'] == { + 'points': [{'id': test_content_three.vector_id, + 'vector': embedding, + 'payload': {'role': 'ds'}}]} - assert list(aioresponses.requests.values())[4][0].kwargs['json'] == {'name': f"{gpt3.bot}_user_details{gpt3.suffix}", - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[5][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, - 'payload': {'name': 'Nupur'}}]} + assert list(aioresponses.requests.values())[4][0].kwargs['json'] == { + 'name': f"{gpt3.bot}_user_details{gpt3.suffix}", + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[5][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, + 'payload': {'name': 'Nupur'}}]} assert response['faq'] == 3 expected = {"model": "text-embedding-3-small", @@ -217,7 +228,8 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a gpt3 = LLMProcessor(bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), method="PUT", status=200 ) @@ -227,18 +239,21 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a payload={"time": 0, "status": "ok", "result": {"collections": []}}) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/test_embed_faq_json_payload_with_int_faq_embd/points"), method="PUT", payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 1 - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_embed_faq_json_payload_with_int_faq_embd', - 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == { + 'name': 'test_embed_faq_json_payload_with_int_faq_embd', + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, 'vector': embedding, 'payload': {'name': 'Ram', 'age': 23, 'color': 'red'} }]} @@ -302,7 +317,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): payload={"result": {"operation_id": 0, "status": "acknowledged"}, "status": "ok", "time": 0.003612634} ) - response = await gpt3.train(user=user, bot=bot) + response = await gpt3.train(user=user) assert response['faq'] == 1 assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'test_int_embd_int_faq_embd', @@ -363,16 +378,18 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aiore url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}{gpt3.suffix}/points"), method="PUT", payload={"result": None, - 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, - "time": 0.003612634} + 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, + "time": 0.003612634} ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train(user=user, bot=bot) + await gpt3.train(user=user) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, 'vectors': gpt3.vector_config} - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, 'payload': {'content': test_content.data}}]} + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': gpt3.bot + gpt3.suffix, + 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, 'payload': {'content': test_content.data}}]} expected = {"model": "text-embedding-3-small", "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, @@ -412,33 +429,38 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb aioresponses.add( method="DELETE", - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd"), ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd"), method="PUT", status=200 ) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/payload_upsert_error_error_json_faq_embd/points"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/payload_upsert_error_error_json_faq_embd/points"), method="PUT", payload={"result": None, - 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, - "time": 0.003612634} + 'status': {'error': 'Json deserialize error: missing field `vectors` at line 1 column 34779'}, + "time": 0.003612634} ) with pytest.raises(AppException, match="Unable to train FAQ! Contact support"): - await gpt3.train(user=user, bot=bot) + await gpt3.train(user=user) - assert list(aioresponses.requests.values())[1][0].kwargs['json'] == {'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} + assert list(aioresponses.requests.values())[1][0].kwargs['json'] == { + 'name': 'payload_upsert_error_error_json_faq_embd', 'vectors': gpt3.vector_config} expected_payload = test_content.data #expected_payload['collection_name'] = 'payload_upsert_error_error_json_faq_embd' - assert list(aioresponses.requests.values())[2][0].kwargs['json'] == {'points': [{'id': test_content.vector_id, - 'vector': embedding, - 'payload': expected_payload - }]} + assert list(aioresponses.requests.values())[2][0].kwargs['json'] == { + 'points': [{'id': test_content.vector_id, + 'vector': embedding, + 'payload': expected_payload + }]} expected = {"model": "text-embedding-3-small", "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, @@ -449,7 +471,7 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict" @@ -469,9 +491,9 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], "hyperparameters": hyperparameters } mock_completion_request = {"messages": [ @@ -487,13 +509,14 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, @@ -514,7 +537,8 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_embed_faq_predict_with_default_collection" @@ -559,7 +583,7 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_emb {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, @@ -599,11 +623,11 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - 'collection': 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], "hyperparameters": hyperparameters - } + } mock_completion_request = {"messages": [ {"role": "system", @@ -615,17 +639,18 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + with mock.patch.dict(Utility.environment, {'vector': {"key": "test", 'db': "http://localhost:6333"}}): gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=gpt3.bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert gpt3.logs == [ {'messages': [{'role': 'system', @@ -638,10 +663,12 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock 'role': 'assistant'}}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, 'stream': False, 'stop': None, 'presence_penalty': 0.0, + 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 @@ -659,25 +686,133 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) - user = "test" - bot = "test_gpt3_faq_embedding_predict_with_values_with_instructions" - key = 'test' + test_content = CognitionData( - data="Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ", - collection='java', bot=bot, user=user).save() - BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() + data="Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.", + collection='python', bot="test_gpt3_faq_embedding_predict_with_values_and_stream", user="test").save() + generated_text = "Python is dynamically typed, garbage-collected, high level, general purpose programming." query = "What kind of language is python?" hyperparameters = Utility.get_default_llm_hyperparameters() + hyperparameters['stream'] = True + key = 'test' + user = "tests" + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=test_content.bot, user=user).save() k_faq_action_config = { "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": "java"}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + 'collection': 'python'}], + "hyperparameters": hyperparameters + } + + mock_completion_request = {"messages": [ + {"role": "system", + "content": "You are a personal assistant. Answer the question according to the below context"}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"} + ]} + mock_completion_request.update(hyperparameters) + mock_embedding.return_value = {'data': [{'embedding': embedding}]} + mock_completion.side_effect = [{'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[29:60]}, + 'finish_reason': None, 'index': 0}]}, + {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[60:]}, + 'finish_reason': 'stop', 'index': 0}]} + ] + + with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): + gpt3 = LLMProcessor(test_content.bot) + + aioresponses.add( + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + method="POST", + payload={'result': [ + {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} + ) + + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) + assert response['content'] == "Python is dynamically typed, " + assert gpt3.logs == [ + {'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], + 'raw_completion_response': {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, + 'n': 1, 'stream': True, 'stop': None, 'presence_penalty': 0.0, + 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} + + assert isinstance(time_elapsed, float) and time_elapsed > 0.0 + + expected = {"model": "text-embedding-3-small", + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot}, + "api_key": key, + "num_retries": 3} + assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) + expected = mock_completion_request.copy() + expected['metadata'] = {'user': user, 'bot': gpt3.bot} + expected['api_key'] = key + expected['num_retries'] = 3 + assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + @pytest.mark.asyncio + @mock.patch.object(litellm, "acompletion", autospec=True) + @mock.patch.object(litellm, "aembedding", autospec=True) + async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, + mock_embedding, + mock_completion, + aioresponses): + embedding = list(np.random.random(LLMProcessor.__embedding__)) + user = "test" + bot = "payload_with_instruction" + key = 'test' + CognitionSchema( + metadata=[{"column_name": "name", "data_type": "str", "enable_search": True, "create_embeddings": True}, + {"column_name": "city", "data_type": "str", "enable_search": True, "create_embeddings": True}], + collection_name="User_details", + bot=bot, user=user + ).save() + test_content1 = CognitionData( + data={"name": "Nupur", "city": "Pune"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + test_content2 = CognitionData( + data={"name": "Fahad", "city": "Mumbai"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + test_content3 = CognitionData( + data={"name": "Hitesh", "city": "Mumbai"}, + content_type="json", + collection="User_details", + bot=bot, user=user).save() + + BotSecrets(secret_type=BotSecretType.gpt_key.value, value=key, bot=bot, user=user).save() + + generated_text = "Hitesh and Fahad lives in mumbai city." + query = "List all the user lives in mumbai city" + hyperparameters = Utility.get_default_llm_hyperparameters() + k_faq_action_config = { + "system_prompt": "You are a personal assistant. Answer the question according to the below context", + "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", + "similarity_prompt": [{"top_results": 10, + "similarity_threshold": 0.70, + 'use_similarity_prompt': True, + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": "user_details"}], 'instructions': ['Answer in a short way.', 'Keep it simple.'], "hyperparameters": hyperparameters } @@ -686,39 +821,39 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mo {'role': 'system', 'content': 'You are a personal assistant. Answer the question according to the below context'}, {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"} + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n[{'name': 'Fahad', 'city': 'Mumbai'}, {'name': 'Hitesh', 'city': 'Mumbai'}]\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: List all the user lives in mumbai city \nA:"} ]} mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot) - + gpt3 = LLMProcessor(bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content1.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ - {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} + {'id': test_content2.vector_id, 'score': 0.80, "payload": test_content2.data}, + {'id': test_content3.vector_id, 'score': 0.80, "payload": test_content3.data} + ]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text - assert gpt3.logs == [ - {'messages': [{'role': 'system', - 'content': 'You are a personal assistant. Answer the question according to the below context'}, - {'role': 'user', - 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Java is a high-level, general-purpose programming language. Java is known for its write once, run anywhere capability. ']\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': { - 'choices': [{'message': {'content': 'Python is dynamically typed, garbage-collected, ' - 'high level, general purpose programming.', - 'role': 'assistant'}}]}, - 'type': 'answer_query', - 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, - 'n': 1, - 'stream': False, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, - 'logit_bias': {}}}] - - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert gpt3.logs == [{'messages': [{'role': 'system', + 'content': 'You are a personal assistant. Answer the question according to the below context'}, + {'role': 'user', + 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n[{'name': 'Fahad', 'city': 'Mumbai'}, {'name': 'Hitesh', 'city': 'Mumbai'}]\nAnswer according to this context.\n \nAnswer in a short way.\nKeep it simple. \nQ: List all the user lives in mumbai city \nA:"}], + 'raw_completion_response': {'choices': [{'message': { + 'content': 'Hitesh and Fahad lives in mumbai city.', 'role': 'assistant'}}]}, + 'type': 'answer_query', + 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', + 'top_p': 0.0, 'n': 1, 'stop': None, + 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}}}] + + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 @@ -736,7 +871,8 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mo @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_gpt3_faq_embedding_predict_completion_connection_error" user = 'test' @@ -755,11 +891,11 @@ async def test_gpt3_faq_embedding_predict_completion_connection_error(self, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters - } + } def __mock_connection_error(*args, **kwargs): raise Exception("Connection reset by peer!") @@ -770,18 +906,21 @@ def __mock_connection_error(*args, **kwargs): gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response == {'exception': "Connection reset by peer!", 'is_failure': True, "content": None} assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Connection reset by peer!'}] - assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, 'with_payload': True, 'score_threshold': 0.70} + assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, + 'with_payload': True, + 'score_threshold': 0.70} assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", @@ -795,7 +934,7 @@ def __mock_connection_error(*args, **kwargs): 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error'}, 'api_key': 'test', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, - 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': False, 'stop': None, + 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -804,7 +943,7 @@ def __mock_connection_error(*args, **kwargs): @mock.patch.object(litellm, "aembedding", autospec=True) async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock_llm_request): embedding = list(np.random.random(LLMProcessor.__embedding__)) - user ="test" + user = "test" bot = "test_gpt3_faq_embedding_predict_exact_match" key = 'test' test_content = CognitionData( @@ -818,9 +957,9 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock "system_prompt": "You are a personal assistant. Answer the question according to the below context", "context_prompt": "Based on below context answer question, if answer not in context check previous logs.", "similarity_prompt": [{"top_results": 10, "similarity_threshold": 0.70, 'use_similarity_prompt': True, - 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters } @@ -829,16 +968,17 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock gpt3 = LLMProcessor(test_content.bot) - response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_exact_match", **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} - assert gpt3.logs == [{'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] + assert gpt3.logs == [ + {'error': 'Retrieving chat completion for the provided query. Failed to connect to service: localhost'}] assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, - "api_key": key, - "num_retries": 3} + "input": [query], 'metadata': {'user': user, 'bot': bot}, + "api_key": key, + "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @pytest.mark.asyncio @@ -867,7 +1007,7 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ gpt3 = LLMProcessor(test_content.bot) mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] - response, time_elapsed = await gpt3.predict(query, user="test", bot="test_gpt3_faq_embedding_predict_embedding_connection_error", **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) assert response == {'exception': 'Connection reset by peer!', 'is_failure': True, "content": None} assert gpt3.logs == [{'error': 'Creating a new embedding for the provided query. Connection reset by peer!'}] @@ -882,7 +1022,8 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) bot = "test_gpt3_faq_embedding_predict_with_previous_bot_responses" @@ -921,13 +1062,14 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot,**k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, @@ -970,11 +1112,11 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding "query_prompt": "A programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.", "use_query_prompt": True}, "similarity_prompt": [ - {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', - 'similarity_prompt_instructions': 'Answer according to this context.', - "collection": 'python'}], + {'use_similarity_prompt': True, 'similarity_prompt_name': 'Similarity Prompt', + 'similarity_prompt_instructions': 'Answer according to this context.', + "collection": 'python'}], "hyperparameters": hyperparameters - } + } mock_rephrase_request = {"messages": [ {"role": "system", @@ -993,18 +1135,20 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} + mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, { + 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} gpt3 = LLMProcessor(test_content.bot) aioresponses.add( - url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), + url=urljoin(Utility.environment['vector']['db'], + f"/collections/{gpt3.bot}_{test_content.collection}{gpt3.suffix}/points/search"), method="POST", payload={'result': [ {'id': test_content.vector_id, 'score': 0.80, "payload": {'content': test_content.data}}]} ) - response, time_elapsed = await gpt3.predict(query, user=user, bot=bot, **k_faq_action_config) + response, time_elapsed = await gpt3.predict(query, user=user, **k_faq_action_config) assert response['content'] == generated_text assert list(aioresponses.requests.values())[0][0].kwargs['json'] == {'vector': embedding, 'limit': 10, From 887adc1fd68a31cabd35d4dc9413829d792f140c Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 10:40:04 +0530 Subject: [PATCH 44/57] test cased fixed --- tests/unit_test/data_processor/data_processor_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 5a9a0bd30..08320bc27 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -60,7 +60,7 @@ from kairon.shared.data.constant import UTTERANCE_TYPE, EVENT_STATUS, STORY_EVENT, ALLOWED_DOMAIN_FORMATS, \ ALLOWED_CONFIG_FORMATS, ALLOWED_NLU_FORMATS, ALLOWED_STORIES_FORMATS, ALLOWED_RULES_FORMATS, REQUIREMENTS, \ DEFAULT_NLU_FALLBACK_RULE, SLOT_TYPE, KAIRON_TWO_STAGE_FALLBACK, AuditlogActions, TOKEN_TYPE, GPT_LLM_FAQ, \ - DEFAULT_CONTEXT_PROMPT, DEFAULT_NLU_FALLBACK_RESPONSE, DEFAULT_SYSTEM_PROMPT + DEFAULT_CONTEXT_PROMPT, DEFAULT_NLU_FALLBACK_RESPONSE, DEFAULT_SYSTEM_PROMPT, DEFAULT_LLM from kairon.shared.data.data_objects import (TrainingExamples, Slots, Entities, EntitySynonyms, RegexFeatures, @@ -8488,7 +8488,9 @@ def test_delete_action_with_attached_http_action(self): 'data': 'tester_action', 'instructions': 'Answer according to the context', 'type': 'user', 'source': 'action', - 'is_enabled': True}] + 'is_enabled': True}], + llm_type=DEFAULT_LLM, + hyperparameters=Utility.get_default_llm_hyperparameters() ) processor.add_http_action_config(http_action_config.dict(), user, bot) processor.add_prompt_action(prompt_action_config.dict(), bot, user) From 8252520f5d01819871e5e19496b03ac5816e6141 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 15:48:54 +0530 Subject: [PATCH 45/57] 1. added missing test case 2. updated litellm --- kairon/shared/llm/logger.py | 27 ++++++++++++--------------- requirements/prod.txt | 2 +- tests/unit_test/llm_test.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py index 06b720b48..dbb93e469 100644 --- a/kairon/shared/llm/logger.py +++ b/kairon/shared/llm/logger.py @@ -1,14 +1,10 @@ from litellm.integrations.custom_logger import CustomLogger from .data_objects import LLMLogs import ujson as json +from loguru import logger class LiteLLMLogger(CustomLogger): - def log_pre_api_call(self, model, messages, kwargs): - pass - - def log_post_api_call(self, kwargs, response_obj, start_time, end_time): - pass def log_stream_event(self, kwargs, response_obj, start_time, end_time): self.__logs_litellm(**kwargs) @@ -29,15 +25,16 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti self.__logs_litellm(**kwargs) def __logs_litellm(self, **kwargs): - litellm_params = kwargs['litellm_params'] - self.__save_logs(**{'response': json.loads(kwargs['original_response']), - 'start_time': kwargs['start_time'], - 'end_time': kwargs['end_time'], - 'cost': kwargs["response_cost"], - 'llm_call_id': litellm_params['litellm_call_id'], - 'llm_provider': litellm_params['custom_llm_provider'], - 'model_params': kwargs["additional_args"]["complete_input_dict"], - 'metadata': litellm_params['metadata']}) + logger.info("logging llms call") + litellm_params = kwargs.get('litellm_params') + self.__save_logs(**{'response': json.loads(kwargs.get('original_response')) if kwargs.get('original_response') else None, + 'start_time': kwargs.get('start_time'), + 'end_time': kwargs.get('end_time'), + 'cost': kwargs.get("response_cost"), + 'llm_call_id': litellm_params.get('litellm_call_id'), + 'llm_provider': litellm_params.get('custom_llm_provider'), + 'model_params': kwargs.get("additional_args", {}).get("complete_input_dict"), + 'metadata': litellm_params.get('metadata')}) def __save_logs(self, **kwargs): - LLMLogs(**kwargs).save() + print(LLMLogs(**kwargs).save().to_mongo().to_dict()) diff --git a/requirements/prod.txt b/requirements/prod.txt index e00d448a9..13fcc40e7 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -64,6 +64,6 @@ opentelemetry-instrumentation-requests==0.46b0 opentelemetry-instrumentation-sklearn==0.46b0 pykwalify==1.8.0 gunicorn==22.0.0 -litellm==1.38.11 +litellm==1.39.5 jsonschema_rs==0.18.0 mongoengine-jsonschema==0.1.3 \ No newline at end of file diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 7738dfeb0..bb446e802 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1,4 +1,5 @@ import os +import time from urllib.parse import urljoin import mock @@ -17,6 +18,7 @@ from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT from kairon.shared.llm.processor import LLMProcessor +from kairon.shared.llm.data_objects import LLMLogs import litellm from deepdiff import DeepDiff @@ -1166,3 +1168,33 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) + + @pytest.mark.asyncio + async def test_llm_logging(self): + from kairon.shared.llm.logger import LiteLLMLogger + bot = "test_llm_logging" + user = "test" + litellm.callbacks = [LiteLLMLogger()] + + result = await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + metadata={'user': user, 'bot': bot}) + assert result + + result = litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + metadata={'user': user, 'bot': bot}) + assert result + + result = litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + stream=True, + metadata={'user': user, 'bot': bot}) + for chunk in result: + print(chunk["choices"][0]["delta"]["content"]) + assert chunk["choices"][0]["delta"]["content"] + + assert list(LLMLogs.objects(metadata__bot=bot)) From b556412c66380e55db91c73fe3216a1061694f7a Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 17:28:09 +0530 Subject: [PATCH 46/57] 1. added missing test case --- tests/unit_test/llm_test.py | 56 +++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index bb446e802..09eea5168 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -688,7 +688,8 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock @pytest.mark.asyncio @mock.patch.object(litellm, "acompletion", autospec=True) @mock.patch.object(litellm, "aembedding", autospec=True) - async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, aioresponses): + async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embedding, mock_completion, + aioresponses): embedding = list(np.random.random(LLMProcessor.__embedding__)) test_content = CognitionData( @@ -720,7 +721,8 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe ]} mock_completion_request.update(hyperparameters) mock_embedding.return_value = {'data': [{'embedding': embedding}]} - mock_completion.side_effect = [{'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + mock_completion.side_effect = [{'choices': [ + {'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[29:60]}, 'finish_reason': None, 'index': 0}]}, {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[60:]}, @@ -745,7 +747,9 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe 'content': 'You are a personal assistant. Answer the question according to the below context'}, {'role': 'user', 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], - 'raw_completion_response': {'choices': [{'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, 'index': 0}]}, + 'raw_completion_response': {'choices': [ + {'delta': {'role': 'assistant', 'content': generated_text[0:29]}, 'finish_reason': None, + 'index': 0}]}, 'type': 'answer_query', 'hyperparameters': {'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': True, 'stop': None, 'presence_penalty': 0.0, @@ -1177,17 +1181,17 @@ async def test_llm_logging(self): litellm.callbacks = [LiteLLMLogger()] result = await litellm.acompletion(messages=["Hi"], - model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", - metadata={'user': user, 'bot': bot}) - assert result - - result = litellm.completion(messages=["Hi"], model="gpt-3.5-turbo", mock_response="Hi, How may i help you?", metadata={'user': user, 'bot': bot}) assert result + result = litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + metadata={'user': user, 'bot': bot}) + assert result + result = litellm.completion(messages=["Hi"], model="gpt-3.5-turbo", mock_response="Hi, How may i help you?", @@ -1197,4 +1201,38 @@ async def test_llm_logging(self): print(chunk["choices"][0]["delta"]["content"]) assert chunk["choices"][0]["delta"]["content"] + result = await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response="Hi, How may i help you?", + stream=True, + metadata={'user': user, 'bot': bot}) + async for chunk in result: + print(chunk["choices"][0]["delta"]["content"]) + assert chunk["choices"][0]["delta"]["content"] + assert list(LLMLogs.objects(metadata__bot=bot)) + + with pytest.raises(Exception) as e: + await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + metadata={'user': user, 'bot': bot}) + + assert str(e) == "Authentication error" + + with pytest.raises(Exception) as e: + litellm.completion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + metadata={'user': user, 'bot': bot}) + + assert str(e) == "Authentication error" + + with pytest.raises(Exception) as e: + await litellm.acompletion(messages=["Hi"], + model="gpt-3.5-turbo", + mock_response=Exception("Authentication error"), + stream=True, + metadata={'user': user, 'bot': bot}) + + assert str(e) == "Authentication error" From 70b8c71b9edb7830040ad292c216722da2ce234c Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Tue, 25 Jun 2024 18:50:36 +0530 Subject: [PATCH 47/57] 1. added missing test case --- tests/unit_test/llm_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 09eea5168..bcc2ed661 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1210,8 +1210,6 @@ async def test_llm_logging(self): print(chunk["choices"][0]["delta"]["content"]) assert chunk["choices"][0]["delta"]["content"] - assert list(LLMLogs.objects(metadata__bot=bot)) - with pytest.raises(Exception) as e: await litellm.acompletion(messages=["Hi"], model="gpt-3.5-turbo", From 181cfa73b01404c11f602c3201e3302a4bc10d1a Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 11:10:39 +0530 Subject: [PATCH 48/57] 1. added missing test case 2. removed deprecated api --- tests/integration_test/action_service_test.py | 2 +- tests/unit_test/data_processor/data_processor_test.py | 2 +- tests/unit_test/llm_test.py | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index e54daa27e..cd5e2c63c 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -3,7 +3,7 @@ from urllib.parse import urlencode, urljoin import litellm -import mock +from unittest import mock import numpy as np import pytest import responses diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index 08320bc27..c79165738 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -14,7 +14,7 @@ Utility.load_system_metadata() -from mock import patch +from unittest.mock import patch import numpy as np import pandas as pd import pytest diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index bcc2ed661..16daaec64 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1,13 +1,13 @@ import os -import time +from unittest import mock from urllib.parse import urljoin -import mock import numpy as np import pytest import ujson as json from aiohttp import ClientConnectionError from mongoengine import connect + from kairon.shared.utils import Utility Utility.load_system_metadata() @@ -18,7 +18,6 @@ from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT from kairon.shared.llm.processor import LLMProcessor -from kairon.shared.llm.data_objects import LLMLogs import litellm from deepdiff import DeepDiff @@ -1233,4 +1232,4 @@ async def test_llm_logging(self): stream=True, metadata={'user': user, 'bot': bot}) - assert str(e) == "Authentication error" + assert str(e) == "Authentication error" \ No newline at end of file From d0d5b1dec20fc5ff67bbbc1f7c5bb09f7c2ccf91 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 13:38:29 +0530 Subject: [PATCH 49/57] test cases fixed --- kairon/shared/llm/logger.py | 3 ++- tests/unit_test/llm_test.py | 46 +++++++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/kairon/shared/llm/logger.py b/kairon/shared/llm/logger.py index dbb93e469..3af7a7870 100644 --- a/kairon/shared/llm/logger.py +++ b/kairon/shared/llm/logger.py @@ -37,4 +37,5 @@ def __logs_litellm(self, **kwargs): 'metadata': litellm_params.get('metadata')}) def __save_logs(self, **kwargs): - print(LLMLogs(**kwargs).save().to_mongo().to_dict()) + logs = LLMLogs(**kwargs).save().to_mongo().to_dict() + logger.info(f"llm logs: {logs}") diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 16daaec64..1042be91d 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -1179,38 +1179,50 @@ async def test_llm_logging(self): user = "test" litellm.callbacks = [LiteLLMLogger()] - result = await litellm.acompletion(messages=["Hi"], + messages = [{"role":"user", "content":"Hi"}] + expected = "Hi, How may i help you?" + + result = await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, metadata={'user': user, 'bot': bot}) - assert result + assert result['choices'][0]['message']['content'] == expected - result = litellm.completion(messages=["Hi"], + result = litellm.completion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, metadata={'user': user, 'bot': bot}) - assert result + assert result['choices'][0]['message']['content'] == expected - result = litellm.completion(messages=["Hi"], + result = litellm.completion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, stream=True, metadata={'user': user, 'bot': bot}) + response = '' for chunk in result: - print(chunk["choices"][0]["delta"]["content"]) - assert chunk["choices"][0]["delta"]["content"] + content = chunk["choices"][0]["delta"]["content"] + if content: + response = response + content + + assert response == expected - result = await litellm.acompletion(messages=["Hi"], + result = await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", - mock_response="Hi, How may i help you?", + mock_response=expected, stream=True, metadata={'user': user, 'bot': bot}) + response = '' async for chunk in result: - print(chunk["choices"][0]["delta"]["content"]) - assert chunk["choices"][0]["delta"]["content"] + content = chunk["choices"][0]["delta"]["content"] + print(chunk) + if content: + response += content + + assert response.__contains__(expected) with pytest.raises(Exception) as e: - await litellm.acompletion(messages=["Hi"], + await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), metadata={'user': user, 'bot': bot}) @@ -1218,7 +1230,7 @@ async def test_llm_logging(self): assert str(e) == "Authentication error" with pytest.raises(Exception) as e: - litellm.completion(messages=["Hi"], + litellm.completion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), metadata={'user': user, 'bot': bot}) @@ -1226,7 +1238,7 @@ async def test_llm_logging(self): assert str(e) == "Authentication error" with pytest.raises(Exception) as e: - await litellm.acompletion(messages=["Hi"], + await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), stream=True, From 2ed8e06bb7a5706d149cf39a55b1324975d8c73e Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 14:53:59 +0530 Subject: [PATCH 50/57] removed unused variable --- kairon/actions/definitions/prompt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 4c7bf6bc4..6323589b9 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -66,7 +66,6 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma k_faq_action_config, bot_settings = self.retrieve_config() user_question = k_faq_action_config.get('user_question') user_msg = self.__get_user_msg(tracker, user_question) - llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) llm_processor = LLMProcessor(self.bot) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, From dd80d847852c0b94121a0a63a91f4d61fe5aa7f1 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 15:15:10 +0530 Subject: [PATCH 51/57] fixed unused variable --- kairon/actions/definitions/prompt.py | 3 +- kairon/shared/llm/processor.py | 3 +- kairon/shared/vector_embeddings/db/qdrant.py | 6 ++-- kairon/train.py | 4 +-- tests/unit_test/llm_test.py | 36 ++++++++++---------- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 6323589b9..6441d607f 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -66,8 +66,9 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma k_faq_action_config, bot_settings = self.retrieve_config() user_question = k_faq_action_config.get('user_question') user_msg = self.__get_user_msg(tracker, user_question) + llm_type = k_faq_action_config['llm_type'] llm_params = await self.__get_llm_params(k_faq_action_config, dispatcher, tracker, domain) - llm_processor = LLMProcessor(self.bot) + llm_processor = LLMProcessor(self.bot, llm_type) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, user=tracker.sender_id, **llm_params) diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index c6e7fa8af..adbb039ae 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -26,13 +26,14 @@ class LLMProcessor(LLMBase): __embedding__ = 1536 - def __init__(self, bot: Text): + def __init__(self, bot: Text, llm_type: str): super().__init__(bot) self.db_url = Utility.environment['vector']['db'] self.headers = {} if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.suffix = "_faq_embd" + self.llm_type = llm_type self.vector_config = {'size': self.__embedding__, 'distance': 'Cosine'} self.api_key = Sysadmin.get_bot_secret(bot, BotSecretType.gpt_key.value, raise_err=True) self.tokenizer = get_encoding("cl100k_base") diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index 12b5268e2..454eeb8fe 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -5,10 +5,8 @@ from kairon import Utility from kairon.shared.actions.utils import ActionUtility -from kairon.shared.admin.constants import BotSecretType -from kairon.shared.admin.processor import Sysadmin -from kairon.shared.constants import GPT3ResourceTypes from kairon.shared.llm.processor import LLMProcessor +from kairon.shared.data.constant import DEFAULT_LLM from kairon.shared.vector_embeddings.db.base import VectorEmbeddingsDbBase @@ -25,7 +23,7 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: if Utility.environment['vector']['key']: self.headers = {"api-key": Utility.environment['vector']['key']} self.llm_settings = llm_settings - self.llm = LLMProcessor(self.bot) + self.llm = LLMProcessor(self.bot, DEFAULT_LLM) self.tokenizer = get_encoding("cl100k_base") self.EMBEDDING_CTX_LENGTH = 8191 diff --git a/kairon/train.py b/kairon/train.py index 3ddcf9eb0..d8360ce67 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -6,7 +6,7 @@ from rasa.api import train from rasa.model import DEFAULT_MODELS_PATH from rasa.shared.constants import DEFAULT_CONFIG_PATH, DEFAULT_DATA_PATH, DEFAULT_DOMAIN_PATH - +from kairon.shared.data.constant import DEFAULT_LLM from kairon.chat.agent.agent import KaironAgent from kairon.exceptions import AppException from kairon.shared.account.processor import AccountProcessor @@ -101,7 +101,7 @@ def start_training(bot: str, user: str, token: str = None): settings = processor.get_bot_settings(bot, user) settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: - llm_processor = LLMProcessor(bot) + llm_processor = LLMProcessor(bot, DEFAULT_LLM) faqs = asyncio.run(llm_processor.train(user=user)) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 1042be91d..a385106cd 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -16,7 +16,7 @@ from kairon.shared.admin.constants import BotSecretType from kairon.shared.admin.data_objects import BotSecrets from kairon.shared.cognition.data_objects import CognitionData, CognitionSchema -from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT +from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_LLM from kairon.shared.llm.processor import LLMProcessor import litellm from deepdiff import DeepDiff @@ -44,7 +44,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM)[[]] aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -123,7 +123,7 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore embedding = list(np.random.random(LLMProcessor.__embedding__)) mock_embedding.side_effect = {'data': [{'embedding': embedding}]}, {'data': [{'embedding': embedding}]}, { 'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -227,7 +227,7 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a input = {"name": "Ram", "color": "red"} mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/test_embed_faq_json_payload_with_int_faq_embd"), @@ -288,7 +288,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): input = {"name": "Ram", "color": "red"} mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -339,7 +339,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): def test_gpt3_faq_embedding_train_failure(self): with pytest.raises(AppException, match=f"Bot secret '{BotSecretType.gpt_key.value}' not configured!"): - LLMProcessor('test_failure') + LLMProcessor('test_failure', DEFAULT_LLM) @pytest.mark.asyncio @mock.patch.object(litellm, "aembedding", autospec=True) @@ -357,7 +357,7 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aiore mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -421,7 +421,7 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb mock_embedding.return_value = {'data': [{'embedding': embedding}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), @@ -507,7 +507,7 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -574,7 +574,7 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_emb mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -641,7 +641,7 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} with mock.patch.dict(Utility.environment, {'vector': {"key": "test", 'db': "http://localhost:6333"}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -729,7 +729,7 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe ] with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': key}}): - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -832,7 +832,7 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(bot) + gpt3 = LLMProcessor(bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections/{gpt3.bot}_{test_content1.collection}{gpt3.suffix}/points/search"), @@ -908,7 +908,7 @@ def __mock_connection_error(*args, **kwargs): mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.side_effect = __mock_connection_error - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -971,7 +971,7 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_llm_request.side_effect = ClientConnectionError() - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) assert response == {'exception': 'Failed to connect to service: localhost', 'is_failure': True, "content": None} @@ -1009,7 +1009,7 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ } mock_embedding.side_effect = [Exception("Connection reset by peer!"), {'data': [{'embedding': embedding}]}] - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) mock_embedding.side_effect = [Exception("Connection reset by peer!"), embedding] response, time_elapsed = await gpt3.predict(query, user="test", **k_faq_action_config) @@ -1064,7 +1064,7 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock mock_embedding.return_value = {'data': [{'embedding': embedding}]} mock_completion.return_value = {'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], @@ -1143,7 +1143,7 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding mock_completion.side_effect = {'choices': [{'message': {'content': rephrased_query, 'role': 'assistant'}}]}, { 'choices': [{'message': {'content': generated_text, 'role': 'assistant'}}]} - gpt3 = LLMProcessor(test_content.bot) + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], From e68ba65474db1c2fca266edaa76c1cae9293d0b7 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 26 Jun 2024 16:28:11 +0530 Subject: [PATCH 52/57] fixed unused variable --- tests/unit_test/llm_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index a385106cd..53c31bace 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -44,7 +44,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): with mock.patch.dict(Utility.environment, {'llm': {"faq": "GPT3_FAQ_EMBED", 'api_key': secret}, 'vector': {'db': "http://kairon:6333", "key": None}}): mock_embedding.return_value = {'data': [{'embedding': embedding}]} - gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM)[[]] + gpt3 = LLMProcessor(test_content.bot, DEFAULT_LLM) aioresponses.add( url=urljoin(Utility.environment['vector']['db'], f"/collections"), From a0156f48eb634c72646a34f8262cd277cba45aad Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 28 Jun 2024 12:56:08 +0530 Subject: [PATCH 53/57] added test cased for fetching logs --- kairon/shared/llm/processor.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index adbb039ae..7365e0aa7 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -16,6 +16,7 @@ from kairon.shared.data.constant import DEFAULT_SYSTEM_PROMPT, DEFAULT_CONTEXT_PROMPT from kairon.shared.llm.base import LLMBase from kairon.shared.llm.logger import LiteLLMLogger +from kairon.shared.llm.data_objects import LLMLogs from kairon.shared.models import CognitionDataType from kairon.shared.rest_client import AioRestClient from kairon.shared.utils import Utility @@ -260,3 +261,26 @@ async def __attach_similarity_prompt_if_enabled(self, query_embedding, context_p similarity_context = f"Instructions on how to use {similarity_prompt_name}:\n{extracted_values}\n{similarity_prompt_instructions}\n" context_prompt = f"{context_prompt}\n{similarity_context}" return context_prompt + + @staticmethod + def get_logs(bot: str, start_idx: int = 0, page_size: int = 10): + """ + Get all logs for data importer event. + @param bot: bot id. + @param start_idx: start index + @param page_size: page size + @return: list of logs. + """ + for log in LLMLogs.objects(metadata__bot=bot).order_by("-start_time").skip(start_idx).limit(page_size): + llm_log = log.to_mongo().to_dict() + llm_log.pop('_id') + yield llm_log + + @staticmethod + def get_row_count(bot: str): + """ + Gets the count of rows in a LLMLogs for a particular bot. + :param bot: bot id + :return: Count of rows + """ + return LLMLogs.objects(metadata__bot=bot).count() From b357b5e324aaff72adb2c48a6d8493b6d1ab162c Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 28 Jun 2024 15:45:20 +0530 Subject: [PATCH 54/57] removed unused import --- kairon/api/app/routers/bot/bot.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kairon/api/app/routers/bot/bot.py b/kairon/api/app/routers/bot/bot.py index 276404377..839ae0fe8 100644 --- a/kairon/api/app/routers/bot/bot.py +++ b/kairon/api/app/routers/bot/bot.py @@ -39,12 +39,10 @@ from kairon.shared.importer.data_objects import ValidationLogs from kairon.shared.importer.processor import DataImporterLogProcessor from kairon.shared.live_agent.live_agent import LiveAgentHandler +from kairon.shared.llm.processor import LLMProcessor from kairon.shared.models import User, TemplateType from kairon.shared.test.processor import ModelTestingLogProcessor from kairon.shared.utils import Utility -from kairon.shared.llm.processor import LLMProcessor -from kairon.shared.llm.data_objects import LLMLogs - router = APIRouter() v2 = APIRouter() From 43d3c853a75ec0c3d5c972dbc8e06b15711f22cb Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Fri, 28 Jun 2024 20:07:49 +0530 Subject: [PATCH 55/57] added invocation in metadata for litellm --- kairon/actions/definitions/prompt.py | 1 + kairon/shared/llm/processor.py | 25 +++++--- kairon/shared/vector_embeddings/db/qdrant.py | 2 +- kairon/train.py | 2 +- tests/integration_test/action_service_test.py | 24 +++---- tests/unit_test/llm_test.py | 62 +++++++++---------- 6 files changed, 62 insertions(+), 54 deletions(-) diff --git a/kairon/actions/definitions/prompt.py b/kairon/actions/definitions/prompt.py index 6441d607f..2d3b5257b 100644 --- a/kairon/actions/definitions/prompt.py +++ b/kairon/actions/definitions/prompt.py @@ -71,6 +71,7 @@ async def execute(self, dispatcher: CollectingDispatcher, tracker: Tracker, doma llm_processor = LLMProcessor(self.bot, llm_type) llm_response, time_taken_llm_response = await llm_processor.predict(user_msg, user=tracker.sender_id, + invocation='prompt_action', **llm_params) status = "FAILURE" if llm_response.get("is_failure", False) is True else status exception = llm_response.get("exception") diff --git a/kairon/shared/llm/processor.py b/kairon/shared/llm/processor.py index 7365e0aa7..5b6831fe3 100644 --- a/kairon/shared/llm/processor.py +++ b/kairon/shared/llm/processor.py @@ -42,6 +42,7 @@ def __init__(self, bot: Text, llm_type: str): self.__logs = [] async def train(self, user, *args, **kwargs) -> Dict: + invocation = kwargs.pop('invocation', None) await self.__delete_collections() count = 0 processor = CognitionDataProcessor() @@ -61,7 +62,7 @@ async def train(self, user, *args, **kwargs) -> Dict: content['data'], metadata) else: search_payload, embedding_payload = {'content': content["data"]}, content["data"] - embeddings = await self.get_embedding(embedding_payload, user) + embeddings = await self.get_embedding(embedding_payload, user, invocation=invocation) points = [{'id': content['vector_id'], 'vector': embeddings, 'payload': search_payload}] await self.__collection_upsert__(collection, {'points': points}, err_msg="Unable to train FAQ! Contact support") @@ -71,15 +72,16 @@ async def train(self, user, *args, **kwargs) -> Dict: async def predict(self, query: Text, user, *args, **kwargs) -> Tuple: start_time = time.time() embeddings_created = False + invocation = kwargs.pop('invocation', None) try: - query_embedding = await self.get_embedding(query, user) + query_embedding = await self.get_embedding(query, user, invocation=invocation) embeddings_created = True system_prompt = kwargs.pop('system_prompt', DEFAULT_SYSTEM_PROMPT) context_prompt = kwargs.pop('context_prompt', DEFAULT_CONTEXT_PROMPT) context = await self.__attach_similarity_prompt_if_enabled(query_embedding, context_prompt, **kwargs) - answer = await self.__get_answer(query, system_prompt, context, user, **kwargs) + answer = await self.__get_answer(query, system_prompt, context, user, invocation=invocation,**kwargs) response = {"content": answer} except Exception as e: logging.exception(e) @@ -101,11 +103,11 @@ def truncate_text(self, text: Text) -> Text: tokens = self.tokenizer.encode(text)[:self.EMBEDDING_CTX_LENGTH] return self.tokenizer.decode(tokens) - async def get_embedding(self, text: Text, user) -> List[float]: + async def get_embedding(self, text: Text, user, **kwargs) -> List[float]: truncated_text = self.truncate_text(text) result = await litellm.aembedding(model="text-embedding-3-small", input=[truncated_text], - metadata={'user': user, 'bot': self.bot}, + metadata={'user': user, 'bot': self.bot, 'invocation': kwargs.get("invocation")}, api_key=self.api_key, num_retries=3) return result["data"][0]["embedding"] @@ -123,7 +125,7 @@ async def __parse_completion_response(self, response, **kwargs): async def __get_completion(self, messages, hyperparameters, user, **kwargs): response = await litellm.acompletion(messages=messages, - metadata={'user': user, 'bot': self.bot}, + metadata={'user': user, 'bot': self.bot, 'invocation': kwargs.get("invocation")}, api_key=self.api_key, num_retries=3, **hyperparameters) @@ -134,6 +136,7 @@ async def __get_completion(self, messages, hyperparameters, user, **kwargs): async def __get_answer(self, query, system_prompt: Text, context: Text, user, **kwargs): use_query_prompt = False query_prompt = '' + invocation = kwargs.pop('invocation') if kwargs.get('query_prompt', {}): query_prompt_dict = kwargs.pop('query_prompt') query_prompt = query_prompt_dict.get('query_prompt', '') @@ -146,7 +149,8 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, ** if use_query_prompt and query_prompt: query = await self.__rephrase_query(query, system_prompt, query_prompt, hyperparameters=hyperparameters, - user=user) + user=user, + invocation=f"{invocation}_rephrase") messages = [ {"role": "system", "content": system_prompt}, ] @@ -157,12 +161,14 @@ async def __get_answer(self, query, system_prompt: Text, context: Text, user, ** completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user) + user=user, + invocation=invocation) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'answer_query', 'hyperparameters': hyperparameters}) return completion async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, user, **kwargs): + invocation = kwargs.pop('invocation') messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": f"{query_prompt}\n\n Q: {query}\n A:"} @@ -171,7 +177,8 @@ async def __rephrase_query(self, query, system_prompt: Text, query_prompt: Text, completion, raw_response = await self.__get_completion(messages=messages, hyperparameters=hyperparameters, - user=user) + user=user, + invocation=invocation) self.__logs.append({'messages': messages, 'raw_completion_response': raw_response, 'type': 'rephrase_query', 'hyperparameters': hyperparameters}) return completion diff --git a/kairon/shared/vector_embeddings/db/qdrant.py b/kairon/shared/vector_embeddings/db/qdrant.py index 454eeb8fe..400e3103c 100644 --- a/kairon/shared/vector_embeddings/db/qdrant.py +++ b/kairon/shared/vector_embeddings/db/qdrant.py @@ -28,7 +28,7 @@ def __init__(self, bot: Text, collection_name: Text, llm_settings: dict, db_url: self.EMBEDDING_CTX_LENGTH = 8191 async def __get_embedding(self, text: Text, user: str, **kwargs) -> List[float]: - return await self.llm.get_embedding(text, user=user) + return await self.llm.get_embedding(text, user=user, invocation='db_action_qdrant') async def embedding_search(self, request_body: Dict, user: str, **kwargs): url = urljoin(self.db_url, f"/collections/{self.collection_name}/points") diff --git a/kairon/train.py b/kairon/train.py index d8360ce67..aa19943cb 100644 --- a/kairon/train.py +++ b/kairon/train.py @@ -102,7 +102,7 @@ def start_training(bot: str, user: str, token: str = None): settings = settings.to_mongo().to_dict() if settings["llm_settings"]['enable_faq']: llm_processor = LLMProcessor(bot, DEFAULT_LLM) - faqs = asyncio.run(llm_processor.train(user=user)) + faqs = asyncio.run(llm_processor.train(user=user, invocation='model_training')) account = AccountProcessor.get_bot(bot)['account'] MeteringProcessor.add_metrics(bot=bot, metric_type=MetricType.faq_training.value, account=account, **faqs) agent_url = Utility.environment['model']['agent'].get('url') diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index cd5e2c63c..43f5e93ca 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -10495,7 +10495,7 @@ def test_prompt_action_response_action_with_prompt_question_from_slot(mock_searc {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], - 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2l', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10562,7 +10562,7 @@ def test_prompt_action_response_action_with_bot_responses(mock_search, mock_embe {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], - 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b698ca10d35d2k', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10632,7 +10632,7 @@ def test_prompt_action_response_action_with_bot_responses_with_instructions(mock {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nAnswer in a short way.\nKeep it simple. \nQ: What kind of language is python? \nA:"}], - 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandey', 'bot': '5f50fd0a56b678ca10d35d2k', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -10884,7 +10884,7 @@ def test_prompt_response_action_streaming_enabled(mock_search, mock_embedding, m ] expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above.\n \nQ: What kind of language is python? \nA:"}], - 'metadata': {'user': 'udit.pandeyy', 'bot': '5f50k90a56b698ca10d35d2e'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandeyy', 'bot': '5f50k90a56b698ca10d35d2e', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stream': True, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args.kwargs, expected, ignore_order=True) @@ -11167,7 +11167,7 @@ def __mock_fetch_similar(*args, **kwargs): assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Python Prompt:\nA programming language is a system of notation for writing computer programs.[1] Most programming languages are text-based formal languages, but they may also be graphical. They are a kind of computer language.\nInstructions on how to use Python Prompt:\nAnswer according to the context\n\nJava Prompt:\nJava is a programming language and computing platform first released by Sun Microsystems in 1995.\nInstructions on how to use Java Prompt:\nAnswer according to the context\n\nAction Prompt:\nPython is a scripting language because it uses an interpreter to translate and run its code.\nInstructions on how to use Action Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What kind of language is python? \nA:"}], - 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'nupur.khare', 'bot': '5u08kd0a56b698ca10d98e6s', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11252,7 +11252,7 @@ def mock_completion_for_answer(*args, **kwargs): assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': 'Google search Prompt:\nKanban visualizes both the process (the workflow) and the actual work passing through that process.\nTo know more, please visit: Kanban\n\nKanban project management is one of the emerging PM methodologies, and the Kanban approach is suitable for every team and goal.\nTo know more, please visit: Kanban Project management\n\nKanban is a popular framework used to implement agile and DevOps software development.\nTo know more, please visit: Kanban agile\nInstructions on how to use Google search Prompt:\nAnswer according to the context\n\n \nQ: What is kanban \nA:'}], - 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'test_user', 'bot': '5u08kd0a56b698ca10hgjgjkhgjks', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11376,7 +11376,7 @@ def __mock_fetch_similar(*args, **kwargs): assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], - 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhj', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11483,7 +11483,7 @@ def mock_completion_for_answer(*args, **kwargs): assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': 'Qdrant Prompt:\nConvert user questions into json requests in qdrant such that they will either filter, apply range queries and search the payload in qdrant. Sample payload present in qdrant looks like below with each of the points starting with 1 to 5 is a record in qdrant.1. {"Category (Risk, Issue, Action Item)": "Risk", "Date Added": 1673721000.0,2. {"Category (Risk, Issue, Action Item)": "Action Item", "Date Added": 1673721000.0,For eg: to find category of record created on 15/01/2023, the filter query is:{"filter": {"must": [{"key": "Date Added", "match": {"value": 1673721000.0}}]}}\nInstructions on how to use Qdrant Prompt:\nCreate qdrant filter query out of user message based on above instructions.\n\n \nQ: category of record created on 15/01/2023? \nA:'}], - 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2sjhjhjhj', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11581,7 +11581,7 @@ def __mock_fetch_similar(*args, **kwargs): assert not DeepDiff(log['llm_logs'], expected, ignore_order=True) expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "Language Prompt:\nPython is an interpreted, object-oriented, high-level programming language with dynamic semantics.\nInstructions on how to use Language Prompt:\nAnswer according to the context\n\n\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: What is the name of prompt? \nA:"}], - 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s'}, 'api_key': 'keyvalue', + 'metadata': {'user': 'udit.pandey', 'bot': '5u80fd0a56c908ca10d35d2s', 'invocation': 'prompt_action'}, 'api_key': 'keyvalue', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11645,7 +11645,7 @@ def __mock_fetch_similar(*args, **kwargs): ] expected = {'messages': [{'role': 'system', 'content': 'You are a personal assistant.\n'}, {'role': 'user', 'content': "\nInstructions on how to use Similarity Prompt:\n['Scrum teams using Kanban as a visual management tool can get work delivered faster and more often. Prioritized tasks are completed first as the team collectively decides what is best using visual cues from the Kanban board. The best part is that Scrum teams can use Kanban and Scrum at the same time.']\nAnswer question based on the context above, if answer is not in the context go check previous logs.\n \nQ: Kanban And Scrum Together? \nA:"}], - 'metadata': {'user': user, 'bot': bot}, 'api_key': value, + 'metadata': {'user': user, 'bot': bot, 'invocation': 'prompt_action'}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11709,7 +11709,7 @@ def test_prompt_action_response_action_when_similarity_is_empty(mock_search, moc {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], - 'metadata': {'user': 'udit.pandey', 'bot': bot}, 'api_key': value, + 'metadata': {'user': 'udit.pandey', 'bot': bot, 'invocation': 'prompt_action'}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -11776,7 +11776,7 @@ def test_prompt_action_response_action_when_similarity_disabled(mock_search, moc {'role': 'system', 'content': 'You are a personal assistant. Answer question based on the context below.\n'}, {'role': 'user', 'content': 'hello'}, {'role': 'assistant', 'content': 'how are you'}, {'role': 'user', 'content': ' \nQ: What kind of language is python? \nA:'}], - 'metadata': {'user': user, 'bot': bot}, 'api_key': value, + 'metadata': {'user': user, 'bot': bot, 'invocation': 'prompt_action'}, 'api_key': value, 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) diff --git a/tests/unit_test/llm_test.py b/tests/unit_test/llm_test.py index 53c31bace..bf6b54d50 100644 --- a/tests/unit_test/llm_test.py +++ b/tests/unit_test/llm_test.py @@ -80,7 +80,7 @@ async def test_gpt3_faq_embedding_train(self, mock_embedding, aioresponses): }]} expected = {"model": "text-embedding-3-small", - "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, + "input": [test_content.data], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -198,7 +198,7 @@ async def test_gpt3_faq_embedding_train_payload_text(self, mock_embedding, aiore assert response['faq'] == 3 expected = {"model": "text-embedding-3-small", - "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, + "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} print(mock_embedding.call_args) @@ -260,7 +260,7 @@ async def test_gpt3_faq_embedding_train_payload_with_int(self, mock_embedding, a }]} expected = {"model": "text-embedding-3-small", - "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot}, + "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -332,7 +332,7 @@ async def test_gpt3_faq_embedding_train_int(self, mock_embedding, aioresponses): }]} expected = {"model": "text-embedding-3-small", - "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot}, + "input": [json.dumps(input)], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -393,7 +393,7 @@ async def test_gpt3_faq_embedding_train_upsert_error(self, mock_embedding, aiore 'vector': embedding, 'payload': {'content': test_content.data}}]} expected = {"model": "text-embedding-3-small", - "input": [test_content.data], 'metadata': {'user': user, 'bot': bot}, + "input": [test_content.data], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -464,7 +464,7 @@ async def test_gpt3_faq_embedding_train_payload_upsert_error_json(self, mock_emb }]} expected = {"model": "text-embedding-3-small", - "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot}, + "input": [json.dumps(test_content.data)], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -525,12 +525,12 @@ async def test_gpt3_faq_embedding_predict(self, mock_embedding, mock_completion, assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': bot} + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} expected['api_key'] = value expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -594,12 +594,12 @@ async def test_gpt3_faq_embedding_predict_with_default_collection(self, mock_emb assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": value, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': bot} + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} expected['api_key'] = value expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -674,12 +674,12 @@ async def test_gpt3_faq_embedding_predict_with_values(self, mock_embedding, mock assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot}, + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': gpt3.bot} + expected['metadata'] = {'user': user, 'bot': gpt3.bot, 'invocation': None} expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -761,12 +761,12 @@ async def test_gpt3_faq_embedding_predict_with_values_and_stream(self, mock_embe assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot}, + "input": [query], 'metadata': {'user': user, 'bot': gpt3.bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': gpt3.bot} + expected['metadata'] = {'user': user, 'bot': gpt3.bot, 'invocation': None} expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -863,12 +863,12 @@ async def test_gpt3_faq_embedding_predict_with_values_with_instructions(self, assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': bot} + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -929,7 +929,7 @@ def __mock_connection_error(*args, **kwargs): assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -937,7 +937,7 @@ def __mock_connection_error(*args, **kwargs): 'content': 'You are a personal assistant. Answer the question according to the below context'}, {'role': 'user', 'content': "Based on below context answer question, if answer not in context check previous logs.\nInstructions on how to use Similarity Prompt:\n['Python is a high-level, general-purpose programming language. Its design philosophy emphasizes code readability with the use of significant indentation. Python is dynamically typed and garbage-collected.']\nAnswer according to this context.\n \nQ: What kind of language is python? \nA:"}], - 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error'}, + 'metadata': {'user': 'test', 'bot': 'test_gpt3_faq_embedding_predict_completion_connection_error', 'invocation': None}, 'api_key': 'test', 'num_retries': 3, 'temperature': 0.0, 'max_tokens': 300, 'model': 'gpt-3.5-turbo', 'top_p': 0.0, 'n': 1, 'stop': None, 'presence_penalty': 0.0, 'frequency_penalty': 0.0, 'logit_bias': {}} @@ -981,7 +981,7 @@ async def test_gpt3_faq_embedding_predict_exact_match(self, mock_embedding, mock assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -1019,7 +1019,7 @@ async def test_gpt3_faq_embedding_predict_embedding_connection_error(self, mock_ assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) @@ -1084,12 +1084,12 @@ async def test_gpt3_faq_embedding_predict_with_previous_bot_responses(self, mock assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': bot} + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -1162,12 +1162,12 @@ async def test_gpt3_faq_embedding_predict_with_query_prompt(self, mock_embedding assert isinstance(time_elapsed, float) and time_elapsed > 0.0 expected = {"model": "text-embedding-3-small", - "input": [query], 'metadata': {'user': user, 'bot': bot}, + "input": [query], 'metadata': {'user': user, 'bot': bot, 'invocation': None}, "api_key": key, "num_retries": 3} assert not DeepDiff(mock_embedding.call_args[1], expected, ignore_order=True) expected = mock_completion_request.copy() - expected['metadata'] = {'user': user, 'bot': bot} + expected['metadata'] = {'user': user, 'bot': bot, 'invocation': None} expected['api_key'] = key expected['num_retries'] = 3 assert not DeepDiff(mock_completion.call_args[1], expected, ignore_order=True) @@ -1185,20 +1185,20 @@ async def test_llm_logging(self): result = await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", mock_response=expected, - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) assert result['choices'][0]['message']['content'] == expected result = litellm.completion(messages=messages, model="gpt-3.5-turbo", mock_response=expected, - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) assert result['choices'][0]['message']['content'] == expected result = litellm.completion(messages=messages, model="gpt-3.5-turbo", mock_response=expected, stream=True, - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) response = '' for chunk in result: content = chunk["choices"][0]["delta"]["content"] @@ -1211,7 +1211,7 @@ async def test_llm_logging(self): model="gpt-3.5-turbo", mock_response=expected, stream=True, - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) response = '' async for chunk in result: content = chunk["choices"][0]["delta"]["content"] @@ -1225,7 +1225,7 @@ async def test_llm_logging(self): await litellm.acompletion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) assert str(e) == "Authentication error" @@ -1233,7 +1233,7 @@ async def test_llm_logging(self): litellm.completion(messages=messages, model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) assert str(e) == "Authentication error" @@ -1242,6 +1242,6 @@ async def test_llm_logging(self): model="gpt-3.5-turbo", mock_response=Exception("Authentication error"), stream=True, - metadata={'user': user, 'bot': bot}) + metadata={'user': user, 'bot': bot, 'invocation': None}) assert str(e) == "Authentication error" \ No newline at end of file From da13b69087ca7dd3f187c0be9af7360bcde54a2c Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Mon, 1 Jul 2024 17:19:25 +0530 Subject: [PATCH 56/57] 1. changed rasa rule policy to allow max history 2. changed rasa domain.yml schemas to allow unicode Alphabets for slots and form name --- docker/Dockerfile | 2 + kairon/shared/schemas/domain.yml | 142 +++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 kairon/shared/schemas/domain.yml diff --git a/docker/Dockerfile b/docker/Dockerfile index 58f48e6ba..4b9cadd30 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,6 +38,8 @@ COPY . . RUN rm -rf ${TEMPLATE_DIR_DEFAULT}/models/* && \ rasa train --data ${TEMPLATE_DIR_DEFAULT}/data --config ${TEMPLATE_DIR_DEFAULT}/config.yml --domain ${TEMPLATE_DIR_DEFAULT}/domain.yml --out ${TEMPLATE_DIR_DEFAULT}/models +RUN cp kairon/shared/rule_policy.py /usr/local/lib/python3.10/site-packages/rasa/core/policies/rule_policy.py +RUN cp kairon/shared/schemas/domain.yml /usr/local/lib/python3.10/site-packages/rasa/shared/utils/schemas/domain.yml ENV HF_HOME="/home/cache" SENTENCE_TRANSFORMERS_HOME="/home/cache" diff --git a/kairon/shared/schemas/domain.yml b/kairon/shared/schemas/domain.yml new file mode 100644 index 000000000..b5ceed4c3 --- /dev/null +++ b/kairon/shared/schemas/domain.yml @@ -0,0 +1,142 @@ +allowempty: True +mapping: + version: + type: "str" + required: False + allowempty: False + intents: + type: "seq" + sequence: + - type: "map" + mapping: + use_entities: + type: "any" + ignore_entities: + type: "any" + allowempty: True + - type: "str" + entities: + type: "seq" + matching: "any" + sequence: + - type: "map" + mapping: + roles: + type: "seq" + sequence: + - type: "str" + groups: + type: "seq" + sequence: + - type: "str" + allowempty: True + - type: "str" + actions: + type: seq + matching: "any" + seq: + - type: str + - type: map + mapping: + regex;([A-Za-z]+): + type: map + mapping: + send_domain: + type: "bool" + responses: + # see shared/nlu/training_data/schemas/responses.yml + include: responses + + slots: + type: "map" + allowempty: True + mapping: + regex;([A-Za-z\u00C0-\u017F\u0400-\u04FF\u0370-\u03FF\u0530-\u058F\u0590-\u05FF\u0600-\u06FF\u0980-\u09FF\u0A80-\u0AFF\u0B80-\u0BFF\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF]+): + type: "map" + allowempty: True + mapping: + influence_conversation: + type: "bool" + required: False + type: + type: "any" + required: True + values: + type: "seq" + sequence: + - type: "any" + required: False + min_value: + type: "number" + required: False + max_value: + type: "number" + required: False + initial_value: + type: "any" + required: False + mappings: + type: "seq" + required: True + allowempty: False + sequence: + - type: "map" + allowempty: True + mapping: + type: + type: "str" + intent: + type: "any" + not_intent: + type: "any" + entity: + type: "str" + role: + type: "str" + group: + type: "str" + value: + type: "any" + action: + type: "str" + conditions: + type: "seq" + sequence: + - type: "map" + mapping: + active_loop: + type: "str" + nullable: True + requested_slot: + type: "str" + forms: + type: "map" + required: False + mapping: + regex;([A-Za-z\u00C0-\u017F\u0400-\u04FF\u0370-\u03FF\u0530-\u058F\u0590-\u05FF\u0600-\u06FF\u0980-\u09FF\u0A80-\u0AFF\u0B80-\u0BFF\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF]+): + type: "map" + mapping: + required_slots: + type: "seq" + sequence: + - type: str + required: False + allowempty: True + ignored_intents: + type: any + config: + type: "map" + allowempty: True + mapping: + store_entities_as_slots: + type: "bool" + session_config: + type: "map" + allowempty: True + mapping: + session_expiration_time: + type: "number" + range: + min: 0 + carry_over_slots_to_new_session: + type: "bool" From a309505c28f35033daebc12270abfed344d53cf6 Mon Sep 17 00:00:00 2001 From: Fahad Ali Shaikh Date: Wed, 3 Jul 2024 13:49:20 +0530 Subject: [PATCH 57/57] test cases fixed after merging --- kairon/api/models.py | 26 +- kairon/shared/utils.py | 2 +- tests/integration_test/action_service_test.py | 879 +++++++++++++++++- .../data_processor/data_processor_test.py | 89 +- 4 files changed, 954 insertions(+), 42 deletions(-) diff --git a/kairon/api/models.py b/kairon/api/models.py index 789873fff..d61a98126 100644 --- a/kairon/api/models.py +++ b/kairon/api/models.py @@ -903,16 +903,38 @@ def validate_top_n(cls, v, values, **kwargs): return v +class CustomActionParameterModel(BaseModel): + value: Any = None + parameter_type: ActionParameterType = ActionParameterType.value + + @validator("parameter_type") + def validate_parameter_type(cls, v, values, **kwargs): + allowed_values = {ActionParameterType.value, ActionParameterType.slot} + if v not in allowed_values: + raise ValueError(f"Invalid parameter type. Allowed values: {allowed_values}") + return v + + @root_validator + def check(cls, values): + if values.get('parameter_type') == ActionParameterType.slot and not values.get('value'): + raise ValueError("Provide name of the slot as value") + + if values.get('parameter_type') == ActionParameterType.value and not isinstance(values.get('value'), list): + raise ValueError("Provide list of emails as value") + + return values + + class EmailActionRequest(BaseModel): action_name: constr(to_lower=True, strip_whitespace=True) smtp_url: str smtp_port: int smtp_userid: CustomActionParameter = None smtp_password: CustomActionParameter - from_email: str + from_email: CustomActionParameter subject: str custom_text: CustomActionParameter = None - to_email: List[str] + to_email: CustomActionParameterModel response: str tls: bool = False diff --git a/kairon/shared/utils.py b/kairon/shared/utils.py index acf3ff92a..bbc47527f 100644 --- a/kairon/shared/utils.py +++ b/kairon/shared/utils.py @@ -2052,7 +2052,7 @@ def verify_email(email: Text): @staticmethod def get_llms(): - return Utility.system_metadata["llm"].keys() + return Utility.system_metadata.get("llm", {}).keys() @staticmethod def get_default_llm_hyperparameters(): diff --git a/tests/integration_test/action_service_test.py b/tests/integration_test/action_service_test.py index 43f5e93ca..6933be2f0 100644 --- a/tests/integration_test/action_service_test.py +++ b/tests/integration_test/action_service_test.py @@ -24,7 +24,8 @@ EmailActionConfig, ActionServerLogs, GoogleSearchAction, JiraAction, ZendeskAction, PipedriveLeadsAction, SetSlots, \ HubspotFormsAction, HttpActionResponse, HttpActionRequestBody, SetSlotsFromResponse, CustomActionRequestParameters, \ KaironTwoStageFallbackAction, TwoStageFallbackTextualRecommendations, RazorpayAction, PromptAction, FormSlotSet, \ - DatabaseAction, DbQuery, PyscriptActionConfig, WebSearchAction, UserQuestion, LiveAgentActionConfig + DatabaseAction, DbQuery, PyscriptActionConfig, WebSearchAction, UserQuestion, LiveAgentActionConfig, \ + CustomActionParameters from kairon.shared.actions.exception import ActionFailure from kairon.shared.actions.models import ActionType, ActionParameterType, DispatchType, DbActionOperationType, \ DbQueryValueType @@ -5563,9 +5564,9 @@ def test_email_action_execution_script_evaluation(mock_smtp, mock_action_config, smtp_url="test.localhost", smtp_port=293, smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), - from_email="test@demo.com", - subject="test script", - to_email=["test@test.com"], + from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), + to_email=CustomActionParameters(value=["test@test.com"], parameter_type="value"), + subject="test", response="Email Triggered", custom_text=CustomActionRequestParameters(key="custom_text", value="custom_mail_text", parameter_type=ActionParameterType.slot.value), @@ -5617,14 +5618,14 @@ def _get_action_config(*arge, **kwargs): assert {} == kwargs from_email, password = args - assert from_email == action_config.from_email + assert from_email == action_config.from_email.value assert password == action_config.smtp_password.value name, args, kwargs = mock_smtp.method_calls.pop(0) assert name == '().sendmail' assert {} == kwargs - assert args[0] == action_config.from_email + assert args[0] == action_config.from_email.value assert args[1] == ["test@test.com"] assert str(args[2]).__contains__(action_config.subject) assert str(args[2]).__contains__("Content-Type: text/html") @@ -5650,9 +5651,9 @@ def test_email_action_execution(mock_smtp, mock_action_config, mock_action): smtp_url="test.localhost", smtp_port=293, smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), - from_email="test@demo.com", + from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), + to_email=CustomActionParameters(value=["test@test.com"], parameter_type="value"), subject="test", - to_email=["test@test.com"], response="Email Triggered", bot="bot", user="user" @@ -5768,20 +5769,866 @@ def _get_action_config(*arge, **kwargs): assert {} == kwargs from_email, password = args - assert from_email == action_config.from_email + assert from_email == action_config.from_email.value assert password == action_config.smtp_password.value name, args, kwargs = mock_smtp.method_calls.pop(0) assert name == '().sendmail' assert {} == kwargs - assert args[0] == action_config.from_email + assert args[0] == action_config.from_email.value assert args[1] == ["test@test.com"] assert str(args[2]).__contains__(action_config.subject) assert str(args[2]).__contains__("Content-Type: text/html") assert str(args[2]).__contains__("Subject: default test") +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) +def test_email_action_execution_with_sender_email_from_slot(mock_smtp, mock_action_config, mock_action): + Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', + 'rb').read().decode() + Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( + 'template/emails/bot_msg_conversation.html', 'rb').read().decode() + Utility.email_conf['email']['templates']['user_msg_conversation'] = open( + 'template/emails/user_msg_conversation.html', 'rb').read().decode() + Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', + 'rb').read().decode() + action_name = "test_email_action_execution_with_sender_email_from_slot" + action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") + action_config = EmailActionConfig( + action_name=action_name, + smtp_url="test.localhost", + smtp_port=293, + smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), + from_email=CustomActionRequestParameters(value="from_email", parameter_type="slot"), + to_email=CustomActionParameters(value=["test@test.com"], parameter_type="value"), + subject="test", + response="Email Triggered", + bot="bot", + user="user" + ) + + def _get_action(*arge, **kwargs): + return action.to_mongo().to_dict() + + def _get_action_config(*arge, **kwargs): + return action_config.to_mongo().to_dict() + + request_object = { + "next_action": action_name, + "tracker": { + "sender_id": "mahesh.sattala", + "conversation_id": "default", + "slots": {"bot": "5f50fd0a56b698ca10d35d2e", "requested_slot": "from_email", + "from_email": "mahesh@gmail.com"}, + "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, + "latest_event_time": 1537645578.314389, + "followup_action": "action_listen", + "paused": False, + "events": [ + {"event": "action", "timestamp": 1594907100.12764, "name": "action_session_start", "policy": None, + "confidence": None}, {"event": "session_started", "timestamp": 1594907100.12765}, + {"event": "action", "timestamp": 1594907100.12767, "name": "action_listen", "policy": None, + "confidence": None}, {"event": "user", "timestamp": 1594907100.42744, "text": "can't", + "parse_data": { + "intent": {"name": "test intent", "confidence": 0.253578245639801}, + "entities": [], "intent_ranking": [ + {"name": "test intent", "confidence": 0.253578245639801}, + {"name": "goodbye", "confidence": 0.1504897326231}, + {"name": "greet", "confidence": 0.138640150427818}, + {"name": "affirm", "confidence": 0.0857767835259438}, + {"name": "smalltalk_human", "confidence": 0.0721133947372437}, + {"name": "deny", "confidence": 0.069614589214325}, + {"name": "bot_challenge", "confidence": 0.0664894133806229}, + {"name": "faq_vaccine", "confidence": 0.062177762389183}, + {"name": "faq_testing", "confidence": 0.0530692934989929}, + {"name": "out_of_scope", "confidence": 0.0480506233870983}], + "response_selector": { + "default": {"response": {"name": None, "confidence": 0}, + "ranking": [], "full_retrieval_intent": None}}, + "text": "can't"}, "input_channel": None, + "message_id": "bbd413bf5c834bf3b98e0da2373553b2", "metadata": {}}, + {"event": "action", "timestamp": 1594907100.4308, "name": "utter_test intent", + "policy": "policy_0_MemoizationPolicy", "confidence": 1}, + {"event": "bot", "timestamp": 1594907100.4308, "text": "will not = won\"t", + "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, + "image": None, "custom": None}, "metadata": {}}, + {"event": "action", "timestamp": 1594907100.43384, "name": "action_listen", + "policy": "policy_0_MemoizationPolicy", "confidence": 1}, + {"event": "user", "timestamp": 1594907117.04194, "text": "can\"t", + "parse_data": {"intent": {"name": "test intent", "confidence": 0.253578245639801}, "entities": [], + "intent_ranking": [{"name": "test intent", "confidence": 0.253578245639801}, + {"name": "goodbye", "confidence": 0.1504897326231}, + {"name": "greet", "confidence": 0.138640150427818}, + {"name": "affirm", "confidence": 0.0857767835259438}, + {"name": "smalltalk_human", "confidence": 0.0721133947372437}, + {"name": "deny", "confidence": 0.069614589214325}, + {"name": "bot_challenge", "confidence": 0.0664894133806229}, + {"name": "faq_vaccine", "confidence": 0.062177762389183}, + {"name": "faq_testing", "confidence": 0.0530692934989929}, + {"name": "out_of_scope", "confidence": 0.0480506233870983}], + "response_selector": { + "default": {"response": {"name": None, "confidence": 0}, "ranking": [], + "full_retrieval_intent": None}}, "text": "can\"t"}, + "input_channel": None, "message_id": "e96e2a85de0748798748385503c65fb3", "metadata": {}}, + {"event": "action", "timestamp": 1594907117.04547, "name": "utter_test intent", + "policy": "policy_1_TEDPolicy", "confidence": 0.978452920913696}, + {"event": "bot", "timestamp": 1594907117.04548, "text": "can not = can't", + "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, + "image": None, "custom": None}, "metadata": {}}], + "latest_input_channel": "rest", + "active_loop": {}, + "latest_action": {}, + }, + "domain": { + "config": {}, + "session_config": {}, + "intents": [], + "entities": [], + "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, + "responses": {}, + "actions": [], + "forms": {}, + "e2e_actions": [] + }, + "version": "version" + } + mock_action.side_effect = _get_action + mock_action_config.side_effect = _get_action_config + response = client.post("/webhook", json=request_object) + response_json = response.json() + assert response.status_code == 200 + assert len(response_json['events']) == 1 + assert len(response_json['responses']) == 1 + assert response_json['events'] == [ + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "Email Triggered"}] + assert response_json['responses'][0]['text'] == "Email Triggered" + logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() + assert logs.status == "SUCCESS" + + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().connect' + assert {} == kwargs + + host, port = args + assert host == action_config.smtp_url + assert port == action_config.smtp_port + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().login' + assert {} == kwargs + + from_email, password = args + assert from_email == 'mahesh@gmail.com' + assert password == action_config.smtp_password.value + + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().sendmail' + assert {} == kwargs + + assert args[0] == 'mahesh@gmail.com' + assert args[1] == ["test@test.com"] + assert str(args[2]).__contains__(action_config.subject) + assert str(args[2]).__contains__("Content-Type: text/html") + assert str(args[2]).__contains__("Subject: mahesh.sattala test") + + +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) +def test_email_action_execution_with_receiver_email_list_from_slot(mock_smtp, mock_action_config, mock_action): + Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', + 'rb').read().decode() + Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( + 'template/emails/bot_msg_conversation.html', 'rb').read().decode() + Utility.email_conf['email']['templates']['user_msg_conversation'] = open( + 'template/emails/user_msg_conversation.html', 'rb').read().decode() + Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', + 'rb').read().decode() + + action_name = "test_email_action_execution_with_receiver_email_list_from_slot" + action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") + action_config = EmailActionConfig( + action_name=action_name, + smtp_url="test.localhost", + smtp_port=293, + smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), + from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), + to_email=CustomActionParameters(value="to_email", parameter_type="slot"), + subject="test", + response="Email Triggered", + bot="bot", + user="user" + ) + + def _get_action(*arge, **kwargs): + return action.to_mongo().to_dict() + + def _get_action_config(*arge, **kwargs): + return action_config.to_mongo().to_dict() + + request_object = { + "next_action": action_name, + "tracker": { + "sender_id": "mahesh.sattala", + "conversation_id": "default", + "slots": {"bot": "5f50fd0a56b698ca10d35d2e", "requested_slot": "to_email", + "to_email": ["test@gmail.com"]}, + "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, + "latest_event_time": 1537645578.314389, + "followup_action": "action_listen", + "paused": False, + "events": [ + {"event": "action", "timestamp": 1594907100.12764, "name": "action_session_start", "policy": None, + "confidence": None}, {"event": "session_started", "timestamp": 1594907100.12765}, + {"event": "action", "timestamp": 1594907100.12767, "name": "action_listen", "policy": None, + "confidence": None}, {"event": "user", "timestamp": 1594907100.42744, "text": "can't", + "parse_data": { + "intent": {"name": "test intent", "confidence": 0.253578245639801}, + "entities": [], "intent_ranking": [ + {"name": "test intent", "confidence": 0.253578245639801}, + {"name": "goodbye", "confidence": 0.1504897326231}, + {"name": "greet", "confidence": 0.138640150427818}, + {"name": "affirm", "confidence": 0.0857767835259438}, + {"name": "smalltalk_human", "confidence": 0.0721133947372437}, + {"name": "deny", "confidence": 0.069614589214325}, + {"name": "bot_challenge", "confidence": 0.0664894133806229}, + {"name": "faq_vaccine", "confidence": 0.062177762389183}, + {"name": "faq_testing", "confidence": 0.0530692934989929}, + {"name": "out_of_scope", "confidence": 0.0480506233870983}], + "response_selector": { + "default": {"response": {"name": None, "confidence": 0}, + "ranking": [], "full_retrieval_intent": None}}, + "text": "can't"}, "input_channel": None, + "message_id": "bbd413bf5c834bf3b98e0da2373553b2", "metadata": {}}, + {"event": "action", "timestamp": 1594907100.4308, "name": "utter_test intent", + "policy": "policy_0_MemoizationPolicy", "confidence": 1}, + {"event": "bot", "timestamp": 1594907100.4308, "text": "will not = won\"t", + "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, + "image": None, "custom": None}, "metadata": {}}, + {"event": "action", "timestamp": 1594907100.43384, "name": "action_listen", + "policy": "policy_0_MemoizationPolicy", "confidence": 1}, + {"event": "user", "timestamp": 1594907117.04194, "text": "can\"t", + "parse_data": {"intent": {"name": "test intent", "confidence": 0.253578245639801}, "entities": [], + "intent_ranking": [{"name": "test intent", "confidence": 0.253578245639801}, + {"name": "goodbye", "confidence": 0.1504897326231}, + {"name": "greet", "confidence": 0.138640150427818}, + {"name": "affirm", "confidence": 0.0857767835259438}, + {"name": "smalltalk_human", "confidence": 0.0721133947372437}, + {"name": "deny", "confidence": 0.069614589214325}, + {"name": "bot_challenge", "confidence": 0.0664894133806229}, + {"name": "faq_vaccine", "confidence": 0.062177762389183}, + {"name": "faq_testing", "confidence": 0.0530692934989929}, + {"name": "out_of_scope", "confidence": 0.0480506233870983}], + "response_selector": { + "default": {"response": {"name": None, "confidence": 0}, "ranking": [], + "full_retrieval_intent": None}}, "text": "can\"t"}, + "input_channel": None, "message_id": "e96e2a85de0748798748385503c65fb3", "metadata": {}}, + {"event": "action", "timestamp": 1594907117.04547, "name": "utter_test intent", + "policy": "policy_1_TEDPolicy", "confidence": 0.978452920913696}, + {"event": "bot", "timestamp": 1594907117.04548, "text": "can not = can't", + "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, + "image": None, "custom": None}, "metadata": {}}], + "latest_input_channel": "rest", + "active_loop": {}, + "latest_action": {}, + }, + "domain": { + "config": {}, + "session_config": {}, + "intents": [], + "entities": [], + "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, + "responses": {}, + "actions": [], + "forms": {}, + "e2e_actions": [] + }, + "version": "version" + } + mock_action.side_effect = _get_action + mock_action_config.side_effect = _get_action_config + response = client.post("/webhook", json=request_object) + response_json = response.json() + assert response.status_code == 200 + assert len(response_json['events']) == 1 + assert len(response_json['responses']) == 1 + assert response_json['events'] == [ + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "Email Triggered"}] + assert response_json['responses'][0]['text'] == "Email Triggered" + logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() + assert logs.status == "SUCCESS" + + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().connect' + assert {} == kwargs + + host, port = args + assert host == action_config.smtp_url + assert port == action_config.smtp_port + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().login' + assert {} == kwargs + + from_email, password = args + assert from_email == action_config.from_email.value + assert password == action_config.smtp_password.value + + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().sendmail' + assert {} == kwargs + + assert args[0] == action_config.from_email.value + assert args[1] == ["test@gmail.com"] + assert str(args[2]).__contains__(action_config.subject) + assert str(args[2]).__contains__("Content-Type: text/html") + assert str(args[2]).__contains__("Subject: mahesh.sattala test") + + +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) +def test_email_action_execution_with_single_receiver_email_from_slot(mock_smtp, mock_action_config, mock_action): + Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', + 'rb').read().decode() + Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( + 'template/emails/bot_msg_conversation.html', 'rb').read().decode() + Utility.email_conf['email']['templates']['user_msg_conversation'] = open( + 'template/emails/user_msg_conversation.html', 'rb').read().decode() + Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', + 'rb').read().decode() + + action_name = "test_email_action_execution_with_single_receiver_email_from_slot" + action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") + action_config = EmailActionConfig( + action_name=action_name, + smtp_url="test.localhost", + smtp_port=293, + smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), + from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), + to_email=CustomActionParameters(value="to_email", parameter_type="slot"), + subject="test", + response="Email Triggered", + bot="bot", + user="user" + ) + + def _get_action(*arge, **kwargs): + return action.to_mongo().to_dict() + + def _get_action_config(*arge, **kwargs): + return action_config.to_mongo().to_dict() + + request_object = { + "next_action": action_name, + "tracker": { + "sender_id": "mahesh.sattala", + "conversation_id": "default", + "slots": {"bot": "5f50fd0a56b698ca10d35d2e", "requested_slot": "to_email", + "to_email": "example@gmail.com"}, + "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, + "latest_event_time": 1537645578.314389, + "followup_action": "action_listen", + "paused": False, + "events": [ + {"event": "action", "timestamp": 1594907100.12764, "name": "action_session_start", "policy": None, + "confidence": None}, {"event": "session_started", "timestamp": 1594907100.12765}, + {"event": "action", "timestamp": 1594907100.12767, "name": "action_listen", "policy": None, + "confidence": None}, {"event": "user", "timestamp": 1594907100.42744, "text": "can't", + "parse_data": { + "intent": {"name": "test intent", "confidence": 0.253578245639801}, + "entities": [], "intent_ranking": [ + {"name": "test intent", "confidence": 0.253578245639801}, + {"name": "goodbye", "confidence": 0.1504897326231}, + {"name": "greet", "confidence": 0.138640150427818}, + {"name": "affirm", "confidence": 0.0857767835259438}, + {"name": "smalltalk_human", "confidence": 0.0721133947372437}, + {"name": "deny", "confidence": 0.069614589214325}, + {"name": "bot_challenge", "confidence": 0.0664894133806229}, + {"name": "faq_vaccine", "confidence": 0.062177762389183}, + {"name": "faq_testing", "confidence": 0.0530692934989929}, + {"name": "out_of_scope", "confidence": 0.0480506233870983}], + "response_selector": { + "default": {"response": {"name": None, "confidence": 0}, + "ranking": [], "full_retrieval_intent": None}}, + "text": "can't"}, "input_channel": None, + "message_id": "bbd413bf5c834bf3b98e0da2373553b2", "metadata": {}}, + {"event": "action", "timestamp": 1594907100.4308, "name": "utter_test intent", + "policy": "policy_0_MemoizationPolicy", "confidence": 1}, + {"event": "bot", "timestamp": 1594907100.4308, "text": "will not = won\"t", + "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, + "image": None, "custom": None}, "metadata": {}}, + {"event": "action", "timestamp": 1594907100.43384, "name": "action_listen", + "policy": "policy_0_MemoizationPolicy", "confidence": 1}, + {"event": "user", "timestamp": 1594907117.04194, "text": "can\"t", + "parse_data": {"intent": {"name": "test intent", "confidence": 0.253578245639801}, "entities": [], + "intent_ranking": [{"name": "test intent", "confidence": 0.253578245639801}, + {"name": "goodbye", "confidence": 0.1504897326231}, + {"name": "greet", "confidence": 0.138640150427818}, + {"name": "affirm", "confidence": 0.0857767835259438}, + {"name": "smalltalk_human", "confidence": 0.0721133947372437}, + {"name": "deny", "confidence": 0.069614589214325}, + {"name": "bot_challenge", "confidence": 0.0664894133806229}, + {"name": "faq_vaccine", "confidence": 0.062177762389183}, + {"name": "faq_testing", "confidence": 0.0530692934989929}, + {"name": "out_of_scope", "confidence": 0.0480506233870983}], + "response_selector": { + "default": {"response": {"name": None, "confidence": 0}, "ranking": [], + "full_retrieval_intent": None}}, "text": "can\"t"}, + "input_channel": None, "message_id": "e96e2a85de0748798748385503c65fb3", "metadata": {}}, + {"event": "action", "timestamp": 1594907117.04547, "name": "utter_test intent", + "policy": "policy_1_TEDPolicy", "confidence": 0.978452920913696}, + {"event": "bot", "timestamp": 1594907117.04548, "text": "can not = can't", + "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, + "image": None, "custom": None}, "metadata": {}}], + "latest_input_channel": "rest", + "active_loop": {}, + "latest_action": {}, + }, + "domain": { + "config": {}, + "session_config": {}, + "intents": [], + "entities": [], + "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, + "responses": {}, + "actions": [], + "forms": {}, + "e2e_actions": [] + }, + "version": "version" + } + mock_action.side_effect = _get_action + mock_action_config.side_effect = _get_action_config + response = client.post("/webhook", json=request_object) + response_json = response.json() + assert response.status_code == 200 + assert len(response_json['events']) == 1 + assert len(response_json['responses']) == 1 + assert response_json['events'] == [ + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "Email Triggered"}] + assert response_json['responses'][0]['text'] == "Email Triggered" + logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() + assert logs.status == "SUCCESS" + + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().connect' + assert {} == kwargs + + host, port = args + assert host == action_config.smtp_url + assert port == action_config.smtp_port + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().login' + assert {} == kwargs + + from_email, password = args + assert from_email == action_config.from_email.value + assert password == action_config.smtp_password.value + + name, args, kwargs = mock_smtp.method_calls.pop(0) + assert name == '().sendmail' + assert {} == kwargs + + assert args[0] == action_config.from_email.value + assert args[1] == ["example@gmail.com"] + assert str(args[2]).__contains__(action_config.subject) + assert str(args[2]).__contains__("Content-Type: text/html") + assert str(args[2]).__contains__("Subject: mahesh.sattala test") + + +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) +def test_email_action_execution_with_invalid_from_email(mock_smtp, mock_action_config, mock_action): + Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', + 'rb').read().decode() + Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( + 'template/emails/bot_msg_conversation.html', 'rb').read().decode() + Utility.email_conf['email']['templates']['user_msg_conversation'] = open( + 'template/emails/user_msg_conversation.html', 'rb').read().decode() + Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', + 'rb').read().decode() + + action_name = "test_email_action_execution_with_invalid_from_email" + action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") + action_config = EmailActionConfig( + action_name=action_name, + smtp_url="test.localhost", + smtp_port=293, + smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), + from_email=CustomActionRequestParameters(value=["test@demo.com"], parameter_type="value"), + to_email=CustomActionParameters(value="to_email", parameter_type="slot"), + subject="test", + response="Email Triggered", + bot="bot", + user="user" + ) + + def _get_action(*arge, **kwargs): + return action.to_mongo().to_dict() + + def _get_action_config(*arge, **kwargs): + return action_config.to_mongo().to_dict() + + request_object = { + "next_action": action_name, + "tracker": { + "sender_id": "mahesh.sattala", + "conversation_id": "default", + "slots": {"bot": "5f50fd0a56b698ca10d35d2e", "requested_slot": "to_email", + "to_email": "example@gmail.com"}, + "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, + "latest_event_time": 1537645578.314389, + "followup_action": "action_listen", + "paused": False, + "events": [ + {"event": "action", "timestamp": 1594907100.12764, "name": "action_session_start", "policy": None, + "confidence": None}, {"event": "session_started", "timestamp": 1594907100.12765}, + {"event": "action", "timestamp": 1594907100.12767, "name": "action_listen", "policy": None, + "confidence": None}, {"event": "user", "timestamp": 1594907100.42744, "text": "can't", + "parse_data": { + "intent": {"name": "test intent", "confidence": 0.253578245639801}, + "entities": [], "intent_ranking": [ + {"name": "test intent", "confidence": 0.253578245639801}, + {"name": "goodbye", "confidence": 0.1504897326231}, + {"name": "greet", "confidence": 0.138640150427818}, + {"name": "affirm", "confidence": 0.0857767835259438}, + {"name": "smalltalk_human", "confidence": 0.0721133947372437}, + {"name": "deny", "confidence": 0.069614589214325}, + {"name": "bot_challenge", "confidence": 0.0664894133806229}, + {"name": "faq_vaccine", "confidence": 0.062177762389183}, + {"name": "faq_testing", "confidence": 0.0530692934989929}, + {"name": "out_of_scope", "confidence": 0.0480506233870983}], + "response_selector": { + "default": {"response": {"name": None, "confidence": 0}, + "ranking": [], "full_retrieval_intent": None}}, + "text": "can't"}, "input_channel": None, + "message_id": "bbd413bf5c834bf3b98e0da2373553b2", "metadata": {}}, + {"event": "action", "timestamp": 1594907100.4308, "name": "utter_test intent", + "policy": "policy_0_MemoizationPolicy", "confidence": 1}, + {"event": "bot", "timestamp": 1594907100.4308, "text": "will not = won\"t", + "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, + "image": None, "custom": None}, "metadata": {}}, + {"event": "action", "timestamp": 1594907100.43384, "name": "action_listen", + "policy": "policy_0_MemoizationPolicy", "confidence": 1}, + {"event": "user", "timestamp": 1594907117.04194, "text": "can\"t", + "parse_data": {"intent": {"name": "test intent", "confidence": 0.253578245639801}, "entities": [], + "intent_ranking": [{"name": "test intent", "confidence": 0.253578245639801}, + {"name": "goodbye", "confidence": 0.1504897326231}, + {"name": "greet", "confidence": 0.138640150427818}, + {"name": "affirm", "confidence": 0.0857767835259438}, + {"name": "smalltalk_human", "confidence": 0.0721133947372437}, + {"name": "deny", "confidence": 0.069614589214325}, + {"name": "bot_challenge", "confidence": 0.0664894133806229}, + {"name": "faq_vaccine", "confidence": 0.062177762389183}, + {"name": "faq_testing", "confidence": 0.0530692934989929}, + {"name": "out_of_scope", "confidence": 0.0480506233870983}], + "response_selector": { + "default": {"response": {"name": None, "confidence": 0}, "ranking": [], + "full_retrieval_intent": None}}, "text": "can\"t"}, + "input_channel": None, "message_id": "e96e2a85de0748798748385503c65fb3", "metadata": {}}, + {"event": "action", "timestamp": 1594907117.04547, "name": "utter_test intent", + "policy": "policy_1_TEDPolicy", "confidence": 0.978452920913696}, + {"event": "bot", "timestamp": 1594907117.04548, "text": "can not = can't", + "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, + "image": None, "custom": None}, "metadata": {}}], + "latest_input_channel": "rest", + "active_loop": {}, + "latest_action": {}, + }, + "domain": { + "config": {}, + "session_config": {}, + "intents": [], + "entities": [], + "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, + "responses": {}, + "actions": [], + "forms": {}, + "e2e_actions": [] + }, + "version": "version" + } + mock_action.side_effect = _get_action + mock_action_config.side_effect = _get_action_config + response = client.post("/webhook", json=request_object) + response_json = response.json() + assert response.status_code == 200 + assert len(response_json['events']) == 1 + assert len(response_json['responses']) == 1 + assert response_json['events'] == [ + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "I have failed to process your request"}] + assert response_json['responses'][0]['text'] == "I have failed to process your request" + logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() + assert logs.status == "FAILURE" + assert logs.exception == "Invalid 'from_email' type. It must be of type str." + + +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) +def test_email_action_execution_with_invalid_to_email(mock_smtp, mock_action_config, mock_action): + Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', + 'rb').read().decode() + Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( + 'template/emails/bot_msg_conversation.html', 'rb').read().decode() + Utility.email_conf['email']['templates']['user_msg_conversation'] = open( + 'template/emails/user_msg_conversation.html', 'rb').read().decode() + Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', + 'rb').read().decode() + + action_name = "test_email_action_execution_with_invalid_to_email" + action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") + action_config = EmailActionConfig( + action_name=action_name, + smtp_url="test.localhost", + smtp_port=293, + smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), + from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), + to_email=CustomActionParameters(value={"to_email": "test@test.com"}, parameter_type="value"), + subject="test", + response="Email Triggered", + bot="bot", + user="user" + ) + + def _get_action(*arge, **kwargs): + return action.to_mongo().to_dict() + + def _get_action_config(*arge, **kwargs): + return action_config.to_mongo().to_dict() + + request_object = { + "next_action": action_name, + "tracker": { + "sender_id": "mahesh.sattala", + "conversation_id": "default", + "slots": {"bot": "5f50fd0a56b698ca10d35d2e", "requested_slot": "to_email", + "to_email": "example@gmail.com"}, + "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, + "latest_event_time": 1537645578.314389, + "followup_action": "action_listen", + "paused": False, + "events": [ + {"event": "action", "timestamp": 1594907100.12764, "name": "action_session_start", "policy": None, + "confidence": None}, {"event": "session_started", "timestamp": 1594907100.12765}, + {"event": "action", "timestamp": 1594907100.12767, "name": "action_listen", "policy": None, + "confidence": None}, {"event": "user", "timestamp": 1594907100.42744, "text": "can't", + "parse_data": { + "intent": {"name": "test intent", "confidence": 0.253578245639801}, + "entities": [], "intent_ranking": [ + {"name": "test intent", "confidence": 0.253578245639801}, + {"name": "goodbye", "confidence": 0.1504897326231}, + {"name": "greet", "confidence": 0.138640150427818}, + {"name": "affirm", "confidence": 0.0857767835259438}, + {"name": "smalltalk_human", "confidence": 0.0721133947372437}, + {"name": "deny", "confidence": 0.069614589214325}, + {"name": "bot_challenge", "confidence": 0.0664894133806229}, + {"name": "faq_vaccine", "confidence": 0.062177762389183}, + {"name": "faq_testing", "confidence": 0.0530692934989929}, + {"name": "out_of_scope", "confidence": 0.0480506233870983}], + "response_selector": { + "default": {"response": {"name": None, "confidence": 0}, + "ranking": [], "full_retrieval_intent": None}}, + "text": "can't"}, "input_channel": None, + "message_id": "bbd413bf5c834bf3b98e0da2373553b2", "metadata": {}}, + {"event": "action", "timestamp": 1594907100.4308, "name": "utter_test intent", + "policy": "policy_0_MemoizationPolicy", "confidence": 1}, + {"event": "bot", "timestamp": 1594907100.4308, "text": "will not = won\"t", + "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, + "image": None, "custom": None}, "metadata": {}}, + {"event": "action", "timestamp": 1594907100.43384, "name": "action_listen", + "policy": "policy_0_MemoizationPolicy", "confidence": 1}, + {"event": "user", "timestamp": 1594907117.04194, "text": "can\"t", + "parse_data": {"intent": {"name": "test intent", "confidence": 0.253578245639801}, "entities": [], + "intent_ranking": [{"name": "test intent", "confidence": 0.253578245639801}, + {"name": "goodbye", "confidence": 0.1504897326231}, + {"name": "greet", "confidence": 0.138640150427818}, + {"name": "affirm", "confidence": 0.0857767835259438}, + {"name": "smalltalk_human", "confidence": 0.0721133947372437}, + {"name": "deny", "confidence": 0.069614589214325}, + {"name": "bot_challenge", "confidence": 0.0664894133806229}, + {"name": "faq_vaccine", "confidence": 0.062177762389183}, + {"name": "faq_testing", "confidence": 0.0530692934989929}, + {"name": "out_of_scope", "confidence": 0.0480506233870983}], + "response_selector": { + "default": {"response": {"name": None, "confidence": 0}, "ranking": [], + "full_retrieval_intent": None}}, "text": "can\"t"}, + "input_channel": None, "message_id": "e96e2a85de0748798748385503c65fb3", "metadata": {}}, + {"event": "action", "timestamp": 1594907117.04547, "name": "utter_test intent", + "policy": "policy_1_TEDPolicy", "confidence": 0.978452920913696}, + {"event": "bot", "timestamp": 1594907117.04548, "text": "can not = can't", + "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, + "image": None, "custom": None}, "metadata": {}}], + "latest_input_channel": "rest", + "active_loop": {}, + "latest_action": {}, + }, + "domain": { + "config": {}, + "session_config": {}, + "intents": [], + "entities": [], + "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, + "responses": {}, + "actions": [], + "forms": {}, + "e2e_actions": [] + }, + "version": "version" + } + mock_action.side_effect = _get_action + mock_action_config.side_effect = _get_action_config + response = client.post("/webhook", json=request_object) + response_json = response.json() + assert response.status_code == 200 + assert len(response_json['events']) == 1 + assert len(response_json['responses']) == 1 + assert response_json['events'] == [ + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "I have failed to process your request"}] + assert response_json['responses'][0]['text'] == "I have failed to process your request" + logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() + assert logs.status == "FAILURE" + assert logs.exception == "Invalid 'from_email' type. It must be of type str." + + +@mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") +@mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") +@mock.patch("kairon.shared.utils.SMTP", autospec=True) +def test_email_action_execution_with_invalid_to_email(mock_smtp, mock_action_config, mock_action): + Utility.email_conf['email']['templates']['conversation'] = open('template/emails/conversation.html', + 'rb').read().decode() + Utility.email_conf['email']['templates']['bot_msg_conversation'] = open( + 'template/emails/bot_msg_conversation.html', 'rb').read().decode() + Utility.email_conf['email']['templates']['user_msg_conversation'] = open( + 'template/emails/user_msg_conversation.html', 'rb').read().decode() + Utility.email_conf['email']['templates']['button_template'] = open('template/emails/button.html', + 'rb').read().decode() + + action_name = "test_email_action_execution_with_invalid_to_email" + action = Actions(name=action_name, type=ActionType.email_action.value, bot="bot", user="user") + action_config = EmailActionConfig( + action_name=action_name, + smtp_url="test.localhost", + smtp_port=293, + smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), + from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), + to_email=CustomActionParameters(value={"to_email": "test@test.com"}, parameter_type="value"), + subject="test", + response="Email Triggered", + bot="bot", + user="user" + ) + + def _get_action(*arge, **kwargs): + return action.to_mongo().to_dict() + + def _get_action_config(*arge, **kwargs): + return action_config.to_mongo().to_dict() + + request_object = { + "next_action": action_name, + "tracker": { + "sender_id": "mahesh.sattala", + "conversation_id": "default", + "slots": {"bot": "5f50fd0a56b698ca10d35d2e", "requested_slot": "to_email", + "to_email": "example@gmail.com"}, + "latest_message": {'text': 'get intents', 'intent_ranking': [{'name': 'test_run'}]}, + "latest_event_time": 1537645578.314389, + "followup_action": "action_listen", + "paused": False, + "events": [ + {"event": "action", "timestamp": 1594907100.12764, "name": "action_session_start", "policy": None, + "confidence": None}, {"event": "session_started", "timestamp": 1594907100.12765}, + {"event": "action", "timestamp": 1594907100.12767, "name": "action_listen", "policy": None, + "confidence": None}, {"event": "user", "timestamp": 1594907100.42744, "text": "can't", + "parse_data": { + "intent": {"name": "test intent", "confidence": 0.253578245639801}, + "entities": [], "intent_ranking": [ + {"name": "test intent", "confidence": 0.253578245639801}, + {"name": "goodbye", "confidence": 0.1504897326231}, + {"name": "greet", "confidence": 0.138640150427818}, + {"name": "affirm", "confidence": 0.0857767835259438}, + {"name": "smalltalk_human", "confidence": 0.0721133947372437}, + {"name": "deny", "confidence": 0.069614589214325}, + {"name": "bot_challenge", "confidence": 0.0664894133806229}, + {"name": "faq_vaccine", "confidence": 0.062177762389183}, + {"name": "faq_testing", "confidence": 0.0530692934989929}, + {"name": "out_of_scope", "confidence": 0.0480506233870983}], + "response_selector": { + "default": {"response": {"name": None, "confidence": 0}, + "ranking": [], "full_retrieval_intent": None}}, + "text": "can't"}, "input_channel": None, + "message_id": "bbd413bf5c834bf3b98e0da2373553b2", "metadata": {}}, + {"event": "action", "timestamp": 1594907100.4308, "name": "utter_test intent", + "policy": "policy_0_MemoizationPolicy", "confidence": 1}, + {"event": "bot", "timestamp": 1594907100.4308, "text": "will not = won\"t", + "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, + "image": None, "custom": None}, "metadata": {}}, + {"event": "action", "timestamp": 1594907100.43384, "name": "action_listen", + "policy": "policy_0_MemoizationPolicy", "confidence": 1}, + {"event": "user", "timestamp": 1594907117.04194, "text": "can\"t", + "parse_data": {"intent": {"name": "test intent", "confidence": 0.253578245639801}, "entities": [], + "intent_ranking": [{"name": "test intent", "confidence": 0.253578245639801}, + {"name": "goodbye", "confidence": 0.1504897326231}, + {"name": "greet", "confidence": 0.138640150427818}, + {"name": "affirm", "confidence": 0.0857767835259438}, + {"name": "smalltalk_human", "confidence": 0.0721133947372437}, + {"name": "deny", "confidence": 0.069614589214325}, + {"name": "bot_challenge", "confidence": 0.0664894133806229}, + {"name": "faq_vaccine", "confidence": 0.062177762389183}, + {"name": "faq_testing", "confidence": 0.0530692934989929}, + {"name": "out_of_scope", "confidence": 0.0480506233870983}], + "response_selector": { + "default": {"response": {"name": None, "confidence": 0}, "ranking": [], + "full_retrieval_intent": None}}, "text": "can\"t"}, + "input_channel": None, "message_id": "e96e2a85de0748798748385503c65fb3", "metadata": {}}, + {"event": "action", "timestamp": 1594907117.04547, "name": "utter_test intent", + "policy": "policy_1_TEDPolicy", "confidence": 0.978452920913696}, + {"event": "bot", "timestamp": 1594907117.04548, "text": "can not = can't", + "data": {"elements": None, "quick_replies": None, "buttons": None, "attachment": None, + "image": None, "custom": None}, "metadata": {}}], + "latest_input_channel": "rest", + "active_loop": {}, + "latest_action": {}, + }, + "domain": { + "config": {}, + "session_config": {}, + "intents": [], + "entities": [], + "slots": {"bot": "5f50fd0a56b698ca10d35d2e"}, + "responses": {}, + "actions": [], + "forms": {}, + "e2e_actions": [] + }, + "version": "version" + } + mock_action.side_effect = _get_action + mock_action_config.side_effect = _get_action_config + response = client.post("/webhook", json=request_object) + response_json = response.json() + assert response.status_code == 200 + assert len(response_json['events']) == 1 + assert len(response_json['responses']) == 1 + assert response_json['events'] == [ + {'event': 'slot', 'timestamp': None, 'name': 'kairon_action_response', + 'value': "I have failed to process your request"}] + assert response_json['responses'][0]['text'] == "I have failed to process your request" + logs = ActionServerLogs.objects(type=ActionType.email_action.value).order_by("-id").first() + print(logs.to_mongo().to_dict()) + assert logs.status == "FAILURE" + assert logs.exception == "Invalid 'to_email' type. It must be of type str or list." + + @mock.patch("kairon.shared.actions.utils.ActionUtility.get_action") @mock.patch("kairon.actions.definitions.email.ActionEmail.retrieve_config") @mock.patch("kairon.shared.utils.SMTP", autospec=True) @@ -5802,9 +6649,9 @@ def test_email_action_execution_varied_utterances(mock_smtp, mock_action_config, smtp_url="test.localhost", smtp_port=293, smtp_password=CustomActionRequestParameters(key='smtp_password', value="test"), - from_email="test@demo.com", + from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), + to_email=CustomActionParameters(value=["test@test.com"], parameter_type="value"), subject="test", - to_email=["test@test.com"], response="Email Triggered", bot="bot", user="user" @@ -6126,14 +6973,14 @@ def _get_action_config(*arge, **kwargs): assert {} == kwargs from_email, password = args - assert from_email == action_config.from_email + assert from_email == action_config.from_email.value assert password == action_config.smtp_password.value name, args, kwargs = mock_smtp.method_calls.pop(0) assert name == '().sendmail' assert {} == kwargs - assert args[0] == action_config.from_email + assert args[0] == action_config.from_email.value assert args[1] == ["test@test.com"] assert str(args[2]).__contains__(action_config.subject) assert str(args[2]).__contains__("Content-Type: text/html") @@ -6248,9 +7095,9 @@ def test_email_action_failed_execution(mock_action_config, mock_action): smtp_url="test.localhost", smtp_port=293, smtp_password=CustomActionRequestParameters(value="test"), - from_email="test@demo.com", + from_email=CustomActionRequestParameters(value="test@demo.com", parameter_type="value"), + to_email=CustomActionParameters(value="test@test.com", parameter_type="value"), subject="test", - to_email="test@test.com", response="Email Triggered", bot="bot", user="user" diff --git a/tests/unit_test/data_processor/data_processor_test.py b/tests/unit_test/data_processor/data_processor_test.py index c79165738..484fc75ac 100644 --- a/tests/unit_test/data_processor/data_processor_test.py +++ b/tests/unit_test/data_processor/data_processor_test.py @@ -80,6 +80,7 @@ from kairon.shared.data.training_data_generation_processor import TrainingDataGenerationProcessor from kairon.shared.data.utils import DataUtility from kairon.shared.importer.processor import DataImporterLogProcessor +from kairon.shared.live_agent.live_agent import LiveAgentHandler from kairon.shared.metering.constants import MetricType from kairon.shared.metering.data_object import Metering from kairon.shared.models import StoryEventType, HttpContentType, CognitionDataType @@ -6492,6 +6493,7 @@ def _mock_bot_info(*args, **kwargs): actual_config.config.pop('live_agent_socket_url') headers = actual_config.config.pop('headers') expected_config['multilingual'] = {'enable': False, 'bots': []} + expected_config['live_agent_enabled'] = True assert expected_config == actual_config.config primary_token_claims = Utility.decode_limited_access_token(headers['authorization']['access_token']) @@ -6510,6 +6512,22 @@ def _mock_bot_info(*args, **kwargs): '/api/bot/.+/metric/user/logs/user_metrics' ], 'access-limit': ['/api/auth/.+/token/refresh']} + def test_get_chat_client_config_live_agent_enabled_false(self, monkeypatch): + def _mock_bot_info(*args, **kwargs): + return { + "_id": "9876543210", 'name': 'test_bot', 'account': 2, 'user': 'user@integration.com', + 'status': True, + "metadata": {"source_bot_id": None} + } + def _mock_is_live_agent_service_available(*args, **kwargs): + return False + monkeypatch.setattr(AccountProcessor, 'get_bot', _mock_bot_info) + monkeypatch.setattr(LiveAgentHandler, 'is_live_agent_service_available', _mock_is_live_agent_service_available) + processor = MongoProcessor() + actual_config = processor.get_chat_client_config('test_bot', 'user@integration.com') + assert actual_config.config['live_agent_enabled'] == False + + def test_save_chat_client_config_without_whitelisted_domain(self, monkeypatch): def _mock_bot_info(*args, **kwargs): return {'name': 'test', 'account': 1, 'user': 'user@integration.com', 'status': True} @@ -13199,8 +13217,8 @@ def test_add_email_action(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": "test@demo.com", - "to_email": ["test@test.com", "test1@test.com"], + "from_email": {"value": "from_email", "parameter_type": "slot"}, + "to_email": {"value": ["test@test.com", "test1@test.com"], "parameter_type": "value"}, "subject": "Test Subject", "response": "Test Response", "tls": False @@ -13215,8 +13233,8 @@ def test_add_email_action_with_custom_text(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": "test@demo.com", - "to_email": ["test@test.com", "test1@test.com"], + "from_email": {"value": "from_email", "parameter_type": "slot"}, + "to_email": {"value": ["test@test.com", "test1@test.com"], "parameter_type": "value"}, "subject": "Test Subject", "response": "Test Response", "tls": False, @@ -13253,8 +13271,8 @@ def test_add_email_action_validation_error(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": "test@demo.com", - "to_email": "test@test.com", + "from_email": {"value": "from_email", "parameter_type": "slot"}, + "to_email": {"value": ["test@test.com", "test1@test.com"], "parameter_type": "value"}, "subject": "Test Subject", "response": "Test Response", "tls": False @@ -13278,16 +13296,29 @@ def test_add_email_action_validation_error(self): email_config['smtp_url'] = temp temp = email_config['from_email'] - email_config['from_email'] = "test@test" + email_config['from_email'] = {"value": "test@test", "parameter_type": "value"} with pytest.raises(ValidationError, match="Invalid From or To email address"): processor.add_email_action(email_config, "TEST", "tests") + + email_config['from_email'] = {"value": "", "parameter_type": "slot"} + with pytest.raises(ValidationError, match="Provide name of the slot as value"): + processor.add_email_action(email_config, "TEST", "tests") email_config['from_email'] = temp temp = email_config['to_email'] - email_config['to_email'] = "test@test" + email_config['to_email'] = {"value": "test@test", "parameter_type": "value"} + with pytest.raises(ValidationError, match="Provide list of emails as value"): + processor.add_email_action(email_config, "TEST", "tests") + + email_config['to_email'] = {"value": ["test@test"], "parameter_type": "value"} with pytest.raises(ValidationError, match="Invalid From or To email address"): processor.add_email_action(email_config, "TEST", "tests") - email_config['to_email'] = ["test@demo.com"] + + email_config['to_email'] = {"value": "", "parameter_type": "slot"} + with pytest.raises(ValidationError, match="Provide name of the slot as value"): + processor.add_email_action(email_config, "TEST", "tests") + email_config['to_email'] = temp + email_config["custom_text"] = {"value": "custom_text_slot", "parameter_type": "sender_id"} with pytest.raises(ValidationError, match="custom_text can only be of type value or slot!"): processor.add_email_action(email_config, "TEST", "tests") @@ -13299,8 +13330,8 @@ def test_add_email_action_duplicate(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": "test@demo.com", - "to_email": ["test@test.com"], + "from_email": {"value": "from_email", "parameter_type": "slot"}, + "to_email": {"value": ["test@test.com", "test1@test.com"], "parameter_type": "value"}, "subject": "Test Subject", "response": "Test Response", "tls": False @@ -13316,8 +13347,8 @@ def test_add_email_action_existing_name(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": "test@demo.com", - "to_email": ["test@test.com"], + "from_email": {"value": "test@demo.com", "parameter_type": "value"}, + "to_email": {"value": "to_email", "parameter_type": "slot"}, "subject": "Test Subject", "response": "Test Response", "tls": False @@ -13333,8 +13364,8 @@ def test_edit_email_action(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": "test@demo.com", - "to_email": ["test@test.com", "test1@test.com"], + "from_email": {"value": "test@demo.com", "parameter_type": "value"}, + "to_email": {"value": "to_email", "parameter_type": "slot"}, "subject": "Test Subject", "response": "Test Response", "tls": False @@ -13353,8 +13384,8 @@ def test_edit_email_action_validation_error(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": "test@demo.com", - "to_email": "test@test.com", + "from_email": {"value": "test@demo.com", "parameter_type": "value"}, + "to_email": {"value": "to_email", "parameter_type": "slot"}, "subject": "Test Subject", "response": "Test Response", "tls": False @@ -13378,15 +13409,27 @@ def test_edit_email_action_validation_error(self): email_config['smtp_url'] = temp temp = email_config['from_email'] - email_config['from_email'] = "test@test" + email_config['from_email'] = {"value": "test@demo", "parameter_type": "value"} with pytest.raises(ValidationError, match="Invalid From or To email address"): processor.edit_email_action(email_config, "TEST", "tests") + + email_config['from_email'] = {"value": "", "parameter_type": "slot"} + with pytest.raises(ValidationError, match="Provide name of the slot as value"): + processor.edit_email_action(email_config, "TEST", "tests") email_config['from_email'] = temp temp = email_config['to_email'] - email_config['to_email'] = "test@test" + email_config['to_email'] = {"value": "test@test", "parameter_type": "value"} + with pytest.raises(ValidationError, match="Provide list of emails as value"): + processor.edit_email_action(email_config, "TEST", "tests") + + email_config['to_email'] = {"value": ["test@test"], "parameter_type": "value"} with pytest.raises(ValidationError, match="Invalid From or To email address"): processor.edit_email_action(email_config, "TEST", "tests") + + email_config['to_email'] = {"value": "", "parameter_type": "slot"} + with pytest.raises(ValidationError, match="Provide name of the slot as value"): + processor.edit_email_action(email_config, "TEST", "tests") email_config['to_email'] = temp def test_edit_email_action_does_not_exist(self): @@ -13396,8 +13439,8 @@ def test_edit_email_action_does_not_exist(self): "smtp_port": 25, "smtp_userid": None, "smtp_password": {'value': "test"}, - "from_email": "test@demo.com", - "to_email": "test@test.com", + "from_email": {"value": "test@demo.com", "parameter_type": "value"}, + "to_email": {"value": "to_email", "parameter_type": "slot"}, "subject": "Test Subject", "response": "Test Response", "tls": False @@ -14251,8 +14294,8 @@ def test_delete_secret_attached_to_email_action(self): "smtp_port": 25, "smtp_userid": smtp_userid_list, "smtp_password": {'value': "test"}, - "from_email": "test@demo.com", - "to_email": ["test@test.com", "test1@test.com"], + "from_email": {"value": "test@demo.com", "parameter_type": "value"}, + "to_email": {"value": "to_email", "parameter_type": "slot"}, "subject": "Test Subject", "response": "Test Response", "tls": False