-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #252 from parea-ai/PAI-446-add-eval-metric-cookbook
feat: update cookbook for chat
- Loading branch information
Showing
9 changed files
with
154 additions
and
94 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1,2 @@ | ||
from .in_memory import InMemoryCache | ||
from .redis import RedisCache |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
from typing import List, Optional | ||
|
||
import json | ||
|
||
from attr import asdict | ||
|
||
from parea.cache.cache import Cache | ||
from parea.schemas.models import CacheRequest, TraceLog | ||
|
||
|
||
class InMemoryCache(Cache): | ||
def __init__(self): | ||
self.cache = {} | ||
self.logs = [] | ||
|
||
def get(self, key: CacheRequest) -> Optional[TraceLog]: | ||
return self.cache.get(json.dumps(asdict(key))) | ||
|
||
async def aget(self, key: CacheRequest) -> Optional[TraceLog]: | ||
return self.get(key) | ||
|
||
def set(self, key: CacheRequest, value: TraceLog): | ||
self.cache[json.dumps(asdict(key))] = value | ||
|
||
async def aset(self, key: CacheRequest, value: TraceLog): | ||
self.set(key, value) | ||
|
||
def invalidate(self, key: CacheRequest): | ||
key = json.dumps(asdict(key)) | ||
if key in self.cache: | ||
del self.cache[key] | ||
|
||
async def ainvalidate(self, key: CacheRequest): | ||
self.invalidate(key) | ||
|
||
def log(self, value: TraceLog): | ||
self.logs.append(value) | ||
|
||
def read_logs(self) -> List[TraceLog]: | ||
return self.logs.copy() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,40 +1,64 @@ | ||
from typing import Callable, Optional | ||
|
||
import json | ||
|
||
from parea.evals.utils import call_openai | ||
from parea.schemas.models import Log | ||
|
||
|
||
def goal_success_ratio(log: Log) -> float: | ||
"""Returns the average amount of turns the user had to converse with the AI to reach their goals.""" | ||
messages = [m.to_dict() for m in log.configuration.messages] | ||
messages.append({"role": "assistant", "content": log.output}) | ||
|
||
# need to determine where does a new goal start | ||
conversation_segments = [] | ||
start_index = 0 | ||
end_index = 3 | ||
while end_index < len(messages): | ||
user_follows_same_goal = call_openai( | ||
[ | ||
{ | ||
"role": "system", | ||
"content": "Look at the conversation and to determine if the user is still following the same goal " | ||
"or if they are following a new goal. If they are following the same goal, respond " | ||
"SAME_GOAL. Otherwise, respond NEW_GOAL. In any case do not answer the user request!", | ||
} | ||
] | ||
+ messages[start_index:end_index], | ||
model="gpt-4", | ||
) | ||
|
||
if user_follows_same_goal == "SAME_GOAL": | ||
end_index += 2 | ||
def goal_success_ratio_factory(use_output: Optional[bool] = False, message_field: Optional[str] = None) -> Callable[[Log], float]: | ||
"""Factory function that returns a function that calculates the goal success ratio of a log. | ||
Args: | ||
use_output (Optional[bool], optional): Whether to use the output of the log to access the messages. Defaults to False. | ||
message_field (Optional[str], optional): The name of the field in the log that contains the messages. | ||
Defaults to None. If None, the messages are taken from the configuration attribute. | ||
""" | ||
if use_output and message_field: | ||
raise ValueError("Only one of use_output and message_field can be set.") | ||
|
||
def goal_success_ratio(log: Log) -> float: | ||
"""Returns the average amount of turns the user had to converse with the AI to reach their goals.""" | ||
if use_output: | ||
output_list_dicts = json.loads(log.output) | ||
messages = [m for m in output_list_dicts] | ||
elif message_field: | ||
messages = [m for m in log.inputs[message_field]] | ||
else: | ||
conversation_segments.append(messages[start_index : end_index - 1]) | ||
start_index = end_index - 1 | ||
end_index += 2 | ||
messages = [m.to_dict() for m in log.configuration.messages] | ||
if log.output: | ||
messages.append({"role": "assistant", "content": log.output}) | ||
|
||
# need to determine where does a new goal start | ||
conversation_segments = [] | ||
start_index = 0 | ||
end_index = 3 | ||
while end_index < len(messages): | ||
user_follows_same_goal = call_openai( | ||
[ | ||
{ | ||
"role": "system", | ||
"content": "Look at the conversation and to determine if the user is still following the same goal " | ||
"or if they are following a new goal. If they are following the same goal, respond " | ||
"SAME_GOAL. Otherwise, respond NEW_GOAL. In any case do not answer the user request!", | ||
} | ||
] | ||
+ messages[start_index:end_index], | ||
model="gpt-4", | ||
) | ||
|
||
if user_follows_same_goal == "SAME_GOAL": | ||
end_index += 2 | ||
else: | ||
conversation_segments.append(messages[start_index : end_index - 1]) | ||
start_index = end_index - 1 | ||
end_index += 2 | ||
|
||
if start_index < len(messages): | ||
conversation_segments.append(messages[start_index:]) | ||
|
||
if start_index < len(messages): | ||
conversation_segments.append(messages[start_index:]) | ||
# for now assume that the user reached their goal in every segment | ||
# return the average amount of turns the user had to converse with the AI to reach their goals | ||
return sum([2 / len(segment) for segment in conversation_segments]) / len(conversation_segments) | ||
|
||
# for now assume that the user reached their goal in every segment | ||
# return the average amount of turns the user had to converse with the AI to reach their goals | ||
return sum([2 / len(segment) for segment in conversation_segments]) / len(conversation_segments) | ||
return goal_success_ratio |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api" | |
[tool.poetry] | ||
name = "parea-ai" | ||
packages = [{ include = "parea" }] | ||
version = "0.2.19" | ||
version = "0.2.20" | ||
description = "Parea python sdk" | ||
readme = "README.md" | ||
authors = ["joel-parea-ai <[email protected]>"] | ||
|