agents: update docs with slight prompt adjustment (#391)

Co-authored-by: Douglas Reid <[email protected]>
Co-authored-by: Ted Benson <[email protected]>

src/steamship/agents/examples/my_assistant.py

@@ -7,13 +7,18 @@
 from steamship.agents.schema import AgentContext
 from steamship.agents.schema.context import Metadata
 from steamship.agents.service.agent_service import AgentService
-from steamship.agents.tools.image_generation.dalle import DalleTool
-from steamship.agents.tools.search.search import SearchTool
+from steamship.agents.tools.image_generation import DalleTool
+from steamship.agents.tools.search import SearchTool
 from steamship.invocable import post
 from steamship.utils.repl import AgentREPL
 
 
 class MyAssistant(AgentService):
+    """MyAssistant is an example AgentService that exposes a single test endpoint
+    for trying out Agent-based invocations. It is configured with two simple Tools
+    to provide an overview of the types of tasks it can accomplish (here, search
+    and image generation)."""
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self._agent = ReACTAgent(
@@ -26,11 +31,23 @@ def __init__(self, **kwargs):
 
     @post("prompt")
     def prompt(self, prompt: str) -> str:
+        """Run an agent with the provided text as the input."""
+
+        # AgentContexts serve to allow the AgentService to run agents
+        # with appropriate information about the desired tasking.
+        # Here, we create a new context on each prompt, and append the
+        # prompt to the message history stored in the context.
         context_id = uuid.uuid4()
         context = AgentContext.get_or_create(self.client, {"id": f"{context_id}"})
         context.chat_history.append_user_message(prompt)
 
-        # TODO: is this preferred over taking the last step in completed step?
+        # AgentServices provide an emit function hook to access the output of running
+        # agents and tools. The emit functions fire at after the supplied agent emits
+        # a "FinishAction".
+        #
+        # Here, we show one way of accessing the output in a synchronous fashion. An
+        # alternative way would be to access the final Action in the `context.completed_steps`
+        # after the call to `run_agent()`.
         output = ""
 
         def sync_emit(blocks: List[Block], meta: Metadata):
@@ -46,4 +63,7 @@ def sync_emit(blocks: List[Block], meta: Metadata):
 
 
 if __name__ == "__main__":
+    # AgentREPL provides a mechanism for local execution of an AgentService method.
+    # This is used for simplified debugging as agents and tools are developed and
+    # added.
     AgentREPL(MyAssistant, "prompt", agent_package_config={}).run()

src/steamship/agents/react/react.py

@@ -24,24 +24,31 @@ class ReACTAgent(LLMAgent):
 Observation: the result of the action
 ```
 
+
 If you decide that you should use a Tool, you must generate the associated Action and Action Input.
 
-Some tools will return Observations in the format of `Block(string)`. This will represent a successful completion
+Some tools will return Observations in the format of `Block(<identifier>)`. This will represent a successful completion
 of that step and can be passed to subsequent tools, or returned to a user to answer their questions.
 
-If you have generated an image using a Tool in response to a user request and wish to return it to the user, please
-tell the user directly where to find the image. To do so, you MUST use the format:
+When you have a final response to say to the Human, or if you do not need to use a tool, you MUST use the format:
 
 ```
 Thought: Do I need to use a tool? No
-AI: Image available via: Block(<identifier>).
+AI: [your final response here]
 ```
 
-When you have a response to say to the Human, or if you do not need to use a tool, you MUST use the format:
+If a Tool generated an Observation that includes `Block(<identifier>)` and you wish to return it to the user, ALWAYS
+end your response with the `Block(<identifier>)` observation. To do so, you MUST use the format:
+
 ```
 Thought: Do I need to use a tool? No
-AI: [your response here]
+AI: [your response with a suffix of: "Block(<identifier>)"].
 ```
+
+Make sure to use all observations to come up with your final response.
+You MUST include `Block(<identifier>)` segments in responses that generate images or audio.
+DO NOT include `Block(<identifier>)` segments in responses that do not have generated images or audio.
+
 Begin!
 
 New input: {input}

src/steamship/agents/schema/agent.py

@@ -13,8 +13,8 @@
 class Agent(BaseModel, ABC):
     """Agent is responsible for choosing the next action to take for an AgentService.
 
-    It uses a passed in memory to decide on an action that will be executed by the AgentService. To decide
-    on the next action, it may use Tools.
+    It uses the provided context, and a set of Tools, to decide on an action that will
+    be executed by the AgentService.
     """
 
     tools: List[Tool]

src/steamship/agents/schema/context.py

@@ -9,9 +9,9 @@
 
 
 class AgentContext:
-    """AgentContext contains all relevant information about a particular execution of an Agent. It used by the
+    """AgentContext contains all relevant information about a particular execution of an Agent. It is used by the
     AgentService to manage execution history as well as store/retrieve information and metadata that will be used
-    in the process of an agetn execution.
+    in the process of an agent execution.
     """
 
     # A steamship-assigned ID for this memory object
@@ -45,8 +45,6 @@ def get_or_create(
         context_keys: Dict[str, str],
         tags: List[Tag] = None,
     ):
-        # TODO: This is problematic, as agent context stores agent history too!
-        # For now, start with no completed steps
         history = ChatHistory.get_or_create(client, context_keys, tags)
         context = AgentContext()
         context.chat_history = history

src/steamship/agents/schema/output_parser.py

@@ -7,7 +7,7 @@
 
 
 class OutputParser(BaseModel, ABC):
-    """Uses to convert text into Actions.
+    """Used to convert text into Actions.
 
     Primarily used by LLM-based agents that generate textual descriptions of
     selected actions and their inputs. OutputParsers can be used to convert

src/steamship/agents/schema/tool.py

@@ -23,16 +23,16 @@ class Tool(BaseModel):
 
     name: str
     """The short name for the tool.
-
     This will be used by Agents to refer to this tool during action selection."""
 
     agent_description: str
-    """Description for use in an agent in order to enable Action selection. It should
-    include a short summary of what the Tool does, what the inputs to the Tool should be,
+    """Description for use in an agent in order to enable Action selection.
+    It should include a short summary of what the Tool does, what the inputs to the Tool should be,
     and what the outputs of the tool are."""
 
     human_description: str
-    """Human-friendly description. Used for logging, tool indices, etc."""
+    """Human-friendly description.
+    Used for logging, tool indices, etc."""
 
     @abstractmethod
     def run(self, tool_input: List[Block], context: AgentContext) -> Union[List[Block], Task[Any]]:

src/steamship/agents/tools/audio_transcription/__init__.py

@@ -1 +1,9 @@
-"""Tool for generating images."""
+from .assembly_speech_to_text_tool import AssemblySpeechToTextTool
+from .fetch_audio_urls_from_rss_tool import FetchAudioUrlsFromRssTool
+from .whisper_speech_to_text_tool import WhisperSpeechToTextTool
+
+__all__ = [
+    "AssemblySpeechToTextTool",
+    "FetchAudioUrlsFromRssTool",
+    "WhisperSpeechToTextTool",
+]

src/steamship/agents/tools/audio_transcription/fetch_audio_urls_from_rss_tool.py

@@ -9,6 +9,8 @@
 
 
 class FetchAudioUrlsFromRssTool(Tool):
+    """Given an RSS feed, this tool will extract episode URLs."""
+
     name: str = "FetchAudioUrlsFromRssTool"
     human_description: str = "Fetches the episode URLs from a Podcast RSS feed."
     agent_description: str = (

src/steamship/agents/tools/classification/__init__.py

@@ -0,0 +1,7 @@
+from .sentiment_analysis_tool import SentimentAnalysisTool
+from .zero_shot_classifier_tool import ZeroShotClassifierTool
+
+__all__ = [
+    "SentimentAnalysisTool",
+    "ZeroShotClassifierTool",
+]

src/steamship/agents/tools/conversation_starters/__init__.py

@@ -0,0 +1,3 @@
+from .knock_knock_tool import KnockKnockTool
+
+__all__ = ["KnockKnockTool"]

src/steamship/agents/tools/conversation_starters/knock_knock_tool.py

@@ -18,7 +18,7 @@ class KnockKnockTool(Tool):
     agent_description = "Used to begin telling a Knock Knock joke."
 
     def run(self, tool_input: List[Block], context: AgentContext) -> Union[List[Block], Task[Any]]:
-        return [Block(text="Knock-Knock..")]
+        return [Block(text="Knock-Knock...")]
 
 
 if __name__ == "__main__":

src/steamship/agents/tools/image_generation/__init__.py

@@ -0,0 +1,9 @@
+from .dalle import DalleTool
+from .google_image_search import GoogleImageSearchTool
+from .stable_diffusion import StableDiffusionTool
+
+__all__ = [
+    "DalleTool",
+    "GoogleImageSearchTool",
+    "StableDiffusionTool",
+]

src/steamship/agents/tools/image_generation/dalle.py

@@ -4,7 +4,7 @@
 
 
 class DalleTool(ImageGeneratorTool):
-    """Tool to generate images from text."""
+    """Tool to generate images from text using OpenAI's DALL-E."""
 
     name: str = "DalleTool"
     human_description: str = "Generates an image from text."

src/steamship/agents/tools/image_generation/google_image_search.py

@@ -4,7 +4,7 @@
 
 
 class GoogleImageSearchTool(ImageGeneratorTool):
-    """Tool to generate images from text."""
+    """Tool to generate images from text by using Google Image Search."""
 
     name: str = "GoogleImageSearchTool"
     human_description: str = "Fetches an image from Google Image Search."

src/steamship/agents/tools/image_generation/stable_diffusion.py

@@ -4,7 +4,7 @@
 
 
 class StableDiffusionTool(ImageGeneratorTool):
-    """Tool to generate images from text."""
+    """Tool to generate images from text using StableDiffusion."""
 
     name: str = "StableDiffusionTool"
     human_description: str = "Generates an image from text."

src/steamship/agents/tools/question_answering/__init__.py

@@ -0,0 +1,7 @@
+from .prompt_database_question_answerer import PromptDatabaseQATool
+from .vector_search_qa_tool import VectorSearchQATool
+
+__all__ = [
+    "PromptDatabaseQATool",
+    "VectorSearchQATool",
+]

src/steamship/agents/tools/search/__init__.py

@@ -0,0 +1,3 @@
+from .search import SearchTool
+
+__all__ = ["SearchTool"]

src/steamship/agents/tools/speech_generation/__init__.py

@@ -0,0 +1,3 @@
+from .generate_speech import GenerateSpeechTool
+
+__all__ = ["GenerateSpeechTool"]

src/steamship/agents/tools/text_generation/__init__.py

@@ -0,0 +1,13 @@
+from .image_prompt_generator_tool import ImagePromptGeneratorTool
+from .personality_tool import PersonalityTool
+from .summarize_text_with_prompt_tool import SummarizeTextWithPromptTool
+from .text_rewrite_tool import TextRewritingTool
+from .text_translation_tool import TextTranslationTool
+
+__all__ = [
+    "ImagePromptGeneratorTool",
+    "PersonalityTool",
+    "SummarizeTextWithPromptTool",
+    "TextRewritingTool",
+    "TextTranslationTool",
+]

src/steamship/agents/tools/text_generation/image_prompt_generator_tool.py

@@ -11,7 +11,7 @@
 Image Generation Prompt:"""
 
 
-class ImagePromptGenerator(TextRewritingTool):
+class ImagePromptGeneratorTool(TextRewritingTool):
     """
     Example tool to illustrate rewriting an input query to become a better prompt.
     """
@@ -28,4 +28,4 @@ class ImagePromptGenerator(TextRewritingTool):
 
 
 if __name__ == "__main__":
-    ToolREPL(ImagePromptGenerator()).run()
+    ToolREPL(ImagePromptGeneratorTool()).run()

src/steamship/agents/utils.py

@@ -9,7 +9,9 @@
 def with_llm(llm: LLM, context: Optional[AgentContext] = None) -> AgentContext:
     """Sets an LLM for general purpose lookup and usage on an AgentContext."""
     if context is None:
+        # TODO: should we have a default context somehow?
         context = AgentContext()
+        context.completed_steps = []
     context.metadata[_LLM_KEY] = llm
     return context