lots of minor improvements, /rag improved and working woo! . /brainblast as well. a few more things to take care of

Author: cagostino

npcpy/data/load.py

@@ -6,6 +6,7 @@
 import numpy as np
 from typing import Optional
 
+import os
 
 def load_csv(file_path):
     df = pd.read_csv(file_path)
@@ -113,3 +114,82 @@ def load_pdf(file_path):
     "ISO": "archives",
 }
 
+
+def load_file_contents(file_path, chunk_size=250):
+    """
+    Load and format the contents of a file based on its extension.
+    Returns a list of chunks from the file content.
+    """
+    file_ext = os.path.splitext(file_path)[1].upper().lstrip('.')
+    chunks = []
+    
+    try:
+        if file_ext == 'PDF':
+            # Load PDF content
+            pdf_document = fitz.open(file_path)
+            full_text = ""
+            
+            # Extract text from each page
+            for page in pdf_document:
+                full_text += page.get_text() + "\n\n"
+            
+            # Chunk the text
+            for i in range(0, len(full_text), chunk_size):
+                chunk = full_text[i:i+chunk_size].strip()
+                if chunk:  # Skip empty chunks
+                    chunks.append(chunk)
+                    
+        elif file_ext == 'CSV':
+            df = pd.read_csv(file_path)
+            # Add metadata as first chunk
+            meta = f"CSV Columns: {', '.join(df.columns)}\nRows: {len(df)}"
+            chunks.append(meta)
+            
+            # Convert sample data to string and chunk it
+            sample = df.head(20).to_string()
+            for i in range(0, len(sample), chunk_size):
+                chunk = sample[i:i+chunk_size].strip()
+                if chunk:
+                    chunks.append(chunk)
+                    
+        elif file_ext in ['XLS', 'XLSX']:
+            df = pd.read_excel(file_path)
+            # Add metadata as first chunk
+            meta = f"Excel Columns: {', '.join(df.columns)}\nRows: {len(df)}"
+            chunks.append(meta)
+            
+            # Convert sample data to string and chunk it
+            sample = df.head(20).to_string()
+            for i in range(0, len(sample), chunk_size):
+                chunk = sample[i:i+chunk_size].strip()
+                if chunk:
+                    chunks.append(chunk)
+                    
+        elif file_ext == 'TXT':
+            with open(file_path, 'r', encoding='utf-8') as f:
+                content = f.read()
+            
+            # Chunk the text
+            for i in range(0, len(content), chunk_size):
+                chunk = content[i:i+chunk_size].strip()
+                if chunk:
+                    chunks.append(chunk)
+                    
+        elif file_ext == 'JSON':
+            with open(file_path, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+            content = json.dumps(data, indent=2)
+            
+            # Chunk the JSON
+            for i in range(0, len(content), chunk_size):
+                chunk = content[i:i+chunk_size].strip()
+                if chunk:
+                    chunks.append(chunk)
+                    
+        else:
+            chunks.append(f"Unsupported file format: {file_ext}")
+            
+        return chunks
+            
+    except Exception as e:
+        return [f"Error loading file {file_path}: {str(e)}"]

npcpy/gen/embeddings.py

@@ -70,79 +70,6 @@ def delete_embeddings_from_collection(collection, ids):
         collection.delete(ids=ids)  # Only delete if ids are provided
 
 
-def search_similar_texts(
-    query: str,
-    chroma_client,    
-    embedding_model: str,
-    embedding_provider: str ,    
-    docs_to_embed: Optional[List[str]] = None,
-    top_k: int = 5,
-) -> List[Dict[str, any]]:
-    """
-    Search for similar texts using either a Chroma database or direct embedding comparison.
-    """
-
-    print(f"\nQuery to embed: {query}")
-    embedded_search_term = get_ollama_embeddings([query], embedding_model)[0]
-    # print(f"Query embedding: {embedded_search_term}")
-
-    if docs_to_embed is None:
-        # Fetch from the database if no documents to embed are provided
-        collection_name = f"{embedding_provider}_{embedding_model}_embeddings"
-        collection = chroma_client.get_collection(collection_name)
-        results = collection.query(
-            query_embeddings=[embedded_search_term], n_results=top_k
-        )
-        # Constructing and returning results
-        return [
-            {"id": id, "score": float(distance), "text": document}
-            for id, distance, document in zip(
-                results["ids"][0], results["distances"][0], results["documents"][0]
-            )
-        ]
-
-    print(f"\nNumber of documents to embed: {len(docs_to_embed)}")
-
-    # Get embeddings for provided documents
-    raw_embeddings = get_ollama_embeddings(docs_to_embed, embedding_model)
-
-    output_embeddings = []
-    for idx, emb in enumerate(raw_embeddings):
-        if emb:  # Exclude any empty embeddings
-            output_embeddings.append(emb)
-
-    # Convert to numpy arrays for calculations
-    doc_embeddings = np.array(output_embeddings)
-    query_embedding = np.array(embedded_search_term)
-
-    # Check for zero-length embeddings
-    if len(doc_embeddings) == 0:
-        raise ValueError("No valid document embeddings found")
-
-    # Normalize embeddings to avoid division by zeros
-    doc_norms = np.linalg.norm(doc_embeddings, axis=1, keepdims=True)
-    query_norm = np.linalg.norm(query_embedding)
-
-    # Ensure no zero vectors are being used in cosine similarity
-    if query_norm == 0:
-        raise ValueError("Query embedding is zero-length")
-
-    # Calculate cosine similarities
-    cosine_similarities = np.dot(doc_embeddings, query_embedding) / (
-        doc_norms.flatten() * query_norm
-    )
-
-    # Get indices of top K documents
-    top_indices = np.argsort(cosine_similarities)[::-1][:top_k]
-
-    return [
-        {
-            "id": str(idx),
-            "score": float(cosine_similarities[idx]),
-            "text": docs_to_embed[idx],
-        }
-        for idx in top_indices
-    ]
 def get_embeddings(
     texts: List[str],
     model: str ,

npcpy/gen/image_gen.py

@@ -7,10 +7,7 @@
 from PIL import Image
 
 from litellm import image_generation
-from npcpy.npc_sysenv import (
-    NPCSH_IMAGE_GEN_MODEL,
-    NPCSH_IMAGE_GEN_PROVIDER,
-)
+
 
 
 def generate_image_diffusers(
@@ -35,7 +32,7 @@ def generate_image_diffusers(
 
 def openai_image_gen(
     prompt: str,
-    model: str = "gpt-image-1",
+    model: str = "dall-e-2",
     attachments: Union[List[Union[str, bytes, Image.Image]], None] = None,
     height: int = 1024,
     width: int = 1024,
@@ -65,8 +62,6 @@ def openai_image_gen(
                 attachment.save(img_byte_arr, format='PNG')
                 img_byte_arr.seek(0)
                 processed_images.append(img_byte_arr)
-        print(processed_images )
-        print(height, width, n_images, model, prompt)
         # Use images.edit for image editing
         result = client.images.edit(
             model=model,
@@ -82,14 +77,12 @@ def openai_image_gen(
             prompt=prompt,
             n=n_images,
             size=f"{height}x{width}",
-            response_format="b64_json"  # Request base64 encoded image
         )
 
-    # Process the result
     image_base64 = result.data[0].b64_json
     image_bytes = base64.b64decode(image_base64)
     image = Image.open(io.BytesIO(image_bytes))
-    
+    image.save('generated_image.png') 
     return image
 
 
@@ -161,8 +154,8 @@ def gemini_image_gen(
 
 def generate_image(
     prompt: str,
-    model: str = NPCSH_IMAGE_GEN_MODEL,
-    provider: str = NPCSH_IMAGE_GEN_PROVIDER,
+    model: str ,
+    provider: str ,
     height: int = 1024,
     width: int = 1024,
     n_images: int = 1,
@@ -192,7 +185,7 @@ def generate_image(
     # Set default model if none provided
     if model is None:
         if provider == "openai":
-            model = "gpt-image-1"
+            model = "dall-e-2"
         elif provider == "diffusers":
             model = "runwayml/stable-diffusion-v1-5"
         elif provider == "gemini":

npcpy/llm_funcs.py

@@ -18,6 +18,7 @@
 from npcpy.npc_sysenv import (
     render_markdown,
     lookup_provider,
+    request_user_input
 )
 from npcpy.gen.response import get_litellm_response
 from npcpy.gen.image_gen import generate_image, edit_image
@@ -27,7 +28,6 @@ def gen_image(
     prompt: str,
     model: str = None,
     provider: str = None,
-    filename: str = None,
     npc: Any = None,
     height: int = 1024,
     width: int = 1024,
@@ -44,7 +44,6 @@ def gen_image(
     Returns:
         str: The filename of the saved image.
     """
-    print(height)
     if model is not None and provider is not None:
         pass
     elif model is not None and provider is None:
@@ -56,13 +55,7 @@ def gen_image(
             model = npc.model
         if npc.api_url is not None:
             api_url = npc.api_url
-    if filename is None:
-        # Generate a filename based on the prompt and the date time
-        os.makedirs(os.path.expanduser("~/.npcsh/images/"), exist_ok=True)
-        filename = (
-            os.path.expanduser("~/.npcsh/images/")
-            + f"image_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
-        )
+
     image = generate_image(
         prompt=prompt,
         model=model,
@@ -72,26 +65,8 @@ def gen_image(
         attachments=input_images,
 
     )
-    if isinstance(image, PIL.Image.Image):
-        image.save(filename)
-        return filename
+    return image
 
-    else:
-        try:
-            # image is at a private url (for dall-e?)
-            response = requests.get(image.data[0].url)
-            with open(filename, "wb") as file:
-                file.write(response.content)
-            from PIL import Image
-
-            img = Image.open(filename)
-            img.show()
-            # console = Console()
-            # console.print(Image.from_path(filename))
-            return filename
-
-        except AttributeError as e:
-            print(f"Error saving image: {e}")
 
 
 def generate_video(
@@ -508,6 +483,7 @@ def handle_tool_call(
             api_url=api_url,
             api_key=api_key,
             npc=npc,
+            context=context
         )
         try:
             # Clean the response of markdown formatting
@@ -553,6 +529,7 @@ def handle_tool_call(
                     stream=stream,
                     attempt=attempt + 1,
                     n_attempts=n_attempts,
+                    context=context
                 )
             return {
                 "response": f"Missing inputs for tool '{tool_name}': {missing_inputs}",
@@ -610,9 +587,10 @@ def handle_tool_call(
                     stream=stream,
                     attempt=attempt + 1,
                     n_attempts=n_attempts,
+                    context=context,
                 )
         # process the tool call
-        render_markdown(f""" ## TOOL OUTPUT FROM CALLING {tool_name} \n \n {tool_output}""" )
+        render_markdown(f""" ## TOOL OUTPUT FROM CALLING {tool_name} \n \n output:{tool_output['output']}""" )
         response = get_llm_response(f"""
             The user had the following request: {command}. 
             Here were the tool outputs from calling {tool_name}: {tool_output}
@@ -626,6 +604,7 @@ def handle_tool_call(
             api_key=api_key,
             npc=npc,
             messages=messages,
+            context=context, 
             stream=stream,
         )
         messages = response['messages']
@@ -634,6 +613,46 @@ def handle_tool_call(
         return {'messages': messages, 'response': response}
 
 
+def handle_request_input(
+    context: str,
+    model: str ,
+    provider: str 
+):
+    """
+    Analyze text and decide what to request from the user
+    """
+    prompt = f"""
+    Analyze the text:
+    {context}
+    and determine what additional input is needed.
+    Return a JSON object with:
+    {{
+        "input_needed": boolean,
+        "request_reason": string explaining why input is needed,
+        "request_prompt": string to show user if input needed
+    }}
+
+    Do not include any additional markdown formatting or leading ```json tags. Your response
+    must be a valid JSON object.
+    """
+
+    response = get_llm_response(
+        prompt,
+        model=model,
+        provider=provider,
+        messages=[],
+        format="json",
+    )
+
+    result = response.get("response", {})
+    if isinstance(result, str):
+        result = json.loads(result)
+
+    user_input = request_user_input(
+        {"reason": result["request_reason"], "prompt": result["request_prompt"]},
+    )
+    return user_input
+
 
 def check_llm_command(
     command: str,
@@ -900,7 +919,7 @@ def check_llm_command(
         print(npc_to_pass)
         agent_passes = []
         if team is not None:
-            print(f"team npcs: {team.npcs}")
+            #print(f"team npcs: {team.npcs}")
             match = team.npcs.get(npc_to_pass)
             if match is not None:
                 npc_to_pass_obj = match