oai.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install azure-cognitiveservices-speech --quiet\n",
    "%pip install openai==1.30.1 --quiet\n",
    "%pip install sounddevice --quiet\n",
    "%pip install azure-ai-formrecognizer azure-cognitiveservices-speech azure-identity --quiet\n",
    "%pip install load_dotenv --quiet\n",
    "%pip install prompt_toolkit --quiet\n",
    "%pip install azure-storage-blob --quiet\n",
    "%pip install neo4j --quiet\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "vscode": {
     "languageId": "ruby"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Speech API Key: f07276fad5414743b35c407402f0e57d\n",
      "Speech Region: swedencentral\n",
      "Whisper API Key: ad0371a819a94fafba0e78d596097a45\n",
      "Whisper Endpoint: https://ut-oai-sweden.openai.azure.com/\n",
      "Whisper Deployment: whisper\n",
      "GPT API Key: 8873c265d70546a3a87dcb073310a16b\n",
      "GPT Deployment: gpt-4o\n",
      "GPT Endpoint: https://ut-oai-eastus.openai.azure.com/\n",
      "Storage Endpoint: https://stutwenteai759783997450.blob.core.windows.net/\n",
      "Storage SAS Token: sv=2022-11-02&ss=b&srt=co&sp=rl&se=2024-09-24T02:44:59Z&st=2024-06-02T18:44:59Z&spr=https&sig=5bc8o%2F1eSUUYSp5%2FwmrXESAdZJYz%2BqEIRkXaKnvE3gY%3D\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "# Load environment variables from .env file\n",
    "load_dotenv('.env')\n",
    "\n",
    "# Access the environment variables\n",
    "speech_key = os.getenv('SPEECH_KEY')\n",
    "speech_region = os.getenv('SPEECH_REGION')\n",
    "whisper_key = os.getenv('WHISPER_KEY')\n",
    "whisper_endpoint = os.getenv('WHISPER_ENDPOINT')\n",
    "whisper_deployment = os.getenv('WHISPER_DEPLOYMENT')\n",
    "gpt_key = os.getenv('GPT_KEY')\n",
    "gpt_deployment = os.getenv('GPT_DEPLOYMENT')\n",
    "gpt_endpoint = os.getenv('GPT_ENDPOINT')\n",
    "storage_endpoint = os.getenv('AZURE_STORAGE_ENDPOINT')\n",
    "storage_sas = os.getenv('AZURE_STORAGE_SAS_TOKEN')\n",
    "\n",
    "# Use the environment variables in your code\n",
    "print(f'Speech API Key: {speech_key}')\n",
    "print(f'Speech Region: {speech_region}')\n",
    "print(f'Whisper API Key: {whisper_key}')\n",
    "print(f'Whisper Endpoint: {whisper_endpoint}')\n",
    "print(f'Whisper Deployment: {whisper_deployment}')\n",
    "print(f'GPT API Key: {gpt_key}')\n",
    "print(f'GPT Deployment: {gpt_deployment}')\n",
    "print(f'GPT Endpoint: {gpt_endpoint}')\n",
    "print(f'Storage Endpoint: {storage_endpoint}')\n",
    "print(f'Storage SAS Token: {storage_sas}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Speak into your microphone.\n",
      "Hello there.\n"
     ]
    }
   ],
   "source": [
    "import azure.cognitiveservices.speech as speechsdk\n",
    "\n",
    "def recognize_from_microphone():\n",
    "    # This example requires environment variables named \"SPEECH_KEY\" and \"SPEECH_REGION\"\n",
    "    speech_config = speechsdk.SpeechConfig(subscription=speech_key, region=speech_region)\n",
    "    speech_config.speech_recognition_language=\"en-US\"\n",
    "\n",
    "    audio_config = speechsdk.audio.AudioConfig(use_default_microphone=True)\n",
    "    speech_recognizer = speechsdk.SpeechRecognizer(speech_config=speech_config, audio_config=audio_config)\n",
    "\n",
    "    print(\"Speak into your microphone.\")\n",
    "    speech_recognition_result = speech_recognizer.recognize_once_async().get()\n",
    "\n",
    "    if speech_recognition_result.reason == speechsdk.ResultReason.RecognizedSpeech:\n",
    "        #print(\"Recognized: {}\".format(speech_recognition_result.text))\n",
    "        return speech_recognition_result.text\n",
    "    elif speech_recognition_result.reason == speechsdk.ResultReason.NoMatch:\n",
    "        print(\"No speech could be recognized: {}\".format(speech_recognition_result.no_match_details))\n",
    "    elif speech_recognition_result.reason == speechsdk.ResultReason.Canceled:\n",
    "        cancellation_details = speech_recognition_result.cancellation_details\n",
    "        print(\"Speech Recognition canceled: {}\".format(cancellation_details.reason))\n",
    "        if cancellation_details.reason == speechsdk.CancellationReason.Error:\n",
    "            print(\"Error details: {}\".format(cancellation_details.error_details))\n",
    "            print(\"Did you set the speech resource key and region values?\")\n",
    "\n",
    "text = recognize_from_microphone()\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import AzureOpenAI\n",
    "\n",
    "client = AzureOpenAI(\n",
    "    api_key=whisper_key,\n",
    "    api_version=\"2024-02-01\",\n",
    "    azure_endpoint = whisper_endpoint\n",
    ")\n",
    "\n",
    "audio_test_file = \"./helloThere.m4a\"\n",
    "\n",
    "result = client.audio.transcriptions.create(\n",
    "    file=open(audio_test_file, \"rb\"),            \n",
    "    model=whisper_deployment\n",
    ")\n",
    "\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Speech synthesized for text [Hi, this is Ryan]\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "  For more samples please visit https://github.com/Azure-Samples/cognitive-services-speech-sdk \n",
    "'''\n",
    "\n",
    "import azure.cognitiveservices.speech as speechsdk\n",
    "\n",
    "speech_config = speechsdk.SpeechConfig(subscription=speech_key, region=speech_region)\n",
    "# Note: the voice setting will not overwrite the voice element in input SSML.\n",
    "speech_config.speech_synthesis_voice_name = \"en-GB-RyanNeural\"\n",
    "\n",
    "text = \"Hi, this is Ryan\"\n",
    "\n",
    "def text_to_speech(text):\n",
    "  # use the default speaker as audio output.\n",
    "  speech_synthesizer = speechsdk.SpeechSynthesizer(speech_config=speech_config)\n",
    "  result = speech_synthesizer.speak_text_async(text).get()\n",
    "  # Check result\n",
    "  if result.reason == speechsdk.ResultReason.SynthesizingAudioCompleted:\n",
    "      print(\"Speech synthesized for text [{}]\".format(text))\n",
    "  elif result.reason == speechsdk.ResultReason.Canceled:\n",
    "      cancellation_details = result.cancellation_details\n",
    "      print(\"Speech synthesis canceled: {}\".format(cancellation_details.reason))\n",
    "      if cancellation_details.reason == speechsdk.CancellationReason.Error:\n",
    "          print(\"Error details: {}\".format(cancellation_details.error_details))\n",
    "\n",
    "text_to_speech(text)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from azure.storage.blob import BlobServiceClient\n",
    "\n",
    "\n",
    "# Create the BlobServiceClient object using the account URL and SAS token\n",
    "blob_service_client = BlobServiceClient(account_url=storage_endpoint, credential=storage_sas)\n",
    "blob_container_client = blob_service_client.get_container_client(\"paintings\")\n",
    "blob_list = blob_container_client.list_blobs()\n",
    "\n",
    "for blob in blob_list:\n",
    "    print(blob.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Painting Name: Head of a Boy in a Turban\n",
      "Image URL: https://upload.wikimedia.org/wikipedia/commons/thumb/9/9b/Portrait_of_a_Boy%2C_Gerrit_Dou%2C_c._1630%2C_oil_on_panel%2C_42_by_33.5_cm%2C_Nieders%C3%A4chsisches_Landesmuseum_Hanover.jpg/330px-Portrait_of_a_Boy%2C_Gerrit_Dou%2C_c._1630%2C_oil_on_panel%2C_42_by_33.5_cm%2C_Nieders%C3%A4chsisches_Landesmuseum_Hanover.jpg\n",
      "Description: Dou was Rembrandt’s first pupil. He took up his master’s idea of studying black people. The result was this endearing tronie of a young man in a fantasy costume, who looks at us over his shoulder.\n"
     ]
    }
   ],
   "source": [
    "from neo4j import GraphDatabase\n",
    "\n",
    "\n",
    "# Replace these with your Neo4j connection details\n",
    "uri = \"bolt://localhost:7687\"  # Update this with your database URI\n",
    "user = \"neo4j\"                 # Update this with your database username\n",
    "password = \"Pa$$w0rd\"          # Update this with your database password\n",
    "\n",
    "# Create a Neo4j driver instance\n",
    "driver = GraphDatabase.driver(uri, auth=(user, password))\n",
    "\n",
    "def get_cultural_heritage_node(name):\n",
    "    with driver.session() as session:\n",
    "        result = session.run(\"MATCH (n:CulturalHeritage {name: $name}) RETURN n\", name=name)\n",
    "        return [record[\"n\"] for record in result]\n",
    "\n",
    "# Query for a specific CulturalHeritage node by name\n",
    "node_name = \"Head of a Boy in a Turban\"\n",
    "nodes = get_cultural_heritage_node(node_name)\n",
    "\n",
    "# Extract the img URL and exhibit description\n",
    "if nodes:\n",
    "    node = nodes[0]  # Assuming you want the first node in the result\n",
    "    properties = node._properties  # Access the properties of the node\n",
    "\n",
    "    # Extract the `img` URL and `exhibit` description\n",
    "    img_url = properties.get('img', 'No image URL found')\n",
    "    exhibit_description = properties.get('exhibit', 'No description found')\n",
    "    node_name = properties.get('name', 'No name found')\n",
    "\n",
    "    print(\"Painting Name:\", node_name)\n",
    "    print(\"Image URL:\", img_url)\n",
    "    print(\"Description:\", exhibit_description)\n",
    "else:\n",
    "    print(\"No nodes found\")\n",
    "\n",
    "# Close the driver connection\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Painting: https://upload.wikimedia.org/wikipedia/commons/thumb/e/e5/Hendrick-Heerschop-Koning-Caspar-1654.-Olieverf-op-paneel.-Berlijn-Staatliche-Museen.jpg/485px-Hendrick-Heerschop-Koning-Caspar-1654.-Olieverf-op-paneel.-Berlijn-Staatliche-Museen.jpg\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://upload.wikimedia.org/wikipedia/commons/thumb/e/e5/Hendrick-Heerschop-Koning-Caspar-1654.-Olieverf-op-paneel.-Berlijn-Staatliche-Museen.jpg/485px-Hendrick-Heerschop-Koning-Caspar-1654.-Olieverf-op-paneel.-Berlijn-Staatliche-Museen.jpg\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sure, John. The painting you're referring to is of King Caspar, created by Hendrick Heerschop in 1654. It's interesting because it depicts one of the magi, believed by legend to be an African king who came to worship the Christ child. Heerschop painted him without any background or narrative elements. The identity comes through his elaborate clothing and the jar of incense, his gift. However, the most captivating aspect is his face, which exudes pride and self-confidence.\n"
     ]
    }
   ],
   "source": [
    "from promptflow.core import Prompty\n",
    "from IPython.display import Image\n",
    "\n",
    "image_path = img_url\n",
    "print(\"Painting: \" + image_path)\n",
    "display(Image(url=image_path))\n",
    "firstName = \"John\"\n",
    "context = f\"Painting Name: {node_name} Painting Information: {exhibit_description}\"\n",
    "question = \"What can you tell me about the painting?\"\n",
    "\n",
    "def get_oai_response(firstName=firstName, context=context, question=question, image_path=image_path, conversation_history=[]):\n",
    "    path_to_prompty = \"./prompts/basic.prompty\"\n",
    "    flow = Prompty.load(path_to_prompty)\n",
    "    result = flow(\n",
    "        firstName = firstName,\n",
    "        context = context,\n",
    "        question = question,\n",
    "        image = image_path,\n",
    "        conversation_history = conversation_history\n",
    "    )\n",
    "\n",
    "    return result\n",
    "\n",
    "result = get_oai_response(firstName=firstName, question=\"What can you tell me about the painting?\", image_path=image_path)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import Image, display\n",
    "\n",
    "# URL of the image\n",
    "image_url = \"https://th.bing.com/th/id/R.842fb9a1885e50a762ef352821d9078d?rik=ykwt7kPK%2f3ngFA&riu=http%3a%2f%2fupload.wikimedia.org%2fwikipedia%2fcommons%2fe%2fe8%2fVan_Gogh_The_Olive_Trees..jpg&ehk=o8ZPcWcu3H0Vdk%2b2E5YJ63CXJSSHb3BFrzKL3UG5HOU%3d&risl=1&pid=ImgRaw&r=0\"\n",
    "\n",
    "# Display the image\n",
    "display(Image(url=image_url))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Painting: https://upload.wikimedia.org/wikipedia/commons/thumb/9/9b/Portrait_of_a_Boy%2C_Gerrit_Dou%2C_c._1630%2C_oil_on_panel%2C_42_by_33.5_cm%2C_Nieders%C3%A4chsisches_Landesmuseum_Hanover.jpg/330px-Portrait_of_a_Boy%2C_Gerrit_Dou%2C_c._1630%2C_oil_on_panel%2C_42_by_33.5_cm%2C_Nieders%C3%A4chsisches_Landesmuseum_Hanover.jpg\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<img src=\"https://upload.wikimedia.org/wikipedia/commons/thumb/9/9b/Portrait_of_a_Boy%2C_Gerrit_Dou%2C_c._1630%2C_oil_on_panel%2C_42_by_33.5_cm%2C_Nieders%C3%A4chsisches_Landesmuseum_Hanover.jpg/330px-Portrait_of_a_Boy%2C_Gerrit_Dou%2C_c._1630%2C_oil_on_panel%2C_42_by_33.5_cm%2C_Nieders%C3%A4chsisches_Landesmuseum_Hanover.jpg\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Speech synthesized for text [Hello! I'm an AI assistant here to help you learn more about this painting. Feel free to ask me anything.]\n",
      "Speak into your microphone.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[28], line 23\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m     22\u001b[0m     dialog()\n\u001b[0;32m---> 23\u001b[0m \u001b[43mdialog\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[28], line 10\u001b[0m, in \u001b[0;36mdialog\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdialog\u001b[39m():\n\u001b[0;32m---> 10\u001b[0m     text \u001b[38;5;241m=\u001b[39m \u001b[43mrecognize_from_microphone\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     11\u001b[0m     \u001b[38;5;28mprint\u001b[39m(text)\n\u001b[1;32m     12\u001b[0m     result \u001b[38;5;241m=\u001b[39m get_oai_response(firstName\u001b[38;5;241m=\u001b[39mfirstName, context\u001b[38;5;241m=\u001b[39mcontext, question\u001b[38;5;241m=\u001b[39mtext, image_path\u001b[38;5;241m=\u001b[39mimage_path, conversation_history\u001b[38;5;241m=\u001b[39mconversation_history)\n",
      "Cell \u001b[0;32mIn[11], line 12\u001b[0m, in \u001b[0;36mrecognize_from_microphone\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m speech_recognizer \u001b[38;5;241m=\u001b[39m speechsdk\u001b[38;5;241m.\u001b[39mSpeechRecognizer(speech_config\u001b[38;5;241m=\u001b[39mspeech_config, audio_config\u001b[38;5;241m=\u001b[39maudio_config)\n\u001b[1;32m     11\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSpeak into your microphone.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 12\u001b[0m speech_recognition_result \u001b[38;5;241m=\u001b[39m \u001b[43mspeech_recognizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecognize_once_async\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     14\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m speech_recognition_result\u001b[38;5;241m.\u001b[39mreason \u001b[38;5;241m==\u001b[39m speechsdk\u001b[38;5;241m.\u001b[39mResultReason\u001b[38;5;241m.\u001b[39mRecognizedSpeech:\n\u001b[1;32m     15\u001b[0m     \u001b[38;5;66;03m#print(\"Recognized: {}\".format(speech_recognition_result.text))\u001b[39;00m\n\u001b[1;32m     16\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m speech_recognition_result\u001b[38;5;241m.\u001b[39mtext\n",
      "File \u001b[0;32m/usr/local/lib/python3.11/site-packages/azure/cognitiveservices/speech/speech.py:576\u001b[0m, in \u001b[0;36mResultFuture.get\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    572\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    573\u001b[0m \u001b[38;5;124;03mWaits until the result is available, and returns it.\u001b[39;00m\n\u001b[1;32m    574\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    575\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__resolved:\n\u001b[0;32m--> 576\u001b[0m     result_handle \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__get_function\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_handle\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    577\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__wrapped_type \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    578\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__wrapped_type(result_handle)\n",
      "File \u001b[0;32m/usr/local/lib/python3.11/site-packages/azure/cognitiveservices/speech/speech.py:1061\u001b[0m, in \u001b[0;36mSpeechRecognizer.recognize_once_async.<locals>.resolve_future\u001b[0;34m(handle)\u001b[0m\n\u001b[1;32m   1059\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mresolve_future\u001b[39m(handle: _spx_handle):\n\u001b[1;32m   1060\u001b[0m     result_handle \u001b[38;5;241m=\u001b[39m _spx_handle(\u001b[38;5;241m0\u001b[39m)\n\u001b[0;32m-> 1061\u001b[0m     \u001b[43m_call_hr_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_sdk_lib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecognizer_recognize_once_async_wait_for\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_uint32\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mctypes\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbyref\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresult_handle\u001b[49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1062\u001b[0m     _sdk_lib\u001b[38;5;241m.\u001b[39mrecognizer_async_handle_release(handle)\n\u001b[1;32m   1063\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result_handle\n",
      "File \u001b[0;32m/usr/local/lib/python3.11/site-packages/azure/cognitiveservices/speech/interop.py:61\u001b[0m, in \u001b[0;36m_call_hr_fn\u001b[0;34m(fn, *args)\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_call_hr_fn\u001b[39m(\u001b[38;5;241m*\u001b[39margs, fn):\n\u001b[1;32m     60\u001b[0m     fn\u001b[38;5;241m.\u001b[39mrestype \u001b[38;5;241m=\u001b[39m _spx_hr\n\u001b[0;32m---> 61\u001b[0m     hr \u001b[38;5;241m=\u001b[39m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(args) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m fn()\n\u001b[1;32m     62\u001b[0m     _raise_if_failed(hr)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "conversation_history = []\n",
    "image_path = img_url\n",
    "print(\"Painting: \" + image_path)\n",
    "display(Image(url=image_path))\n",
    "firstName = \"Thomas\"\n",
    "context = f\"Painting Name: {node_name}, Painting Information: {exhibit_description}\"\n",
    "welcome_message = f\"Hello {firstName}! We are looking at the {node_name}. What do you see on this painting?\"\n",
    "text_to_speech(welcome_message)\n",
    "def dialog():\n",
    "    text = recognize_from_microphone()\n",
    "    print(text)\n",
    "    result = get_oai_response(firstName=firstName, context=context, question=text, image_path=image_path, conversation_history=conversation_history)\n",
    "    #print(conversation_history)\n",
    "    print(result)\n",
    "    conversation_history.append(f'''\n",
    "User: {text}\n",
    "Assistant: {result}\n",
    "''')\n",
    "    speech = text_to_speech(result)\n",
    "    if \"goodbye\" in text.lower():\n",
    "        return\n",
    "    dialog()\n",
    "dialog()\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}