Code/Turn_Based_Dialogue_Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Turn-Based Dialogue Evaluation\n",
    "* Dataset: CPsyCounE\n",
    "* Model: GPT-4 Turbo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_json_files(folder_path):\n",
    "    \"\"\"\n",
    "    读取指定文件夹路径下的所有json文件。\n",
    "    \"\"\"\n",
    "    json_files = [pos_json for pos_json in os.listdir(folder_path) if pos_json.endswith('.json')]\n",
    "    dialogues = []\n",
    "    for json_file in json_files:\n",
    "        file_path = os.path.join(folder_path, json_file)\n",
    "        with open(file_path, 'r', encoding='utf-8') as f:\n",
    "            dialogue_data = json.load(f)\n",
    "            dialogues.append(dialogue_data)\n",
    "    return dialogues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def construct_turn_based_dialogues(dialogue_data):\n",
    "    \"\"\"\n",
    "    根据对话数据构造Turn-Based对话。\n",
    "    每个Turn-Based对话包含当前求助者提问及之前的所有历史对话。\n",
    "    \"\"\"\n",
    "    turn_based_dialogues = []\n",
    "    history_dialogue = \"\"  # 用于累积所有历史对话\n",
    "\n",
    "    # 遍历对话数据列表，构造Turn-Based对话\n",
    "    for utterance in dialogue_data:\n",
    "        # 如果是求助者发言，则开始新的Turn-Based对话\n",
    "        if \"求助者：\" in utterance:\n",
    "            # 如果历史对话非空，说明这不是第一轮对话，需要保存当前Turn-Based对话\n",
    "            history_dialogue += f\"{utterance}\" # 当前轮次求助者提问加入历史对话\n",
    "            turn_based_dialogues.append(history_dialogue)\n",
    "        else:\n",
    "            # 如果是支持者发言，则累积到历史对话中\n",
    "            history_dialogue += f\" {utterance}\"\n",
    "\n",
    "    return turn_based_dialogues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def generate_replies(turn_based_dialogues, model_reply):\n",
    "    \"\"\"\n",
    "    使用用户提供的模型生成每个Turn-Based对话的回复。\n",
    "    \"\"\"\n",
    "    turn_based_replies = []\n",
    "    for dialogue in turn_based_dialogues:\n",
    "        reply = model_reply(dialogue)\n",
    "        turn_based_replies.append(reply)\n",
    "    return turn_based_replies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def evaluate_replies(turn_based_dialogues, turn_based_replies, evaluate_reply):\n",
    "    \"\"\"\n",
    "    使用GPT-4评价每个Turn-Based对话回复的得分。\n",
    "    \"\"\"\n",
    "    scores = []\n",
    "    for dialogue, reply in zip(turn_based_dialogues, turn_based_replies):\n",
    "        score = evaluate_reply(dialogue, reply)\n",
    "        scores.append(score)\n",
    "    return scores\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def write_evaluation_results(scores, theme_folder, cnt):\n",
    "    # 定位到仓库的根目录\n",
    "    base_dir = os.path.abspath(os.path.join('..'))\n",
    "    # 构建结果文件夹的完整路径\n",
    "    results_dir = os.path.join(base_dir, \"Results_Turn_Based_Dialogue_Evaluation\", theme_folder)\n",
    "    # 检查结果文件夹是否存在，如果不存在，则创建它\n",
    "    if not os.path.exists(results_dir):\n",
    "        os.makedirs(results_dir)\n",
    "    \n",
    "    # 完整路径\n",
    "    result_file_path = os.path.join(results_dir, f\"evaluation_results_{cnt}.txt\")\n",
    "\n",
    "    # 写入评价结果\n",
    "    with open(result_file_path, 'w') as f:\n",
    "        # 在文件开始处写入主题名\n",
    "        f.write(f\"{theme_folder}\\n\")\n",
    "        # 逐行写入每轮的评分，前面带有轮数信息\n",
    "        for i, score in enumerate(scores, start=1):\n",
    "            f.write(f\"Round {i}, Score: {score}\\n\")\n",
    "        # 计算并写入平均评分\n",
    "        avg_score = [round(sum(col) / len(col), 2) for col in zip(*scores)]\n",
    "        f.write(f\"Average Scores: {avg_score}\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BUG\n",
    "# 需用户完善模型的回复生成，并返回reply\n",
    "# 示例模型回复函数(需要替换为实际使用的模型)\n",
    "def model_reply(history):\n",
    "    # 这里应该是模型生成回复的代码\n",
    "    user_message = f\"\"\"\n",
    "    你是一位有着二十年从业经验的心理咨询师。你旨在通过专业心理咨询，帮助来访者解决心理问题。请参考历史对话记录，并仅对来访者当前问题提供回复。\n",
    "    历史对话记录:\n",
    "    '''\n",
    "    {history}\n",
    "    '''\n",
    "    \"\"\"\n",
    "\n",
    "    messages = [{\"role\": \"user\", \"content\": user_message}]\n",
    "    \n",
    "    # 返回生成的回复\n",
    "    return reply "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 默认评价函数(默认GPT-4-Turbo，可按需更换模型)\n",
    "# 用户需要创建.env文件，并在其中添加OPENAI_API_KEY = \"\"\n",
    "# 返回内容需按照规定格式返回评分, 例如\"[2,2,3,3]\" (已在prompt中限定)\n",
    "import os\n",
    "import openai\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "openai.api_key = os.environ[\"OPENAI_API_KEY\"]\n",
    "\n",
    "def evaluate_reply(history, reply):\n",
    "    # GPT-4评价模型\n",
    "    \n",
    "    system_message = f\"\"\"\n",
    "    # Role\n",
    "    You are an impartial judge, familiar with psychological knowledge and psychological counseling.\n",
    "\n",
    "    ## Attention\n",
    "    You are responsible for evaluating the quality of the response provided by the AI Psychological counselors to the client's psychological problems. Your evaluation should refer to the History content and score based solely on the Evaluation Standard.\n",
    "\n",
    "    ## Evaluation Standard：\n",
    "    ### Comprehensiveness (0-2 points)：\n",
    "    The client’s situation and the degree to which psychological problems are reflected in the responses.\n",
    "    Including but not limited to the following aspects:\n",
    "    - 1.1 Does the response reflect the basic information about the client?\n",
    "    - 1.2 Does the response reflect the client's psychological problems?\n",
    "        \n",
    "    ### Professionalism (0-3 points)：\n",
    "    The professionalism of the psychological counselor in the responses.\n",
    "    Including but not limited to the following aspects:\n",
    "    - 2.1 Does the response reflect the counselor's professional ability to diagnose psychological problems?\n",
    "    - 2.2 Does the response reflect the professional psychological counseling techniques used by the counselor?\n",
    "    - 2.3 Is the response worded professionally and is there a guided dialogue?\n",
    "    - 2.4 Does the response actually help the client?\n",
    "    - 2.5 Does the response proceed in the order of the professional consultation framework (Reception and inquiry stage, Diagnostic stage, Consultation\n",
    "    stage, Consolidation and ending stage)? (1 point)\n",
    "    - 2.6 Is there a specific implementation process for psychological counseling technology, as detailed and clear as possible? (1 point)\n",
    "        \n",
    "    ### Authenticity (0-3 points):\n",
    "    The degree of authenticity between the client and the counselor in the responses.\n",
    "    Including but not limited to the following aspects:\n",
    "    - 3.1 Is the response relevant to the multi-turn consultation dialogue's history? (1 point)\n",
    "    - 3.2 Does the response reflect the counselor's understanding and empathy for the client?\n",
    "    - 3.3 Does the response avoid expressions that may cause misunderstanding or discomfort?\n",
    "    - 3.4 Does the response avoid long statements and is consistent with real psychological counseling scenarios? (1 point)\n",
    "        \n",
    "    ### Safety (0-1 point):\n",
    "    The degree of privacy protection of clients.\n",
    "    Including but not limited to the following aspects:\n",
    "    - 4.1 Does the response comply with psychological counseling privacy guidelines and avoid disclosing sensitive information (personal name, workplace, contact information, home address)?\n",
    "    - 4.2 Does the response respect the client’s thoughts and emotions?\n",
    "\n",
    "    ## History\n",
    "    '''\n",
    "    {history}\n",
    "    '''\n",
    "\n",
    "    ## Constraints\n",
    "    - Avoid any position biases and ensure that the order in which the responses were presented does not influence your decision\n",
    "    - Do not allow the length of the responses to influence your evaluation\n",
    "    - Do not favor certain names of the assistants. Be as objective as possible\n",
    "\n",
    "    ## Workflow\n",
    "    Output your final verdict by strictly following this format: \"[Comprehensiveness's score, Professionalism's score, Authenticity's score, Safety's score]\". \n",
    "\n",
    "    Take a deep breath and think step by step! \n",
    "    \"\"\"\n",
    "   \n",
    "    messages = [\n",
    "        {\"role\": \"system\", \"content\":system_message},\n",
    "        {\"role\": \"user\", \"content\": reply},\n",
    "]\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model = 'gpt-4-turbo',\n",
    "        messages = messages,\n",
    "        temperature = 0.0,\n",
    "    )\n",
    "    \n",
    "    # 返回评价得分\n",
    "    return response.choices[0].message[\"content\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 单一主题得分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 主题：'Career', 'Education', 'Emotion&Stress', 'Family Relationship', 'Love&Marriage', 'Mental Disease', 'Self-growth', 'Sex', 'Social Relationship'\n",
    "# 主题文件夹的路径\n",
    "theme_folder = '' # 填入主题文件夹的名称    \n",
    "theme_folder_path = os.path.abspath(os.path.join('..', 'CPsyCounE', theme_folder))\n",
    "# 执行读取JSON文件\n",
    "dialogues = read_json_files(theme_folder_path)\n",
    "\n",
    "for i in range(len(dialogues)):\n",
    "    cnt = i\n",
    "    dialogue_data = dialogues[i]\n",
    "    # 构造Turn-Based对话\n",
    "    turn_based_dialogues = construct_turn_based_dialogues(dialogue_data)\n",
    "    # 生成回复\n",
    "    turn_based_replies = generate_replies(turn_based_dialogues, model_reply)\n",
    "    # 评价得分\n",
    "    scores = evaluate_replies(turn_based_dialogues, turn_based_replies, evaluate_reply)\n",
    "    # 写入评价结果\n",
    "    write_evaluation_results(scores, theme_folder, cnt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 全部主题得分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 9个主题文件夹的路径\n",
    "folders = ['Career', 'Education', 'Emotion&Stress', 'Family Relationship', 'Love&Marriage', 'Mental Disease', 'Self-growth', 'Sex', 'Social Relationship']\n",
    "for theme_folder in folders:\n",
    "    theme_folder_path = os.path.abspath(os.path.join('..', 'CPsyCounE', theme_folder))\n",
    "    # 执行读取JSON文件\n",
    "    dialogues = read_json_files(theme_folder_path)\n",
    "\n",
    "    for i in range(len(dialogues)):\n",
    "        cnt = i\n",
    "        dialogue_data = dialogues[i]\n",
    "        # 构造Turn-Based对话\n",
    "        turn_based_dialogues = construct_turn_based_dialogues(dialogue_data)\n",
    "        # 生成回复\n",
    "        turn_based_replies = generate_replies(turn_based_dialogues, model_reply)\n",
    "        # 评价得分\n",
    "        scores = evaluate_replies(turn_based_dialogues, turn_based_replies, evaluate_reply)\n",
    "        # 写入评价结果\n",
    "        write_evaluation_results(scores, theme_folder, cnt)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}