correlation_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "46e0df9b-db7d-4a52-bcec-d49b5fc492fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\ncorrelation_analysis.ipynb\\n\\nThis notebook calculates the Spearman and Pearson correlations between human and GPT-4o ratings.\\n\\nSteps:\\n1. Load human ratings and GPT-4o ratings from the JSON files.\\n2. Calculate Spearman, Pearson, and Kendalltau correlations.\\n3. Display the correlation results.\\n'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "correlation_analysis.ipynb\n",
    "\n",
    "This notebook calculates the Spearman and Pearson correlations between human and GPT-4o ratings.\n",
    "\n",
    "Steps:\n",
    "1. Load human ratings and GPT-4o ratings from the JSON files.\n",
    "2. Calculate Spearman, Pearson, and Kendalltau correlations.\n",
    "3. Display the correlation results.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2cb3cd14-0533-45f2-a488-d6421d9f0aaa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing complete. Results saved to 'outputs/human_scored_gpt4_scored_Llama-3-8B.json'\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "# 文件路径\n",
    "input_file_path = 'outputs/gpt4_scored_Llama-3-8B.json'\n",
    "output_file_path = 'outputs/human_scored_gpt4_scored_Llama-3-8B.json'\n",
    "\n",
    "# 读取 JSON 文件\n",
    "with open(input_file_path, 'r', encoding='utf-8') as file:\n",
    "    data = json.load(file)\n",
    "\n",
    "# 准备新的数据列表\n",
    "filtered_data = []\n",
    "\n",
    "# 处理数据\n",
    "for item in data:\n",
    "    if 'turns' in item:\n",
    "        filtered_turns = []\n",
    "        for i, turn in enumerate(item['turns']):\n",
    "            # 检查 predict_type 和 gold_type\n",
    "            predict_type = turn.get('predict_type', None)\n",
    "            gold_type = item['turns'][i-1].get('type', '') if i > 0 else None\n",
    "\n",
    "            if predict_type == gold_type and gold_type != \"improper\" and gold_type != \"answerable\":\n",
    "                if 'RQS' in turn and turn['RQS'] != \"N/A\":\n",
    "                    # 添加新的字段 RQS_human_avg\n",
    "                    turn['RQS_human_avg'] = turn['RQS']\n",
    "                    filtered_turns.append(item['turns'][i-1])\n",
    "                    filtered_turns.append(turn)\n",
    "        \n",
    "        if filtered_turns:\n",
    "            item['turns'] = filtered_turns\n",
    "            filtered_data.append(item)\n",
    "\n",
    "# 将修改后的数据写入新的 JSON 文件\n",
    "with open(output_file_path, 'w', encoding='utf-8') as file:\n",
    "    json.dump(filtered_data, file, ensure_ascii=False, indent=4)\n",
    "\n",
    "print(f\"Processing complete. Results saved to '{output_file_path}'\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "64e690ee-13e4-4f94-a338-f76a44216c81",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preview of df:\n",
      "Number of rows where Type is 'unanswerable': 21\n",
      "             Type  RQS  RQS_human\n",
      "0    unanswerable    1          0\n",
      "1    unanswerable   10         10\n",
      "2    unanswerable   10         10\n",
      "3    unanswerable    9         10\n",
      "5    unanswerable    2          2\n",
      "7    unanswerable   10         10\n",
      "8    unanswerable   10         10\n",
      "9    unanswerable   10         10\n",
      "13   unanswerable   10         10\n",
      "15   unanswerable   10         10\n",
      "16   unanswerable   10         10\n",
      "40   unanswerable    9          7\n",
      "44   unanswerable   10         10\n",
      "83   unanswerable   10         10\n",
      "86   unanswerable   10         10\n",
      "100  unanswerable    8         10\n",
      "131  unanswerable   10          4\n",
      "134  unanswerable   10          9\n",
      "135  unanswerable   10         10\n",
      "145  unanswerable   10         10\n",
      "148  unanswerable   10         10\n",
      "Number of rows where Type is 'ambiguous': 14\n",
      "          Type  RQS  RQS_human\n",
      "4    ambiguous    0          0\n",
      "6    ambiguous    8          9\n",
      "10   ambiguous    6          6\n",
      "11   ambiguous    8          8\n",
      "12   ambiguous   10          9\n",
      "14   ambiguous    9          9\n",
      "17   ambiguous    6          6\n",
      "33   ambiguous    8          9\n",
      "46   ambiguous    8          5\n",
      "81   ambiguous    8          8\n",
      "89   ambiguous    8          8\n",
      "93   ambiguous    0          0\n",
      "138  ambiguous    0          0\n",
      "159  ambiguous    9         10\n",
      "Number of rows where Type is 'improper': 145\n",
      "         Type  RQS  RQS_human\n",
      "18   improper   10         10\n",
      "19   improper   10         10\n",
      "20   improper   10         10\n",
      "21   improper   10         10\n",
      "22   improper   10         10\n",
      "..        ...  ...        ...\n",
      "176  improper   10         10\n",
      "177  improper   10         10\n",
      "178  improper   10         10\n",
      "179  improper   10         10\n",
      "180  improper   10         10\n",
      "\n",
      "[145 rows x 3 columns]\n",
      "           Type   Pearson  Pearson P-value  Spearman  Spearman P-value  \\\n",
      "0  unanswerable  0.861021     5.439096e-07  0.564863      7.630475e-03   \n",
      "1     ambiguous  0.961899     4.068285e-08  0.881472      3.086584e-05   \n",
      "2      improper  0.706072     3.457085e-23  0.618931      1.068507e-16   \n",
      "\n",
      "    Kendall  Kendall P-value  \n",
      "0  0.536321     8.761915e-03  \n",
      "1  0.806259     3.861136e-04  \n",
      "2  0.610426     7.137477e-14  \n",
      "Total number of rows for specified types: 180\n",
      "Correlation analysis complete.\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "from scipy.stats import pearsonr, spearmanr, kendalltau\n",
    "\n",
    "human_file_path = 'outputs/human_scored.json'\n",
    "\n",
    "with open(human_file_path, 'r', encoding='utf-8') as human_file:\n",
    "    human_data = json.load(human_file)\n",
    "\n",
    "results = []\n",
    "\n",
    "for item in human_data:\n",
    "    if 'turns' in item:\n",
    "        for i,turn in enumerate(item['turns']):\n",
    "            if 'RQS' in turn and turn['RQS'] != \"N/A\" and 'RQS_human_avg' in turn and turn['RQS_human_avg'] != \"N/A\":\n",
    "                predict_type = turn.get('predict_type', None)\n",
    "                gold_type = item['turns'][i-1].get('type','')\n",
    "                if predict_type == gold_type:\n",
    "                    results.append({\n",
    "                        'Type': predict_type,\n",
    "                        'RQS': int(turn['RQS']),\n",
    "                        'RQS_human': int(turn['RQS_human_avg'])\n",
    "                    })\n",
    "\n",
    "df = pd.DataFrame(results)\n",
    "\n",
    "print(\"Preview of df:\")\n",
    "\n",
    "correlation_results = []\n",
    "\n",
    "total_count = 0\n",
    "\n",
    "for predict_type in ['unanswerable', 'ambiguous', 'improper']:\n",
    "    subset = df[df['Type'] == predict_type]\n",
    "    count = len(subset)\n",
    "    total_count += count\n",
    "    print(f\"Number of rows where Type is '{predict_type}': {count}\")\n",
    "    print(subset)\n",
    "    \n",
    "    if len(subset) > 1:\n",
    "        if subset['RQS'].nunique() > 1 and subset['RQS_human'].nunique() > 1:\n",
    "            pearson_corr, pearson_pval = pearsonr(subset['RQS'], subset['RQS_human'])\n",
    "            spearman_corr, spearman_pval = spearmanr(subset['RQS'], subset['RQS_human'])\n",
    "            kendall_corr, kendall_pval = kendalltau(subset['RQS'], subset['RQS_human'])\n",
    "            correlation_results.append({\n",
    "                'Type': predict_type,\n",
    "                'Pearson': pearson_corr,\n",
    "                'Pearson P-value': pearson_pval,\n",
    "                'Spearman': spearman_corr,\n",
    "                'Spearman P-value': spearman_pval,\n",
    "                'Kendall': kendall_corr,\n",
    "                'Kendall P-value': kendall_pval\n",
    "            })\n",
    "        else:\n",
    "            print(f\"Type '{predict_type}' has constant RQS or RQS_human values, skipping correlation calculation.\")\n",
    "\n",
    "correlation_df = pd.DataFrame(correlation_results)\n",
    "\n",
    "print(correlation_df)\n",
    "print(f\"Total number of rows for specified types: {total_count}\")\n",
    "print(\"Correlation analysis complete.\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}