human_eval_results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "522\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'text': \"ABSTRACT:   during a  well london study , residents were asked about their neighbourhood and its environment . above all other complaints ,    dog poo was mentioned as a key concern . despite low rates of infection and disease among the human population resulting from contact with canine faecal matter   , the concerns of the public continue to rate it as a serious public health issue .   most public health studies , therefore , seek to identify processes of transmission and disease pathology as a method of addressing the problem .   this study approaches the issue through a contextualised analysis of residents complaints , using anthropological theory to examine the symbolic representation of    dog poo. analysis of the interviews shows that these specific complaints were located among less easily defined or articulated experiences of social and environmental neglect , where neighbours were estranged from one another and local authorities seen as negligent .   this approach has important implications for public health , as it provides not only a strong indicator of the level of dissatisfaction within some of london 's more disadvantaged neighbourhoods , but also identifies a need for policies that are grounded in cross - disciplinary research into the relationship between health ,  wellbeing and experiences of marginalisation among urban populations .  \\n=======================\\n  SENTENCE: this article critically engages with residents explicitly focused disgust with canine faecal matter and generates understanding of its symbolic value that helps clarify what has to date , been a poorly resolved public health issue .\",\n",
       " 'meta': {'model': 'hr', 'sentence_index': 0, 'doc_index': 2966},\n",
       " '_input_hash': -1302499015,\n",
       " '_task_hash': 2026043455,\n",
       " 'options': [{'id': 'content', 'text': 'content'},\n",
       "  {'id': 'no_content', 'text': 'no_content'},\n",
       "  {'id': 'important', 'text': 'important'},\n",
       "  {'id': 'not_important', 'text': 'not_important'}],\n",
       " '_session_id': 'hiporank3-a2',\n",
       " '_view_id': 'choice',\n",
       " 'accept': ['content', 'important'],\n",
       " 'answer': 'accept'}"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotations = [json.loads(l) for l in Path(\"human_eval_data.jsonl\").read_text().split(\"\\n\") if l]\n",
    "print(len(annotations))\n",
    "annotations[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "281\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'text': \"ABSTRACT:\\n  during a  well london study , residents were asked about their neighbourhood and its environment . above all other complaints ,  \\n dog poo was mentioned as a key concern . despite low rates of infection and disease among the human population resulting from contact with canine faecal matter \\n , the concerns of the public continue to rate it as a serious public health issue . \\n most public health studies , therefore , seek to identify processes of transmission and disease pathology as a method of addressing the problem . \\n this study approaches the issue through a contextualised analysis of residents complaints , using anthropological theory to examine the symbolic representation of  \\n dog poo. analysis of the interviews shows that these specific complaints were located among less easily defined or articulated experiences of social and environmental neglect , where neighbours were estranged from one another and local authorities seen as negligent . \\n this approach has important implications for public health , as it provides not only a strong indicator of the level of dissatisfaction within some of london 's more disadvantaged neighbourhoods , but also identifies a need for policies that are grounded in cross - disciplinary research into the relationship between health ,  wellbeing and experiences of marginalisation among urban populations . \\n========================\\n SENTENCE:\\nthis article critically engages with residents explicitly focused disgust with canine faecal matter and generates understanding of its symbolic value that helps clarify what has to date , been a poorly resolved public health issue .\",\n",
       " 'meta': {'model': 'hr', 'sentence_index': 0, 'doc_index': 2966}}"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samples = [json.loads(l) for l in Path(\"human_eval_samples.jsonl\").read_text().split(\"\\n\") if l]\n",
    "print(len(samples))\n",
    "samples[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotator_data = dict(\n",
    "    hr=dict(\n",
    "        a1=dict(important=[],content=[]),\n",
    "        a2=dict(important=[],content=[]),\n",
    "        a3=dict(important=[],content=[]),\n",
    "        a4=dict(important=[],content=[]),\n",
    "    ),\n",
    "    ps=dict(\n",
    "        a1=dict(important=[],content=[]),\n",
    "        a2=dict(important=[],content=[]),\n",
    "        a3=dict(important=[],content=[]),\n",
    "        a4=dict(important=[],content=[]),\n",
    "    )\n",
    ")\n",
    "for s in samples:\n",
    "    text = s['text'].replace(\"\\n\", \" \").replace(\"========================\", \"\\n=======================\\n\")\n",
    "    model = s['meta']['model']\n",
    "    answers = [a for a in annotations if a['text'] == text]\n",
    "    for answer in answers:\n",
    "        annotator = answer['_session_id'].split(\"-\")[1]\n",
    "        for a in answer['accept']:\n",
    "            if a == \"important\":\n",
    "                annotator_data[model][annotator][\"important\"] += [1]\n",
    "            elif a == \"not_important\":\n",
    "                annotator_data[model][annotator][\"important\"] += [0]\n",
    "            elif a == \"content\":\n",
    "                annotator_data[model][annotator][\"content\"] += [1]\n",
    "            elif a == \"no_content\":\n",
    "                annotator_data[model][annotator][\"content\"] += [0]\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hr importance:  0.5905511811023622\n",
      "hr content:  0.421259842519685\n",
      "ps importance:  0.487012987012987\n",
      "ps content:  0.3051948051948052\n"
     ]
    }
   ],
   "source": [
    "# human eval results (% yes)\n",
    "model_results = {}\n",
    "for model, annotators in annotator_data.items():\n",
    "    model_results[model] = dict(important=0,not_important=0,total_important=0,\n",
    "                                content=0,not_content=0,total_content=0)\n",
    "    for annotator, measures in annotators.items():\n",
    "        for measure, counts in measures.items():\n",
    "            pos = sum(counts)\n",
    "            total = len(counts)\n",
    "            neg = total - pos\n",
    "            model_results[model][measure] += pos\n",
    "            model_results[model][f\"not_{measure}\"] += neg\n",
    "            model_results[model][f\"total_{measure}\"] += total\n",
    "            \n",
    "print(\"hr importance: \", model_results['hr']['important'] / model_results['hr']['total_important'])\n",
    "print(\"hr content: \", model_results['hr']['content'] / model_results['hr']['total_content'])\n",
    "print(\"ps importance: \", model_results['ps']['important'] / model_results['ps']['total_important'])\n",
    "print(\"ps content: \", model_results['ps']['content'] / model_results['ps']['total_content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k (important,a1a2) = 0.5004133370074402\n",
      "k (important,a3a4) = 0.3351708930540242\n",
      "k (content,a1a2) = 0.505050505050505\n",
      "k (content,a3a4) = 0.4118254583010586\n"
     ]
    }
   ],
   "source": [
    "# Cohen's kappa\n",
    "# using a for yy, b for yn, c for ny, and d for nn\n",
    "k = dict(\n",
    "    important_a1a2=dict(a=0,b=0,c=0,d=0),\n",
    "    content_a1a2=dict(a=0,b=0,c=0,d=0),\n",
    "    important_a3a4=dict(a=0,b=0,c=0,d=0),\n",
    "    content_a3a4=dict(a=0,b=0,c=0,d=0),\n",
    ")\n",
    "\n",
    "# annotators a1/a2\n",
    "# important\n",
    "for model in [\"hr\", \"ps\"]:\n",
    "    for annotator_pair in [(\"a1\",\"a2\"),(\"a3\",\"a4\")]:\n",
    "        for measure in [\"important\", \"content\"]:\n",
    "            l1 = annotator_data[model][annotator_pair[0]][measure]\n",
    "            l2 = annotator_data[model][annotator_pair[1]][measure]\n",
    "            assert len(l1) == len(l2)\n",
    "            k_key = f\"{measure}_{''.join(annotator_pair)}\"\n",
    "            for i1,i2 in zip(l1,l2):\n",
    "                if i1 == i2 and i1 == 1:\n",
    "                    k[k_key][\"a\"] += 1\n",
    "                elif i1 == i2 and i1 == 0:\n",
    "                    k[k_key][\"d\"] += 1\n",
    "                elif i1 == 1:\n",
    "                    k[k_key][\"b\"] += 1\n",
    "                else:\n",
    "                    k[k_key][\"c\"] += 1\n",
    "\n",
    "for measure in [\"important\",\"content\"]:\n",
    "    for annotator_pair in [(\"a1a2\"),(\"a3a4\")]:\n",
    "        k_key = f\"{measure}_{annotator_pair}\"\n",
    "        total = k[k_key][\"a\"] + k[k_key][\"b\"] + k[k_key][\"c\"] + k[k_key][\"d\"]\n",
    "        k[k_key][\"p_o\"] = (k[k_key][\"a\"] + k[k_key][\"d\"]) / total\n",
    "        k[k_key][\"p_yes\"] = (k[k_key][\"a\"] + k[k_key][\"b\"]) / total\n",
    "        k[k_key][\"p_yes\"] *= (k[k_key][\"a\"] + k[k_key][\"c\"]) / total\n",
    "        k[k_key][\"p_no\"] = (k[k_key][\"c\"] + k[k_key][\"d\"]) / total\n",
    "        k[k_key][\"p_no\"] *= (k[k_key][\"b\"] + k[k_key][\"d\"]) / total\n",
    "        k[k_key][\"p_e\"] = k[k_key][\"p_yes\"] + k[k_key][\"p_no\"]\n",
    "        k[k_key][\"k\"] = (k[k_key][\"p_o\"] - k[k_key][\"p_e\"]) / (1 - k[k_key][\"p_e\"])\n",
    "        print(f\"k ({measure},{annotator_pair}) = {k[k_key]['k']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'i': 0.41368956743002566, 'c': 0.46558068383842566}\n"
     ]
    }
   ],
   "source": [
    "# Fleiss's kappa\n",
    "# build table\n",
    "import pandas as pd\n",
    "fleiss = {\n",
    "    \"i\": pd.DataFrame(),\n",
    "    \"c\": pd.DataFrame(),\n",
    "}\n",
    "\n",
    "fleiss[\"i\"][\"y\"] = annotator_data[\"hr\"][\"a1\"][\"important\"] + annotator_data[\"ps\"][\"a1\"][\"important\"] + \\\n",
    "                    annotator_data[\"hr\"][\"a3\"][\"important\"] + annotator_data[\"ps\"][\"a3\"][\"important\"]\n",
    "fleiss[\"i\"][\"y\"] += annotator_data[\"hr\"][\"a2\"][\"important\"] + annotator_data[\"ps\"][\"a2\"][\"important\"] + \\\n",
    "                    annotator_data[\"hr\"][\"a4\"][\"important\"] + annotator_data[\"ps\"][\"a4\"][\"important\"]\n",
    "fleiss[\"i\"][\"n\"] = 2 - fleiss[\"i\"][\"y\"]\n",
    "\n",
    "\n",
    "fleiss[\"c\"][\"y\"] = annotator_data[\"hr\"][\"a1\"][\"content\"] + annotator_data[\"ps\"][\"a1\"][\"content\"] + \\\n",
    "                    annotator_data[\"hr\"][\"a3\"][\"content\"] + annotator_data[\"ps\"][\"a3\"][\"content\"]\n",
    "fleiss[\"c\"][\"y\"] += annotator_data[\"hr\"][\"a2\"][\"content\"] + annotator_data[\"ps\"][\"a2\"][\"content\"] + \\\n",
    "                    annotator_data[\"hr\"][\"a4\"][\"content\"] + annotator_data[\"ps\"][\"a4\"][\"content\"]\n",
    "fleiss[\"c\"][\"n\"] = 2 - fleiss[\"c\"][\"y\"]\n",
    "\n",
    "n = 2 # number of raters\n",
    "k = 2 # number of classes\n",
    "assert len(fleiss[\"c\"]) == len(fleiss[\"i\"])\n",
    "N = len(fleiss[\"c\"]) # number of subjects\n",
    "\n",
    "fleiss[\"i_pi\"] = [x/fleiss[\"i\"].sum().sum() for x in fleiss[\"i\"].sum()]\n",
    "fleiss[\"c_pi\"] = [x/fleiss[\"c\"].sum().sum() for x in fleiss[\"c\"].sum()]\n",
    "\n",
    "fleiss[\"i\"][\"P_i\"] = (1/(n*(n-1))) * (fleiss[\"i\"][\"y\"] ** 2 + fleiss[\"i\"][\"n\"] ** 2 - n)\n",
    "fleiss[\"c\"][\"P_i\"] = (1/(n*(n-1))) * (fleiss[\"c\"][\"y\"] ** 2 + fleiss[\"c\"][\"n\"] ** 2 - n)\n",
    "\n",
    "fleiss[\"i_P\"] = fleiss[\"i\"][\"P_i\"].mean()\n",
    "fleiss[\"c_P\"] = fleiss[\"c\"][\"P_i\"].mean()\n",
    "\n",
    "fleiss[\"i_Pe\"] = sum([x**2 for x in fleiss[\"i_pi\"]])\n",
    "fleiss[\"c_Pe\"] = sum([x**2 for x in fleiss[\"c_pi\"]])\n",
    "\n",
    "fleiss[\"k\"] = {}\n",
    "fleiss[\"k\"][\"i\"] = (fleiss[\"i_P\"] - fleiss[\"i_Pe\"]) / (1 - fleiss[\"i_Pe\"])\n",
    "fleiss[\"k\"][\"c\"] = (fleiss[\"c_P\"] - fleiss[\"c_Pe\"]) / (1 - fleiss[\"c_Pe\"])\n",
    "print(fleiss[\"k\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# statistical significance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stat=40.000, p=0.236\n",
      "Probably the same distribution\n"
     ]
    }
   ],
   "source": [
    "# Example of the Student's t-test\n",
    "from scipy.stats import ttest_ind\n",
    "from scipy.stats import mannwhitneyu\n",
    "data1 = [0.873, 2.817, 0.121, -0.945, -0.055, -1.436, 0.360, -1.478, -1.637, -1.869]\n",
    "data2 = [1.142, -0.432, -0.938, -0.729, -0.846, -0.157, 0.500, 1.183, -1.075, -0.169]\n",
    "stat, p = mannwhitneyu(data1, data2)\n",
    "print('stat=%.3f, p=%.3f' % (stat, p))\n",
    "if p > 0.05:\n",
    "    print('Probably the same distribution')\n",
    "else:\n",
    "    print('Probably different distributions')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stat=34576.000, p=0.002\n",
      "Probably different distributions\n"
     ]
    }
   ],
   "source": [
    "stat, p = mannwhitneyu(df_hr['content'].tolist(), df_ps['content'].tolist())\n",
    "print('stat=%.3f, p=%.3f' % (stat, p))\n",
    "if p > 0.05:\n",
    "    print('Probably the same distribution')\n",
    "else:\n",
    "    print('Probably different distributions')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stat=35066.000, p=0.007\n",
      "Probably different distributions\n"
     ]
    }
   ],
   "source": [
    "stat, p = mannwhitneyu(df_hr['important'].tolist(), df_ps['important'].tolist())\n",
    "print('stat=%.3f, p=%.3f' % (stat, p))\n",
    "if p > 0.05:\n",
    "    print('Probably the same distribution')\n",
    "else:\n",
    "    print('Probably different distributions')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}