natural_language_processing/token_classification/bert_travel_order_1_0_0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:57.077227Z",
     "iopub.status.busy": "2023-04-09T17:20:57.076060Z",
     "iopub.status.idle": "2023-04-09T17:20:57.084563Z",
     "shell.execute_reply": "2023-04-09T17:20:57.082676Z",
     "shell.execute_reply.started": "2023-04-09T17:20:57.077196Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import re\n",
    "from functools import partial\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "from torch.optim import SGD\n",
    "from torch.utils.data import DataLoader\n",
    "from tqdm import tqdm\n",
    "from transformers import BertModel, BertTokenizerFast, BertForTokenClassification\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from transformers import logging\n",
    "logging.set_verbosity_error()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:57.087720Z",
     "iopub.status.busy": "2023-04-09T17:20:57.087333Z",
     "iopub.status.idle": "2023-04-09T17:20:57.534532Z",
     "shell.execute_reply": "2023-04-09T17:20:57.533564Z",
     "shell.execute_reply.started": "2023-04-09T17:20:57.087688Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>to</th>\n",
       "      <th>from</th>\n",
       "      <th>moment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Depuis La chaize-le-vicomte à La roche-sur-foron</td>\n",
       "      <td>La roche-sur-foron</td>\n",
       "      <td>La chaize-le-vicomte</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Itiniréraire jusqu'a Giromagny depuis Quimper</td>\n",
       "      <td>Giromagny</td>\n",
       "      <td>Quimper</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Comment aller de Boigneville à Longjumeau mardi</td>\n",
       "      <td>Longjumeau</td>\n",
       "      <td>Boigneville</td>\n",
       "      <td>mardi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Je suis actuellement à Villers-sur-mer et j’ai...</td>\n",
       "      <td>Ferrières-en-bray</td>\n",
       "      <td>Villers-sur-mer</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Donne moi l'itinéraire pour aller à Fontenay-l...</td>\n",
       "      <td>Fontenay-le-fleury</td>\n",
       "      <td>Lizy-sur-ourcq</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text                  to  \\\n",
       "0   Depuis La chaize-le-vicomte à La roche-sur-foron  La roche-sur-foron   \n",
       "1      Itiniréraire jusqu'a Giromagny depuis Quimper           Giromagny   \n",
       "2    Comment aller de Boigneville à Longjumeau mardi          Longjumeau   \n",
       "3  Je suis actuellement à Villers-sur-mer et j’ai...   Ferrières-en-bray   \n",
       "4  Donne moi l'itinéraire pour aller à Fontenay-l...  Fontenay-le-fleury   \n",
       "\n",
       "                   from moment  \n",
       "0  La chaize-le-vicomte         \n",
       "1               Quimper         \n",
       "2           Boigneville  mardi  \n",
       "3       Villers-sur-mer         \n",
       "4        Lizy-sur-ourcq         "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATASET_PATH = 'dataframe.csv'\n",
    "df = pd.read_csv(DATASET_PATH, sep=\";\", encoding=\"utf-8\")\n",
    "df = df.where(pd.notnull(df), '')\n",
    "df.head(5)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenizer and annotation functions\n",
    "\n",
    "This code appears to define several functions and use them to add annotations to a pandas DataFrame.\n",
    "\n",
    "The first function text_tokenizer takes a string input sentence and performs some basic text preprocessing, such as adding spaces around special characters, removing multiple spaces, and trimming the resulting string. This function appears to be used to preprocess the text data in the DataFrame.\n",
    "\n",
    "The second function label_write takes a row of a pandas DataFrame row, a string key, and a string role. The function creates a dictionary with the key being the value in the row corresponding to the key parameter and the value being an empty list. The function then tokenizes the value, adding a 'B-' tag to the first token and an 'I-' tag to the subsequent tokens. This function appears to be used to generate labels for the named entities in the text data.\n",
    "\n",
    "The third function _tokenize_to_label takes a row of a pandas DataFrame row and uses the label_write function to generate labels for each named entity in the text data. The function then replaces all characters that do not start with 'B-' or 'I-' with 'o'. This function appears to be used to tokenize the text data and add labels to the DataFrame.\n",
    "\n",
    "Finally, the code applies the text_tokenizer and _tokenize_to_label functions to the text column in the DataFrame and stores the resulting annotations in a new column called annotations.\n",
    "\n",
    "## Example\n",
    "- Sentence: `I want to fly from boston to new-york`\n",
    "- Tokenized Sentence: `I want to fly from boston to new - york`\n",
    "- Annotations: `o o o o o B-FROM o B-TO I-TO I-TO`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:57.536763Z",
     "iopub.status.busy": "2023-04-09T17:20:57.536448Z",
     "iopub.status.idle": "2023-04-09T17:20:57.582341Z",
     "shell.execute_reply": "2023-04-09T17:20:57.581320Z",
     "shell.execute_reply.started": "2023-04-09T17:20:57.536723Z"
    }
   },
   "outputs": [],
   "source": [
    "def text_tokenizer(sentence: str) -> str:\n",
    "    # Add space around special characters\n",
    "    sentence = re.sub(r'([^\\w\\s])', r' \\1 ', sentence)\n",
    "\n",
    "    # Remove multiple spaces\n",
    "    sentence = re.sub(r'\\s+', ' ', sentence)\n",
    "\n",
    "    # Trim\n",
    "    sentence = sentence.strip()\n",
    "\n",
    "    return sentence\n",
    "\n",
    "df['tokenized_sentence'] = df.apply(lambda x: text_tokenizer(x['text']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:57.584016Z",
     "iopub.status.busy": "2023-04-09T17:20:57.583771Z",
     "iopub.status.idle": "2023-04-09T17:20:58.565759Z",
     "shell.execute_reply": "2023-04-09T17:20:58.564787Z",
     "shell.execute_reply.started": "2023-04-09T17:20:57.583992Z"
    }
   },
   "outputs": [],
   "source": [
    "def label_write(row: pd.Series, key: str, role: str) -> dict:\n",
    "    if key not in row:\n",
    "        return {}\n",
    "\n",
    "    response = {}\n",
    "\n",
    "    if ';' in row[key]:\n",
    "        values = row[key].split(';')\n",
    "        for value in values:\n",
    "            response.update({value: []})\n",
    "            tokenized_value = text_tokenizer(value).split()\n",
    "            for i, _ in enumerate(tokenized_value):\n",
    "                response[value].append(f'B-{role}' if i == 0 else f'I-{role}')\n",
    "    else:\n",
    "        response.update({row[key]: []})\n",
    "        tokenized_value = text_tokenizer(row[key]).split()\n",
    "        for i, _ in enumerate(tokenized_value):\n",
    "            response[row[key]].append(f'B-{role}' if i == 0 else f'I-{role}')\n",
    "\n",
    "    return response\n",
    "\n",
    "def _tokenize_to_label(row: pd.Series) -> str:\n",
    "    row_to_check = {\n",
    "        # VC Funding\n",
    "        \"to\": lambda k: label_write(row, k, 'TO'),\n",
    "        \"from\": lambda k: label_write(row, k, 'FROM'),\n",
    "        \"moment\": lambda k: label_write(row, k, 'MOMENT'),\n",
    "    }\n",
    "\n",
    "    tokenize = row['tokenized_sentence']\n",
    "\n",
    "    for col, function in row_to_check.items():\n",
    "        tokenize_label = function(col)\n",
    "        for key, value in tokenize_label.items():\n",
    "            key_tokenized = text_tokenizer(key).strip()\n",
    "            if key_tokenized in tokenize and not key_tokenized.isspace():\n",
    "                regex_pattern = r\"(?P<start>^|\\W|\\b)(?P<word>{})(?P<end>\\W|\\b|$)\".format(re.escape(key_tokenized))\n",
    "                toknized_value = \" \".join(value)\n",
    "                regex = re.compile(regex_pattern)\n",
    "                tokenize = regex.sub(\n",
    "                    lambda m: f\"{m.group('start')}{toknized_value}{m.group('end')}\"\n",
    "                    if m.group('start') or m.group('end')\n",
    "                    else f\"{toknized_value}\",\n",
    "                    tokenize\n",
    "                )\n",
    "            elif not key_tokenized.isspace():\n",
    "                print(f'Label not found in tokenize: {key_tokenized} - {row[\"text\"]}')\n",
    "                row['KO'] = 'KO'\n",
    "\n",
    "    # Replace all characters who don't start with \"B-\" or \"I-\" in \"o\"\n",
    "    tokenize = \" \".join(\n",
    "        ['o' if not (word.startswith('B-') or word.startswith('I-'))\n",
    "         else word for word in tokenize.split()]\n",
    "    )\n",
    "\n",
    "    return tokenize\n",
    "\n",
    "df['annotations'] = df.apply(lambda x: _tokenize_to_label(x), axis=1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It seems that you are splitting your original dataset into train and validation sets using train_test_split method from scikit-learn library. You are assigning the resulting subsets to train_ds and val_ds respectively.\n",
    "\n",
    "After that, you are adding a new column to each subset called corpus_type with the corresponding value 'TRAIN' or 'VAL'. Finally, you are concatenating train_ds and val_ds subsets into a single dataframe called df.\n",
    "\n",
    "It is common practice to split the original dataset into train and validation sets to evaluate the model's performance on unseen data. By doing this, you can tune your model's hyperparameters based on its performance on the validation set and avoid overfitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:58.567141Z",
     "iopub.status.busy": "2023-04-09T17:20:58.566890Z",
     "iopub.status.idle": "2023-04-09T17:20:58.579249Z",
     "shell.execute_reply": "2023-04-09T17:20:58.578130Z",
     "shell.execute_reply.started": "2023-04-09T17:20:58.567116Z"
    }
   },
   "outputs": [],
   "source": [
    "train_ds, val_ds = train_test_split(df, train_size=0.8, random_state=42)\n",
    "\n",
    "train_ds['corpus_type'] = 'TRAIN'\n",
    "val_ds['corpus_type'] = 'VAL'\n",
    "\n",
    "df = pd.concat([train_ds, val_ds])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# About the Dataset"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code is used to extract unique labels from the annotations in the dataset.\n",
    "\n",
    "It first splits the annotations for each text instance based on whitespaces and then converts them to a list.\n",
    "\n",
    "It then iterates over all the lists of labels to find unique labels by adding all labels that are not in the unique_labels set.\n",
    "\n",
    "Finally, it returns the set of unique labels found in the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:58.582038Z",
     "iopub.status.busy": "2023-04-09T17:20:58.581803Z",
     "iopub.status.idle": "2023-04-09T17:20:58.592611Z",
     "shell.execute_reply": "2023-04-09T17:20:58.591698Z",
     "shell.execute_reply.started": "2023-04-09T17:20:58.582014Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'B-FROM', 'B-MOMENT', 'B-TO', 'I-FROM', 'I-MOMENT', 'I-TO', 'o'}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Split labels based on whitespace and turn them into a list\n",
    "labels = [i.split() for i in df['annotations'].values.tolist()]\n",
    "\n",
    "# Check how many labels are there in the dataset\n",
    "unique_labels = set()\n",
    "\n",
    "for lb in labels:\n",
    "    [unique_labels.add(i) for i in lb if i not in unique_labels]\n",
    "\n",
    "unique_labels\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code creates a dictionary labels_to_ids which maps each unique label in the dataset to a numerical ID. The IDs are assigned in ascending order starting from 0.\n",
    "\n",
    "For example, if there are three unique labels in the dataset: \"B-LOC\", \"I-LOC\", and \"B-ORG\", the labels_to_ids dictionary would be:\n",
    "\n",
    "```json\n",
    "{\"B-LOC\": 0, \"B-ORG\": 1, \"I-LOC\": 2, \"o\": 3}\n",
    "```\n",
    "The ids_to_labels dictionary is also created, which is just the reverse of labels_to_ids and maps numerical IDs to their corresponding labels.\n",
    "\n",
    "Note that the order in which the labels are assigned IDs is determined by the sorted function applied to unique_labels. This means that if the dataset contains the same set of labels in a different order, the mapping of labels to IDs would still be consistent."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:58.594091Z",
     "iopub.status.busy": "2023-04-09T17:20:58.593869Z",
     "iopub.status.idle": "2023-04-09T17:20:59.214160Z",
     "shell.execute_reply": "2023-04-09T17:20:59.213249Z",
     "shell.execute_reply.started": "2023-04-09T17:20:58.594068Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'B-FROM': 0,\n",
       " 'B-MOMENT': 1,\n",
       " 'B-TO': 2,\n",
       " 'I-FROM': 3,\n",
       " 'I-MOMENT': 4,\n",
       " 'I-TO': 5,\n",
       " 'o': 6}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_to_ids = {k: v for v, k in enumerate(sorted(unique_labels))}\n",
    "ids_to_labels = {v: k for v, k in enumerate(sorted(unique_labels))}\n",
    "\n",
    "labels_to_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing\n",
    "Before we are able to use a BERT model to classify the entity of a token, of course, we need to do data preprocessing first, which includes two parts: tokenization and adjusting the label to match the tokenization. Let’s start with tokenization first.\n",
    "\n",
    "## Tokenization\n",
    "Tokenization can be easily implemented with BERT, as we can use BertTokenizerFast class from a pretrained BERT base model with HuggingFace.\n",
    "To give you an example how BERT tokenizer works, let’s take a look at one of the texts from our dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:59.215595Z",
     "iopub.status.busy": "2023-04-09T17:20:59.215322Z",
     "iopub.status.idle": "2023-04-09T17:20:59.221645Z",
     "shell.execute_reply": "2023-04-09T17:20:59.220730Z",
     "shell.execute_reply.started": "2023-04-09T17:20:59.215554Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Je cherche le train le plus rapide pour faire Vireux-molhain Gargenville'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's take a look at how can we preprocess the text - Take first example\n",
    "text = df['text'].values.tolist()\n",
    "example = text[36]\n",
    "\n",
    "example"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code uses the BertTokenizerFast class from the HuggingFace library to tokenize the example text. The tokenizer uses the pretrained BERT base model to perform tokenization and returns a PyTorch tensor that can be used as input to the model. The padding, max_length, and truncation arguments control the length of the input tensor. padding specifies whether to pad the input to the maximum length or not, max_length sets the maximum length of the input, and truncation specifies whether to truncate the input to the maximum length or not. Finally, the return_tensors argument specifies the type of output tensor to be returned."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:59.222724Z",
     "iopub.status.busy": "2023-04-09T17:20:59.222486Z",
     "iopub.status.idle": "2023-04-09T17:20:59.598717Z",
     "shell.execute_reply": "2023-04-09T17:20:59.597978Z",
     "shell.execute_reply.started": "2023-04-09T17:20:59.222703Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[  101, 27901, 22572,  1200,  4386,  5837,  2669,  5837,  4882,  6099,\n",
       "          1162, 11480,  4652,  1162,   159,  5817,  5025,   118,   182,  4063,\n",
       "         10390,  1179,   144,  1813,  4915,  2138,   102,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
       "             0,     0,     0,     0,     0,     0,     0,     0]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0]])}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = BertTokenizerFast.from_pretrained('bert-base-cased')\n",
    "text_tokenized = tokenizer(example, padding='max_length', max_length=128, truncation=True, return_tensors=\"pt\")\n",
    "\n",
    "text_tokenized"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code decodes the tokenized input from the previous step and returns it back to the original text form using the decode() method of the BertTokenizerFast class.\n",
    "\n",
    "Here's what it does step by step:\n",
    "\n",
    "`text_tokenized.input_ids` contains the tokenized representation of the input text. It is a PyTorch tensor containing the numerical representation of the tokens.\n",
    "\n",
    "`text_tokenized.input_ids[0]` gets the first row of this tensor (there's only one row in this case, because we're tokenizing a single example).\n",
    "\n",
    "`tokenizer.decode()` converts the numerical representation of the tokens back into text form using the vocabulary of the tokenizer.\n",
    "\n",
    "Overall, this code is useful for checking how the tokenization process has affected the input text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:59.600132Z",
     "iopub.status.busy": "2023-04-09T17:20:59.599873Z",
     "iopub.status.idle": "2023-04-09T17:20:59.605869Z",
     "shell.execute_reply": "2023-04-09T17:20:59.605117Z",
     "shell.execute_reply.started": "2023-04-09T17:20:59.600090Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[CLS] Je cherche le train le plus rapide pour faire Vireux - molhain Gargenville [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]'"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(text_tokenized.input_ids[0])\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adjusting Label After Tokenization\n",
    "This is a very important step that we need to do after the tokenization process. This is because the length of the sequence is no longer matching the length of the original label after the tokenization process.\n",
    "\n",
    "The BERT tokenizer uses the so-called word-piece tokenizer under the hood, which is a sub-word tokenizer. This means that BERT tokenizer will likely to split one word into one or more meaningful sub-words.\n",
    "\n",
    "This code will convert the input_ids generated by the BERT tokenizer back into the original tokens using the convert_ids_to_tokens() method of the BertTokenizerFast object.\n",
    "\n",
    "For example, if text_tokenized was generated from the sentence \"I love pizza\", then tokenizer.convert_ids_to_tokens(text_tokenized[\"input_ids\"][0]) will return:\n",
    "\n",
    "```\n",
    "['[CLS]', 'I', 'love', 'pizza', '[SEP]', '[PAD]', '[PAD]', ..., '[PAD]']\n",
    "```\n",
    "\n",
    "Here, [CLS] and [SEP] are special tokens used by BERT to mark the beginning and end of a sentence, respectively, while [PAD] is used for padding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:59.607056Z",
     "iopub.status.busy": "2023-04-09T17:20:59.606822Z",
     "iopub.status.idle": "2023-04-09T17:20:59.613956Z",
     "shell.execute_reply": "2023-04-09T17:20:59.613313Z",
     "shell.execute_reply.started": "2023-04-09T17:20:59.607032Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]',\n",
       " 'Je',\n",
       " 'ch',\n",
       " '##er',\n",
       " '##che',\n",
       " 'le',\n",
       " 'train',\n",
       " 'le',\n",
       " 'plus',\n",
       " 'rapid',\n",
       " '##e',\n",
       " 'pour',\n",
       " 'fair',\n",
       " '##e',\n",
       " 'V',\n",
       " '##ire',\n",
       " '##ux',\n",
       " '-',\n",
       " 'm',\n",
       " '##ol',\n",
       " '##hai',\n",
       " '##n',\n",
       " 'G',\n",
       " '##ar',\n",
       " '##gen',\n",
       " '##ville',\n",
       " '[SEP]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]']"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.convert_ids_to_tokens(text_tokenized[\"input_ids\"][0])\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are two problems that we need to address after tokenization process:\n",
    "\n",
    "The addition of special tokens from BERT such as [CLS], [SEP], and [PAD]\n",
    "The fact that some tokens are splitted into sub-words.\n",
    "As sub-word tokenization, word-piece tokenization splits uncommon words into their sub-words, such as ‘Geir’ and ‘Haarde’ in the example above. This sub-word tokenization helps the BERT model to learn the semantic meaning of related words.\n",
    "\n",
    "The consequence of this word piece tokenization and the addition of special tokens from BERT is that the sequence length after tokenization is no longer matching the length of the initial label.\n",
    "\n",
    "From the example above, now there are in total 128 tokens in the sequence after tokenization, while the length of the label is still the same as before. Also, the first token in a sequence is no longer the word ‘Prime’, but the newly added [CLS] token, so we need to shift our label as well.\n",
    "\n",
    "To solve this problem, we need to adjust the label such that it has the same length as the sequence after tokenization. To do this, we can utilize the word_ids method from the tokenization result as follows:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:59.615036Z",
     "iopub.status.busy": "2023-04-09T17:20:59.614802Z",
     "iopub.status.idle": "2023-04-09T17:20:59.621339Z",
     "shell.execute_reply": "2023-04-09T17:20:59.620676Z",
     "shell.execute_reply.started": "2023-04-09T17:20:59.615013Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 2,\n",
       " 3,\n",
       " 4,\n",
       " 5,\n",
       " 6,\n",
       " 6,\n",
       " 7,\n",
       " 8,\n",
       " 8,\n",
       " 9,\n",
       " 9,\n",
       " 9,\n",
       " 10,\n",
       " 11,\n",
       " 11,\n",
       " 11,\n",
       " 11,\n",
       " 12,\n",
       " 12,\n",
       " 12,\n",
       " 12,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_ids = text_tokenized.word_ids()\n",
    "word_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see from the code snippet above, each splitted token shares the same word_ids , where special tokens from BERT such as [CLS], [SEP], and [PAD] all do not have specificword_ids.\n",
    "\n",
    "These word_ids will be very useful to adjust the length of the label by applying either of these two methods:\n",
    "\n",
    "We only provide a label to the first sub-word of each splitted token. The continuation of the sub-word then will simply have ‘-100’ as a label. All tokens that don’t have word_ids will also be labeled with ‘-100’.\n",
    "We provide the same label among all of the sub-words that belong to the same token. All tokens that don’t have word_ids will be labeled with ‘-100’.\n",
    "The function in the code snippet below will do exactly the step defined above.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:59.622423Z",
     "iopub.status.busy": "2023-04-09T17:20:59.622188Z",
     "iopub.status.idle": "2023-04-09T17:20:59.628482Z",
     "shell.execute_reply": "2023-04-09T17:20:59.627463Z",
     "shell.execute_reply.started": "2023-04-09T17:20:59.622400Z"
    }
   },
   "outputs": [],
   "source": [
    "def align_label_example(tokenized_input, labels, label_all_tokens):\n",
    "\n",
    "    word_ids = tokenized_input.word_ids()\n",
    "\n",
    "    previous_word_idx = None\n",
    "    label_ids = []\n",
    "\n",
    "    for word_idx in word_ids:\n",
    "\n",
    "        if word_idx is None:\n",
    "            label_ids.append(-100)\n",
    "\n",
    "        elif word_idx != previous_word_idx:\n",
    "            try:\n",
    "                label_ids.append(labels_to_ids[labels[word_idx]])\n",
    "            except:\n",
    "                label_ids.append(-100)\n",
    "\n",
    "        else:\n",
    "            label_ids.append(labels_to_ids[labels[word_idx]] if label_all_tokens else -100)\n",
    "        previous_word_idx = word_idx\n",
    "\n",
    "\n",
    "    return label_ids\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you want to apply the first method, set label_all_tokens to False. If you want to apply the second method, set label_all_tokens to True, as you can see in the following code snippet:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:59.629959Z",
     "iopub.status.busy": "2023-04-09T17:20:59.629647Z",
     "iopub.status.idle": "2023-04-09T17:20:59.637832Z",
     "shell.execute_reply": "2023-04-09T17:20:59.637084Z",
     "shell.execute_reply.started": "2023-04-09T17:20:59.629929Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-100,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 3,\n",
       " 3,\n",
       " 3,\n",
       " 3,\n",
       " 3,\n",
       " 2,\n",
       " 2,\n",
       " 2,\n",
       " 2,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label = labels[36]\n",
    "\n",
    "#If we set label_all_tokens to True.....\n",
    "label_all_tokens = True\n",
    "\n",
    "new_label = align_label_example(text_tokenized, label, label_all_tokens)\n",
    "new_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:59.640658Z",
     "iopub.status.busy": "2023-04-09T17:20:59.640399Z",
     "iopub.status.idle": "2023-04-09T17:20:59.647822Z",
     "shell.execute_reply": "2023-04-09T17:20:59.647024Z",
     "shell.execute_reply.started": "2023-04-09T17:20:59.640633Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-100,\n",
       " 6,\n",
       " 6,\n",
       " -100,\n",
       " -100,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " 6,\n",
       " -100,\n",
       " 6,\n",
       " 6,\n",
       " -100,\n",
       " 0,\n",
       " -100,\n",
       " -100,\n",
       " 3,\n",
       " 3,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " 2,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100,\n",
       " -100]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_all_tokens = False\n",
    "\n",
    "new_label = align_label_example(text_tokenized, label, label_all_tokens)\n",
    "new_label\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset Class\n",
    "Before we train our BERT model for NER task, we need to create a dataset class to generate and fetch data in a batch.\n",
    "\n",
    "The align_label function takes in two arguments: texts and labels. texts is a list of strings representing the text data to be tokenized and labels is a list of lists representing the labels associated with each token in the text data.\n",
    "\n",
    "The function first tokenizes the input texts using the BERT tokenizer with the specified parameters. It then retrieves the word IDs for each token in the input texts.\n",
    "\n",
    "The function then iterates over each word ID and determines the label associated with each token. If the word ID is None, the function appends -100 to the label_ids list. If the word ID is not None, the function checks whether the word ID is the same as the previous word ID. If it is not the same, the function looks up the label associated with the current word ID in the labels_to_ids dictionary and appends the corresponding label ID to the label_ids list. If the word ID is the same as the previous word ID, the function determines whether to append the label ID or -100 based on the label_all_tokens parameter. If the label is not found in the labels_to_ids dictionary, -100 is appended to the label_ids list. Finally, the function updates the previous word ID with the current word ID and returns the label_ids list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:59.649472Z",
     "iopub.status.busy": "2023-04-09T17:20:59.649137Z",
     "iopub.status.idle": "2023-04-09T17:20:59.655728Z",
     "shell.execute_reply": "2023-04-09T17:20:59.654968Z",
     "shell.execute_reply.started": "2023-04-09T17:20:59.649398Z"
    }
   },
   "outputs": [],
   "source": [
    "def align_label(texts, labels):\n",
    "    tokenized_inputs = tokenizer(texts, padding='max_length', max_length=128, truncation=True)\n",
    "\n",
    "    word_ids = tokenized_inputs.word_ids()\n",
    "\n",
    "    previous_word_idx = None\n",
    "    label_ids = []\n",
    "\n",
    "    for word_idx in word_ids:\n",
    "\n",
    "        if word_idx is None:\n",
    "            label_ids.append(-100)\n",
    "\n",
    "        elif word_idx != previous_word_idx:\n",
    "            try:\n",
    "                label_ids.append(labels_to_ids[labels[word_idx]])\n",
    "            except:\n",
    "                label_ids.append(-100)\n",
    "        else:\n",
    "            try:\n",
    "                label_ids.append(labels_to_ids[labels[word_idx]] if label_all_tokens else -100)\n",
    "            except:\n",
    "                label_ids.append(-100)\n",
    "        previous_word_idx = word_idx\n",
    "\n",
    "    return label_ids"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a custom PyTorch dataset class called `DataSequence`. It takes in a pandas DataFrame df and performs data preprocessing to tokenize the text data and align the labels with the tokenization.\n",
    "\n",
    "The `__init__` method of the class initializes two lists self.texts and self.labels. The self.texts list stores the tokenized versions of the text data using the tokenizer object. The self.labels list is created by calling the align_label function for each text and its corresponding labels. This function aligns the labels with the tokenization and returns a list of label IDs.\n",
    "\n",
    "The `__len__` method returns the length of the self.labels list, which is the number of samples in the dataset.\n",
    "\n",
    "The `get_batch_data` method takes an index idx as input and returns the tokenized text data at that index.\n",
    "\n",
    "The `get_batch_labels` method takes an index idx as input and returns the label IDs for the text data at that index.\n",
    "\n",
    "Finally, the `__getitem__` method takes an index idx as input and returns a tuple containing the tokenized text data and the corresponding label IDs for the text data at that index. This method is called by the PyTorch DataLoader to generate batches of data during training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:59.656977Z",
     "iopub.status.busy": "2023-04-09T17:20:59.656701Z",
     "iopub.status.idle": "2023-04-09T17:20:59.664284Z",
     "shell.execute_reply": "2023-04-09T17:20:59.663301Z",
     "shell.execute_reply.started": "2023-04-09T17:20:59.656952Z"
    }
   },
   "outputs": [],
   "source": [
    "class DataSequence(torch.utils.data.Dataset):\n",
    "\n",
    "    def __init__(self, df):\n",
    "\n",
    "        lb = [i.split() for i in df['annotations'].values.tolist()]\n",
    "        txt = df['text'].values.tolist()\n",
    "        self.texts = [tokenizer(str(i),  padding='max_length', max_length=128, truncation=True, return_tensors=\"pt\") for i in txt]\n",
    "        self.labels = [align_label(i,j) for i,j in zip(txt, lb)]\n",
    "        print(len(self.labels))\n",
    "        \n",
    "    def __len__(self):\n",
    "\n",
    "        return len(self.labels)\n",
    "\n",
    "    def get_batch_data(self, idx):\n",
    "\n",
    "        return self.texts[idx]\n",
    "\n",
    "    def get_batch_labels(self, idx):\n",
    "\n",
    "        return torch.LongTensor(self.labels[idx])\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        batch_data = self.get_batch_data(idx)\n",
    "        batch_labels = self.get_batch_labels(idx)\n",
    "\n",
    "        return batch_data, batch_labels\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the code snippet above, we call BertTokenizerFast class with tokenizer variable in the __init__ function to tokenize our input texts, and align_label function to adjust our label after tokenization process.\n",
    "\n",
    "Next, let’s split our data randomly into training, vaidation, and test. However, mind you that the total number of data is 2000."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:59.665642Z",
     "iopub.status.busy": "2023-04-09T17:20:59.665362Z",
     "iopub.status.idle": "2023-04-09T17:20:59.672569Z",
     "shell.execute_reply": "2023-04-09T17:20:59.671841Z",
     "shell.execute_reply.started": "2023-04-09T17:20:59.665618Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: 1600\n",
      "Val: 400\n"
     ]
    }
   ],
   "source": [
    "df_train = df[df['corpus_type'] == 'TRAIN']\n",
    "df_val = df[df['corpus_type'] == 'VAL']\n",
    "\n",
    "print(f'Train: {len(df_train)}')\n",
    "print(f'Val: {len(df_val)}')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Building\n",
    "\n",
    "The code defines a PyTorch neural network model class called BertNerModel. This class inherits from the torch.nn.Module class and implements a forward method, which defines how the input data is propagated through the layers of the model.\n",
    "\n",
    "The BertNerModel uses the pre-trained BertForTokenClassification model from the HuggingFace transformers library as the base model. This pre-trained model is initialized with the 'bert-base-cased' architecture, which is a BERT model with 12 layers, 768 hidden units, and 110M parameters.\n",
    "\n",
    "The num_labels parameter is set to the number of unique labels in the dataset, which is used to initialize the final classification layer of the model.\n",
    "\n",
    "The forward method takes in three parameters - input_id, mask, and labels.\n",
    "\n",
    "input_id is the tokenized input sequence of shape (batch_size, sequence_length).\n",
    "mask is the attention mask of shape (batch_size, sequence_length).\n",
    "labels is the true label sequence of shape (batch_size, sequence_length).\n",
    "The forward method passes the input_id and mask through the bert model and returns the output, which is the predicted label sequence of shape (batch_size, sequence_length, num_labels).\n",
    "\n",
    "The labels parameter is also passed to the bert model to compute the loss during training.\n",
    "\n",
    "## Bert Model Architecture\n",
    "![Bert Model Architecture](docs/bert_base_architecture.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:20:59.673962Z",
     "iopub.status.busy": "2023-04-09T17:20:59.673504Z",
     "iopub.status.idle": "2023-04-09T17:20:59.679409Z",
     "shell.execute_reply": "2023-04-09T17:20:59.678485Z",
     "shell.execute_reply.started": "2023-04-09T17:20:59.673937Z"
    }
   },
   "outputs": [],
   "source": [
    "class BertNerModel(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(BertNerModel, self).__init__()\n",
    "        self.bert = BertForTokenClassification.from_pretrained('bert-base-cased', num_labels=len(unique_labels))\n",
    "\n",
    "    def forward(self, input_id, mask, labels):\n",
    "        output = self.bert(input_ids=input_id, attention_mask=mask, return_dict=False, labels=labels)\n",
    "\n",
    "        return output"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Loop\n",
    "\n",
    "This is a function named `train` that takes two arguments `df_train` and `df_val` which are pandas dataframes containing the training and validation data respectively.\n",
    "\n",
    "The function first checks if a GPU is available and assigns the device accordingly. It then initializes a `BertNerModel` object and moves it to the selected device.\n",
    "\n",
    "The function then creates `DataSequence` objects for both the training and validation data and `DataLoader` objects for both the `DataSequence` objects. The training dataloader has `shuffle=True` to shuffle the batches before each epoch.\n",
    "\n",
    "The function then initializes the optimizer as `SGD` with a learning rate of `5e-3` and momentum of `0.9`. It also initializes a learning rate scheduler with a step size of `3` and gamma of `0.1`.\n",
    "\n",
    "The function then starts an outer loop that runs for `EPOCHS` number of times. Inside this loop, it first sets `total_acc_train` and `total_loss_train` to 0 and then sets the model to training mode using `model.train()`.\n",
    "\n",
    "The function then iterates over the batches in the training dataloader using a `for` loop and extracts the input and label tensors from the batch. It then moves them to the selected device using `to(device)`.\n",
    "\n",
    "The function then sets the optimizer gradients to zero using `optimizer.zero_grad()`, performs forward pass through the model using the input tensors using `model(input_id, mask, train_label)`, and calculates the loss.\n",
    "\n",
    "For each training instance in the batch, it then removes the padded tokens from the logits and labels using the `train_label[i] != -100` condition and calculates the accuracy by comparing the predicted labels with the true labels. It accumulates the accuracy and loss for the entire batch.\n",
    "\n",
    "After the inner loop finishes, the function sets the model to evaluation mode using `model.eval()` and iterates over the batches in the validation dataloader, performs forward pass through the model, removes the padded tokens, and calculates accuracy and loss for each validation instance in the batch. It then accumulates the accuracy and loss for the entire validation set.\n",
    "\n",
    "Finally, the function calculates the average accuracy and loss for the entire training and validation sets and prints them out for the current epoch.\n",
    "\n",
    "After the outer loop finishes, the function returns a dictionary containing the trained model, training accuracy, training loss, validation accuracy, and validation loss."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:24:15.922800Z",
     "iopub.status.busy": "2023-04-09T17:24:15.922072Z",
     "iopub.status.idle": "2023-04-09T17:24:15.935092Z",
     "shell.execute_reply": "2023-04-09T17:24:15.934221Z",
     "shell.execute_reply.started": "2023-04-09T17:24:15.922771Z"
    }
   },
   "outputs": [],
   "source": [
    "def train(df_train, df_val):\n",
    "    use_cuda = torch.cuda.is_available()\n",
    "    device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "    \n",
    "    model = BertNerModel().to(device)\n",
    "    \n",
    "    train_dataset = DataSequence(df_train)\n",
    "    val_dataset = DataSequence(df_val)\n",
    "\n",
    "    train_dataloader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n",
    "    val_dataloader = DataLoader(val_dataset, batch_size=8)\n",
    "\n",
    "    optimizer = SGD(model.parameters(), lr=5e-3, momentum=0.9)\n",
    "    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)\n",
    "    \n",
    "    for epoch_num in range(EPOCHS):\n",
    "\n",
    "        total_acc_train = 0\n",
    "        total_loss_train = 0\n",
    "\n",
    "        model.train()\n",
    "\n",
    "        for train_data, train_label in tqdm(train_dataloader):\n",
    "\n",
    "            train_label = train_label.to(device)\n",
    "            mask = train_data['attention_mask'].squeeze(1).to(device)\n",
    "            input_id = train_data['input_ids'].squeeze(1).to(device)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            loss, logits = model(input_id, mask, train_label)\n",
    "\n",
    "            for i in range(logits.shape[0]):\n",
    "\n",
    "                logits_clean = logits[i][train_label[i] != -100]\n",
    "                label_clean = train_label[i][train_label[i] != -100]\n",
    "\n",
    "                predictions = logits_clean.argmax(dim=1)\n",
    "                acc = (predictions == label_clean).float().mean()\n",
    "                total_acc_train += acc.item()\n",
    "                total_loss_train += loss.item()\n",
    "\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "        model.eval()\n",
    "\n",
    "        total_acc_val = 0\n",
    "        total_loss_val = 0\n",
    "\n",
    "        for val_data, val_label in val_dataloader:\n",
    "\n",
    "            val_label = val_label.to(device)\n",
    "            mask = val_data['attention_mask'].squeeze(1).to(device)\n",
    "            input_id = val_data['input_ids'].squeeze(1).to(device)\n",
    "\n",
    "            loss, logits = model(input_id, mask, val_label)\n",
    "\n",
    "            for i in range(logits.shape[0]):\n",
    "\n",
    "                logits_clean = logits[i][val_label[i] != -100]\n",
    "                label_clean = val_label[i][val_label[i] != -100]\n",
    "\n",
    "                predictions = logits_clean.argmax(dim=1)\n",
    "                acc = (predictions == label_clean).float().mean()\n",
    "                total_acc_val += acc.item()\n",
    "                total_loss_val += loss.item()\n",
    "                \n",
    "        train_accuracy = total_acc_train / len(df_train)\n",
    "        train_loss = total_loss_train / len(df_train)\n",
    "        val_accuracy = total_acc_val / len(df_val)\n",
    "        val_loss = total_loss_val / len(df_val)\n",
    "        \n",
    "        scheduler.step()\n",
    "        \n",
    "        print(f'Epoch: {epoch_num} - Train_Loss: {train_loss} | Train_Accuracy: {train_accuracy} | Val_Loss: {val_loss} | Val_Accuracy: {val_accuracy}')\n",
    "\n",
    "    return {\n",
    "        \"model\": model,\n",
    "        \"train_accuracy\": train_accuracy,\n",
    "        \"train_loss\": train_loss,\n",
    "        \"val_accuracy\": val_accuracy,\n",
    "        \"val_loss\": val_loss\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:24:16.621187Z",
     "iopub.status.busy": "2023-04-09T17:24:16.620837Z",
     "iopub.status.idle": "2023-04-09T17:30:19.436701Z",
     "shell.execute_reply": "2023-04-09T17:30:19.435560Z",
     "shell.execute_reply.started": "2023-04-09T17:24:16.621161Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1600\n",
      "400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:38<00:00,  5.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 - Train_Loss: 0.40922343515791 | Train_Accuracy: 0.8443625670881011 | Val_Loss: 0.6086626607179642 | Val_Accuracy: 0.7595963321253657\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:32<00:00,  6.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 - Train_Loss: 0.24247286426834763 | Train_Accuracy: 0.9230699844937772 | Val_Loss: 0.17702844887971877 | Val_Accuracy: 0.947296773865819\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:32<00:00,  6.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 - Train_Loss: 0.11822694221278653 | Train_Accuracy: 0.9694153612665832 | Val_Loss: 0.0428910940955393 | Val_Accuracy: 0.9890069732069969\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:32<00:00,  6.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 - Train_Loss: 0.04157181476708502 | Train_Accuracy: 0.9886558531783521 | Val_Loss: 0.011918385439785198 | Val_Accuracy: 0.997599838078022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:32<00:00,  6.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 - Train_Loss: 0.01779696991550736 | Train_Accuracy: 0.996368617117405 | Val_Loss: 0.0076661504665389655 | Val_Accuracy: 0.9981000185012817\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:32<00:00,  6.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5 - Train_Loss: 0.011867836708552204 | Train_Accuracy: 0.9978275588154792 | Val_Loss: 0.00703899716201704 | Val_Accuracy: 0.9982666851580143\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:32<00:00,  6.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 - Train_Loss: 0.009062750204466284 | Train_Accuracy: 0.9984293616190553 | Val_Loss: 0.006866968900430948 | Val_Accuracy: 0.998405573964119\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:32<00:00,  6.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7 - Train_Loss: 0.007990534739510622 | Train_Accuracy: 0.9987827615439892 | Val_Loss: 0.006878538551973179 | Val_Accuracy: 0.9981783013045787\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:32<00:00,  6.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 - Train_Loss: 0.00782390717882663 | Train_Accuracy: 0.9985744554921985 | Val_Loss: 0.006687318570329808 | Val_Accuracy: 0.998405573964119\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [00:32<00:00,  6.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 - Train_Loss: 0.00773470028914744 | Train_Accuracy: 0.9987424545362592 | Val_Loss: 0.0066637948807328935 | Val_Accuracy: 0.998405573964119\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 10\n",
    "\n",
    "results = train(df_train, df_val) "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation\n",
    "\n",
    "This code defines a function called `evaluate` that takes a trained model and a test dataset as input, and computes the precision, recall, and F1-score of the model's predictions on the test dataset. The function uses PyTorch to make predictions on the test dataset and then computes evaluation metrics based on these predictions.\n",
    "\n",
    "The function first creates a PyTorch `DataLoader` object for the test dataset and sets up some variables to track evaluation metrics, such as the number of correct predictions (`total_sum_ok`) and the number of false positives (`total_fp`) and false negatives (`total_fn`) across all examples in the test dataset. It also creates empty dictionaries to store grouped evaluation metrics for each label in the dataset.\n",
    "\n",
    "Next, the function loops over each batch in the test dataloader and makes predictions using the trained model. For each example in the batch, the function computes the metrics described above (e.g., sum_ok, fp, and fn), and updates the tracking variables accordingly. At the end of the loop, the function computes the overall precision, recall, and F1-score for the entire test dataset, as well as the grouped evaluation metrics for each label.\n",
    "\n",
    "Finally, the function prints out the evaluation metrics and returns a dictionary containing the precision, recall, and F1-score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:35:47.957246Z",
     "iopub.status.busy": "2023-04-09T17:35:47.956375Z",
     "iopub.status.idle": "2023-04-09T17:35:47.973310Z",
     "shell.execute_reply": "2023-04-09T17:35:47.972547Z",
     "shell.execute_reply.started": "2023-04-09T17:35:47.957217Z"
    }
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "def evaluate(model, df_test):\n",
    "\n",
    "    test_dataset = DataSequence(df_test)\n",
    "\n",
    "    test_dataloader = DataLoader(test_dataset, batch_size=1)\n",
    "\n",
    "    use_cuda = torch.cuda.is_available()\n",
    "    device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "\n",
    "    if use_cuda:\n",
    "        model = model.cuda()\n",
    "\n",
    "    total_sum_ok = 0\n",
    "    grouped_sum_ok = {k[1]: 0 for k in ids_to_labels.items()}\n",
    "    total_nb_elem = 0\n",
    "    grouped_fp = {k[1]: 0 for k in ids_to_labels.items()}\n",
    "    total_fp = 0\n",
    "    grouped_fn = {k[1]: 0 for k in ids_to_labels.items()}\n",
    "    total_fn = 0\n",
    "\n",
    "    for test_data, test_label in test_dataloader:\n",
    "\n",
    "        test_label = test_label[0].to(device)\n",
    "        mask = test_data['attention_mask'][0].to(device)\n",
    "        input_id = test_data['input_ids'][0].to(device)\n",
    "\n",
    "        loss, logits = model(input_id, mask, test_label.long())\n",
    "\n",
    "        logits_clean = logits[0][test_label != -100]\n",
    "        label_clean = test_label[test_label != -100]\n",
    "\n",
    "        predictions = logits_clean.argmax(dim=1)\n",
    "\n",
    "        i = 0\n",
    "        nb_elem = 0\n",
    "        sum_ok = 0\n",
    "        fp = 0\n",
    "        fn = 0\n",
    "        \n",
    "        o_index = [k for k, v in ids_to_labels.items() if v == 'o'][0]\n",
    "\n",
    "        for _ in label_clean:\n",
    "            if label_clean[i] != o_index or predictions[i] != o_index:\n",
    "                if predictions[i] == label_clean[i]:\n",
    "                    sum_ok=sum_ok+1\n",
    "                    grouped_sum_ok[ids_to_labels[label_clean[i].item()]] = grouped_sum_ok[ids_to_labels[label_clean[i].item()]] + 1\n",
    "                if predictions[i] != label_clean[i] and predictions[i] != o_index:\n",
    "                    fp = fp+1\n",
    "                    grouped_fp[ids_to_labels[predictions[i].item()]] = grouped_fp[ids_to_labels[predictions[i].item()]] + 1\n",
    "                if predictions[i] != label_clean[i] and label_clean[i] != o_index:\n",
    "                    fn = fn+1\n",
    "                    grouped_fn[ids_to_labels[label_clean[i].item()]] = grouped_fn[ids_to_labels[label_clean[i].item()]] + 1\n",
    "                nb_elem=nb_elem+1\n",
    "            i=i+1\n",
    "        if nb_elem > 0:\n",
    "            total_sum_ok = total_sum_ok+sum_ok\n",
    "            total_nb_elem = total_nb_elem+nb_elem\n",
    "            total_fp = total_fp+fp\n",
    "            total_fn = total_fn+fn\n",
    "            \n",
    "    grouped_precision = defaultdict(int)\n",
    "    grouped_recall = defaultdict(int)\n",
    "    grouped_f1 = defaultdict(int)\n",
    "\n",
    "    total_precision = total_sum_ok / (total_sum_ok+total_fp)\n",
    "    total_recall = total_sum_ok / (total_sum_ok+total_fn)\n",
    "    total_f1 = 2*((total_precision*total_recall)/(total_precision+total_recall))\n",
    "    \n",
    "    for i in grouped_fn.keys():\n",
    "        try:\n",
    "            grouped_precision[i] = grouped_sum_ok[i] / (grouped_sum_ok[i] + grouped_fp[i])\n",
    "        except ZeroDivisionError:\n",
    "            grouped_precision[i] = 0\n",
    "\n",
    "    for i in grouped_fn.keys():\n",
    "        try:\n",
    "            grouped_recall[i] = grouped_sum_ok[i] / (grouped_sum_ok[i] + grouped_fn[i])\n",
    "        except ZeroDivisionError:\n",
    "            grouped_recall[i] = 0\n",
    "\n",
    "    for i in grouped_fn.keys():\n",
    "        try:\n",
    "            grouped_f1[i] = 2 * ((grouped_precision[i] * grouped_recall[i]) / (grouped_precision[i] + grouped_recall[i]))\n",
    "        except ZeroDivisionError:\n",
    "            grouped_f1[i] = 0\n",
    "\n",
    "    print(f'Test Precision: {total_precision: .3f}')\n",
    "    print(f'Test Recall: {total_recall: .3f}')\n",
    "    print(f'Test F1: {total_f1: .3f}')\n",
    "    \n",
    "    print('======== GROUPED EVALUATED ========')\n",
    "    # regroup f1, precision and recall by label\n",
    "    for k, v in grouped_precision.items():\n",
    "        print(f'{k} Precision:    {v: .3f}')\n",
    "    for k, v in grouped_recall.items():\n",
    "        print(f'{k} Recall:       {v: .3f}')\n",
    "    for k, v in grouped_f1.items():\n",
    "        print(f'{k} F1:           {v: .3f}')\n",
    "    \n",
    "    return {\"precision\": total_precision, \"recall\": total_recall, \"F1\": total_f1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:35:49.202586Z",
     "iopub.status.busy": "2023-04-09T17:35:49.202183Z",
     "iopub.status.idle": "2023-04-09T17:35:55.795411Z",
     "shell.execute_reply": "2023-04-09T17:35:55.794636Z",
     "shell.execute_reply.started": "2023-04-09T17:35:49.202558Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "400\n",
      "Test Precision:  0.998\n",
      "Test Recall:  0.997\n",
      "Test F1:  0.997\n",
      "======== GROUPED EVALUATED ========\n",
      "B-FROM Precision:     1.000\n",
      "B-MOMENT Precision:     1.000\n",
      "B-TO Precision:     1.000\n",
      "I-FROM Precision:     0.997\n",
      "I-MOMENT Precision:     1.000\n",
      "I-TO Precision:     0.995\n",
      "o Precision:     0.000\n",
      "B-FROM Recall:        0.998\n",
      "B-MOMENT Recall:        0.963\n",
      "B-TO Recall:        1.000\n",
      "I-FROM Recall:        0.995\n",
      "I-MOMENT Recall:        1.000\n",
      "I-TO Recall:        1.000\n",
      "o Recall:        0.000\n",
      "B-FROM F1:            0.999\n",
      "B-MOMENT F1:            0.981\n",
      "B-TO F1:            1.000\n",
      "I-FROM F1:            0.996\n",
      "I-MOMENT F1:            1.000\n",
      "I-TO F1:            0.998\n",
      "o F1:            0.000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'precision': 0.9977638640429338,\n",
       " 'recall': 0.9968722073279714,\n",
       " 'F1': 0.9973178363880197}"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(results['model'], df_val)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation on one example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:36:03.771385Z",
     "iopub.status.busy": "2023-04-09T17:36:03.770562Z",
     "iopub.status.idle": "2023-04-09T17:36:03.783880Z",
     "shell.execute_reply": "2023-04-09T17:36:03.782705Z",
     "shell.execute_reply.started": "2023-04-09T17:36:03.771345Z"
    }
   },
   "outputs": [],
   "source": [
    "def align_word_ids(texts):\n",
    "\n",
    "    tokenized_inputs = tokenizer(texts, padding='max_length', max_length=128, truncation=True)\n",
    "\n",
    "    word_ids = tokenized_inputs.word_ids()\n",
    "\n",
    "    previous_word_idx = None\n",
    "    label_ids = []\n",
    "\n",
    "    for word_idx in word_ids:\n",
    "\n",
    "        if word_idx is None:\n",
    "            label_ids.append(-100)\n",
    "\n",
    "        elif word_idx != previous_word_idx:\n",
    "            try:\n",
    "                label_ids.append(1)\n",
    "            except:\n",
    "                label_ids.append(-100)\n",
    "        else:\n",
    "            try:\n",
    "                label_ids.append(1 if label_all_tokens else -100)\n",
    "            except:\n",
    "                label_ids.append(-100)\n",
    "        previous_word_idx = word_idx\n",
    "\n",
    "    return label_ids\n",
    "\n",
    "\n",
    "def evaluate_one_text(model, sentence):\n",
    "\n",
    "\n",
    "    use_cuda = torch.cuda.is_available()\n",
    "    device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "\n",
    "    if use_cuda:\n",
    "        model = model.cuda()\n",
    "\n",
    "    text = tokenizer(sentence, padding='max_length', max_length = 128, truncation=True, return_tensors=\"pt\")\n",
    "\n",
    "    mask = text['attention_mask'].to(device)\n",
    "    input_id = text['input_ids'].to(device)\n",
    "    label_ids = torch.Tensor(align_word_ids(sentence)).unsqueeze(0).to(device)\n",
    "\n",
    "    logits = model(input_id, mask, None)\n",
    "    logits_clean = logits[0][label_ids != -100]\n",
    "\n",
    "    predictions = logits_clean.argmax(dim=1).tolist()\n",
    "    prediction_label = [ids_to_labels[i] for i in predictions]\n",
    "    print(sentence)\n",
    "    print(prediction_label)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-09T17:36:26.790867Z",
     "iopub.status.busy": "2023-04-09T17:36:26.790537Z",
     "iopub.status.idle": "2023-04-09T17:36:26.839718Z",
     "shell.execute_reply": "2023-04-09T17:36:26.838918Z",
     "shell.execute_reply.started": "2023-04-09T17:36:26.790843Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Je voudrais partir en vacance à Lyon la semaine prochaine en partant de Marseille\n",
      "['o', 'o', 'o', 'o', 'o', 'o', 'B-TO', 'B-MOMENT', 'I-MOMENT', 'I-MOMENT', 'o', 'o', 'o', 'B-FROM']\n"
     ]
    }
   ],
   "source": [
    "evaluate_one_text(results['model'], 'Je voudrais partir en vacance à Lyon la semaine prochaine en partant de Marseille')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}