fine_tune_hate_speech.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "627b7a2c",
   "metadata": {},
   "source": [
    "### Fine tune an HF model on Italian Hate Speech detection\n",
    "* starting point: bert-base-italian-cased\n",
    "* fine tuned on: cardiffnlp/tweet_sentiment_multilingual, italian subset\n",
    "* + a set of email generated for one Town demo (contained in a csv file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8cf55f6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2023-11-27 19:10:59,781] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from datasets import load_dataset, concatenate_datasets, ClassLabel, Features, Value\n",
    "from transformers import AutoTokenizer\n",
    "from transformers import AutoModelForSequenceClassification\n",
    "from transformers import TrainingArguments\n",
    "from transformers import TrainingArguments, Trainer\n",
    "import evaluate\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "005b1f74",
   "metadata": {},
   "outputs": [],
   "source": [
    "# contains email and sentiment to add to the training dataset\n",
    "# hand labelled\n",
    "\n",
    "DATASET_NAME = \"Paul/hatecheck-italian\"\n",
    "\n",
    "PRETRAINED_MODEL_NAME = \"MilaNLProc/hate-ita-xlm-r-large\"\n",
    "\n",
    "features = Features({\n",
    "    'test_case': Value('string'),\n",
    "    'label_gold': ClassLabel(names=['non-hateful', 'hateful'])\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "22ed992c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# labels are defined in english since the original dataset use english labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6a9d6f77",
   "metadata": {},
   "outputs": [],
   "source": [
    "# all tools here\n",
    "tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(PRETRAINED_MODEL_NAME, num_labels=2)\n",
    "\n",
    "metric = evaluate.load(\"accuracy\")\n",
    "\n",
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True)\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    \n",
    "    acc = metric.compute(predictions=predictions, references=labels)[\"accuracy\"]\n",
    "    f1 =  f1_score(labels, predictions, average='macro')\n",
    "    \n",
    "    return {\"f1_macro\": f1, \"accuracy\": acc}\n",
    "\n",
    "# for my custom dataset\n",
    "def remove_non_breaking_space(example):\n",
    "    # Assuming the text is under the key 'text'\n",
    "    example['text'] = example['text'].replace('\\xa0', ' ')\n",
    "    return example\n",
    "\n",
    "def to_lower(example):\n",
    "    example['text'] = [s.lower() for s in example['text']]\n",
    "    \n",
    "    return example"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a0de9b0",
   "metadata": {},
   "source": [
    "### Load and preprocess the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "aa96145f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load hatespeech ita dataset\n",
    "dataset = load_dataset(DATASET_NAME, usecols=['test_case', 'label_gold'], features=features)[\"test\"]\n",
    "\n",
    "dataset = dataset.shuffle(seed=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c0fa3d4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'test_case': 'Le recensioni di quel film sono semplicemente patetiche.',\n",
       " 'label_gold': 0}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# have a look at one example\n",
    "dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8b0a77ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'test_case': Value(dtype='string', id=None),\n",
       " 'label_gold': ClassLabel(names=['non-hateful', 'hateful'], id=None)}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "42dbd894",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenize\n",
    "dataset = dataset.rename_column(\"test_case\", \"text\")\n",
    "dataset = dataset.rename_column(\"label_gold\", \"label\")\n",
    "\n",
    "dataset = dataset.map(to_lower, batched=True)\n",
    "tokenized_datasets = dataset.map(tokenize_function, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "61dd0b07",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_datasets_train_test_split = tokenized_datasets.train_test_split(test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "24666f55",
   "metadata": {},
   "outputs": [],
   "source": [
    "# build the training and validation dataset\n",
    "# we add the custom dataset, from csv, only to training dataset\n",
    "train_ds = tokenized_datasets_train_test_split[\"train\"].shuffle(seed=42)\n",
    "eval_ds = tokenized_datasets_train_test_split[\"test\"].shuffle(seed=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "09eb59f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have 3321 samples in training dataset...\n",
      "We have 369 samples in validation dataset...\n"
     ]
    }
   ],
   "source": [
    "print(f\"We have {len(train_ds)} samples in training dataset...\")\n",
    "print(f\"We have {len(eval_ds)} samples in validation dataset...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4d256957",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text', 'label', 'input_ids', 'attention_mask'],\n",
       "    num_rows: 3321\n",
       "})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ds"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65f83826",
   "metadata": {},
   "source": [
    "### Prepare for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "be1c89e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "SAVE_DIR = \"haspeech_ita\"\n",
    "\n",
    "EPOCHS = 3\n",
    "\n",
    "batch_size = 8\n",
    "\n",
    "training_args = TrainingArguments(output_dir=SAVE_DIR, \n",
    "                                  evaluation_strategy=\"steps\",\n",
    "                                  save_strategy=\"steps\",\n",
    "                                  logging_strategy=\"steps\",\n",
    "                                  num_train_epochs=EPOCHS,\n",
    "                                  logging_steps=100,\n",
    "                                  save_steps=100,\n",
    "                                  eval_steps=100,\n",
    "                                  warmup_steps=500,\n",
    "                                  learning_rate=2e-5,\n",
    "                                  load_best_model_at_end=True,\n",
    "                                  metric_for_best_model=\"accuracy\",\n",
    "                                  greater_is_better=True,\n",
    "                                  save_total_limit=1,\n",
    "                                  per_device_train_batch_size=8\n",
    "                                 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "bcf2ac59",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_ds,\n",
    "    eval_dataset=eval_ds,\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d832eef6",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1b393486",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='624' max='624' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [624/624 18:34, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>F1 Macro</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>0.400100</td>\n",
       "      <td>0.172156</td>\n",
       "      <td>0.919847</td>\n",
       "      <td>0.937669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.166900</td>\n",
       "      <td>0.035503</td>\n",
       "      <td>0.986532</td>\n",
       "      <td>0.989160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>0.082400</td>\n",
       "      <td>0.083852</td>\n",
       "      <td>0.986368</td>\n",
       "      <td>0.989160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>0.067200</td>\n",
       "      <td>0.027654</td>\n",
       "      <td>0.993266</td>\n",
       "      <td>0.994580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.076100</td>\n",
       "      <td>0.048303</td>\n",
       "      <td>0.993266</td>\n",
       "      <td>0.994580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>0.041300</td>\n",
       "      <td>0.024969</td>\n",
       "      <td>0.996623</td>\n",
       "      <td>0.997290</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=624, training_loss=0.13370367174203962, metrics={'train_runtime': 1117.1415, 'train_samples_per_second': 8.918, 'train_steps_per_second': 0.559, 'total_flos': 9284832172836864.0, 'train_loss': 0.13370367174203962, 'epoch': 3.0})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32cd3275",
   "metadata": {},
   "source": [
    "### Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0fd91b3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('haspeech_ita/tokenizer_config.json',\n",
       " 'haspeech_ita/special_tokens_map.json',\n",
       " 'haspeech_ita/sentencepiece.bpe.model',\n",
       " 'haspeech_ita/added_tokens.json',\n",
       " 'haspeech_ita/tokenizer.json')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save_pretrained(SAVE_DIR)\n",
    "tokenizer.save_pretrained(SAVE_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4468d5e0",
   "metadata": {},
   "source": [
    "### Save to HF Hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2954fbe1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d02cb4584c6b4600814efd47aab609a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "events.out.tfevents.1701111657.eee07ad15600.1142.0:   0%|          | 0.00/5.54k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd8753f0c09049e78654b8025dcc7f25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload 5 LFS files:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "451ba61394ce4e5281edd593149fdfbf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/2.24G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3f51d36a2f44a708dc97fd30f64ad46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "training_args.bin:   0%|          | 0.00/4.16k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "192d03f7675549a9a8431e2121285d3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "events.out.tfevents.1701112130.eee07ad15600.1142.1:   0%|          | 0.00/4.59k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fff1f2f3ff7548c9897dd99da5198a00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "events.out.tfevents.1701112270.eee07ad15600.2832.0:   0%|          | 0.00/8.13k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/luigisaetta/haspeech_ita/tree/main/'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.push_to_hub(SAVE_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd253d28",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch20_p39_gpu_v2]",
   "language": "python",
   "name": "conda-env-pytorch20_p39_gpu_v2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}