model_finetuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b42d90ee",
   "metadata": {},
   "source": [
    "# Summary source prediction: Model finetuning\n",
    "\n",
    "Sébastien Meyer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c8b2d910",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from datasets import load_dataset, load_metric\n",
    "\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "from transformers import TrainingArguments, Trainer\n",
    "\n",
    "from sentence_transformers import CrossEncoder\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b01162e",
   "metadata": {},
   "source": [
    "## Load the data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "475566c0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-0b9c419e93425e40\n",
      "Reusing dataset csv (/home/sebastien/.cache/huggingface/datasets/csv/default-0b9c419e93425e40/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3ee4bb06fea497ea7d4f4f57146682d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = load_dataset(\"csv\", data_files=\"data/train_set.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb30622b",
   "metadata": {},
   "source": [
    "## Initialize the model\n",
    "\n",
    "These are huggingface models that have been tested with our data set:\n",
    "\n",
    "- distilbert-base-uncased-finetuned-sst-2-english\n",
    "- microsoft/xtremedistil-l6-h384-uncased \n",
    "- roberta-base-openai-detector \n",
    "- textattack/distilbert-base-cased-SST-2 \n",
    "- textattack/distilbert-base-uncased-SST-2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a81ea147",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at /home/sebastien/.cache/huggingface/datasets/csv/default-0b9c419e93425e40/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519/cache-938d8004706f71b3.arrow\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\n",
    "    # \"distilbert-base-uncased-finetuned-sst-2-english\",\n",
    "    # \"roberta-base-openai-detector\",\n",
    "    # \"baykenney/bert-base-gpt2detector-random\",\n",
    "    # \"textattack/distilbert-base-uncased-SST-2\",\n",
    "    \"huwendeng/distilroberta_b\",\n",
    "    cache_dir=\"data/transformers_cache\"\n",
    ")\n",
    "\n",
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples[\"summary\"], padding=\"max_length\", truncation=True)\n",
    "\n",
    "tokenized_df = df.map(tokenize_function, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4b9d9c1b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached shuffled indices for dataset at /home/sebastien/.cache/huggingface/datasets/csv/default-0b9c419e93425e40/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519/cache-5ffb12dee33fed2a.arrow\n",
      "Loading cached shuffled indices for dataset at /home/sebastien/.cache/huggingface/datasets/csv/default-0b9c419e93425e40/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519/cache-5ffb12dee33fed2a.arrow\n"
     ]
    }
   ],
   "source": [
    "tokenized_df_train = tokenized_df[\"train\"].shuffle(seed=42).select(range(7000))\n",
    "tokenized_df_eval = tokenized_df[\"train\"].shuffle(seed=42).select(range(7001, 8000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dc11de5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = AutoModelForSequenceClassification.from_pretrained(\n",
    "    # \"distilbert-base-uncased-finetuned-sst-2-english\",\n",
    "    # \"roberta-base-openai-detector\",\n",
    "    # \"baykenney/bert-base-gpt2detector-random\",\n",
    "    # \"textattack/distilbert-base-uncased-SST-2\",\n",
    "    \"huwendeng/distilroberta_b\",\n",
    "    cache_dir=\"data/transformers_cache\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "093ad89a",
   "metadata": {},
   "outputs": [],
   "source": [
    "metric = load_metric(\"accuracy\")\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    return metric.compute(predictions=predictions, references=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "956aac04",
   "metadata": {},
   "source": [
    "## Train the model\n",
    "\n",
    "Do not hesitate to tweak the training parameters to achieve better scores!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b59dfa71",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir=\"data/test_trainer\", \n",
    "    num_train_epochs=5,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    weight_decay=0.01,             \n",
    "    load_best_model_at_end=True,\n",
    "    logging_steps=-1,\n",
    "    eval_steps=500\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1476df44",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_df_train,\n",
    "    eval_dataset=tokenized_df_eval,\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "edc00fc3",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: summary, id, document. If summary, id, document are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "/home/sebastien/.pyenv/versions/3.7.12/envs/itmnlp/lib/python3.7/site-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  FutureWarning,\n",
      "***** Running training *****\n",
      "  Num examples = 7000\n",
      "  Num Epochs = 5\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 4375\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='4375' max='4375' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [4375/4375 29:21, Epoch 5/5]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>1.711000</td>\n",
       "      <td>0.804055</td>\n",
       "      <td>0.845846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.054900</td>\n",
       "      <td>0.366867</td>\n",
       "      <td>0.873874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.060400</td>\n",
       "      <td>0.277780</td>\n",
       "      <td>0.879880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.584100</td>\n",
       "      <td>0.353295</td>\n",
       "      <td>0.886887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>0.002600</td>\n",
       "      <td>0.447940</td>\n",
       "      <td>0.897898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.001400</td>\n",
       "      <td>0.660449</td>\n",
       "      <td>0.882883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3500</td>\n",
       "      <td>0.236800</td>\n",
       "      <td>0.685001</td>\n",
       "      <td>0.889890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4000</td>\n",
       "      <td>0.001300</td>\n",
       "      <td>0.836651</td>\n",
       "      <td>0.885886</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: summary, id, document. If summary, id, document are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 999\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to data/test_trainer/checkpoint-500\n",
      "Configuration saved in data/test_trainer/checkpoint-500/config.json\n",
      "Model weights saved in data/test_trainer/checkpoint-500/pytorch_model.bin\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: summary, id, document. If summary, id, document are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 999\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to data/test_trainer/checkpoint-1000\n",
      "Configuration saved in data/test_trainer/checkpoint-1000/config.json\n",
      "Model weights saved in data/test_trainer/checkpoint-1000/pytorch_model.bin\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: summary, id, document. If summary, id, document are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 999\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to data/test_trainer/checkpoint-1500\n",
      "Configuration saved in data/test_trainer/checkpoint-1500/config.json\n",
      "Model weights saved in data/test_trainer/checkpoint-1500/pytorch_model.bin\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: summary, id, document. If summary, id, document are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 999\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to data/test_trainer/checkpoint-2000\n",
      "Configuration saved in data/test_trainer/checkpoint-2000/config.json\n",
      "Model weights saved in data/test_trainer/checkpoint-2000/pytorch_model.bin\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: summary, id, document. If summary, id, document are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 999\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to data/test_trainer/checkpoint-2500\n",
      "Configuration saved in data/test_trainer/checkpoint-2500/config.json\n",
      "Model weights saved in data/test_trainer/checkpoint-2500/pytorch_model.bin\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: summary, id, document. If summary, id, document are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 999\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to data/test_trainer/checkpoint-3000\n",
      "Configuration saved in data/test_trainer/checkpoint-3000/config.json\n",
      "Model weights saved in data/test_trainer/checkpoint-3000/pytorch_model.bin\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: summary, id, document. If summary, id, document are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 999\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to data/test_trainer/checkpoint-3500\n",
      "Configuration saved in data/test_trainer/checkpoint-3500/config.json\n",
      "Model weights saved in data/test_trainer/checkpoint-3500/pytorch_model.bin\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: summary, id, document. If summary, id, document are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 999\n",
      "  Batch size = 8\n",
      "Saving model checkpoint to data/test_trainer/checkpoint-4000\n",
      "Configuration saved in data/test_trainer/checkpoint-4000/config.json\n",
      "Model weights saved in data/test_trainer/checkpoint-4000/pytorch_model.bin\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from data/test_trainer/checkpoint-1500 (score: 0.2777804732322693).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=4375, training_loss=0.33573853277868143, metrics={'train_runtime': 1761.8925, 'train_samples_per_second': 19.865, 'train_steps_per_second': 2.483, 'total_flos': 4636358952960000.0, 'train_loss': 0.33573853277868143, 'epoch': 5.0})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "itmnlp",
   "language": "python",
   "name": "itmnlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}