Wassa_2023_Shared_Task_EMO_Training.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zuEuDZu5uGB7"
      },
      "source": [
        "# Approach 1: Transformer with OHE (Used in the final model)\n",
        "Make sure you set the runtime type to GPU."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i19EY9rSRR7W"
      },
      "source": [
        "## Importing the Libraries and data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "pkzSkXWJuBuR"
      },
      "outputs": [],
      "source": [
        "#import libraries\n",
        "!pip install transformers\n",
        "!pip install tensorflow-addons\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import torch\n",
        "import tensorflow as tf\n",
        "import tensorflow_addons as tfa\n",
        "import matplotlib.pyplot as plt\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "from tensorflow.keras.optimizers.schedules import PolynomialDecay\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.utils.class_weight import compute_class_weight, compute_sample_weight\n",
        "from transformers import AutoTokenizer, TFAutoModelForSequenceClassification, BartForConditionalGeneration, BartTokenizer, pipeline\n",
        "from huggingface_hub import notebook_login\n",
        "from tqdm.notebook import tqdm\n",
        "tqdm.pandas()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "2mpBz3NMhwjZ"
      },
      "outputs": [],
      "source": [
        "#Parameters for easy toggling\n",
        "use_paraphrasing = False\n",
        "paraphrase_selective = False\n",
        "use_class_weights = True\n",
        "model_name='roberta-large'\n",
        "tokenizer_name = 'roberta-large'\n",
        "epochs=40\n",
        "spell_check = False\n",
        "early_stopping = True\n",
        "patience = 15\n",
        "best_weights = True\n",
        "schedule_lr = False\n",
        "use_swa = True\n",
        "test_mode = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9hCq_PXqH5OC"
      },
      "outputs": [],
      "source": [
        "# Load data\n",
        "\n",
        "#training data\n",
        "data = pd.read_csv(\"WASSA23_essay_level_with_labels_train.tsv\", sep=\"\\t\")\n",
        "data = data[['emotion', 'essay']]\n",
        "\n",
        "if test_mode:\n",
        "  dev_data = pd.read_csv(\"WASSA23_essay_level_test.tsv\", sep=\"\\t\")\n",
        "else:\n",
        "  #development data\n",
        "  dev_data = pd.read_csv(\"WASSA23_essay_level_dev.tsv\", sep=\"\\t\")\n",
        "\n",
        "  #gold standard dev labels\n",
        "  df = pd.read_csv('/content/drive/MyDrive/MSML641 project/goldstandard_dev.tsv', sep=\"\\t\", names=['1', '2', 'emotion',1,2,3,4,5,6,7,8,9])\n",
        "  dev_data['emotion'] = df['emotion']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_LhdLFNsuW-o"
      },
      "outputs": [],
      "source": [
        "#create a list of unique emotions\n",
        "#unique_emotions = list(set(data[\"emotion\"].str.split(\"/\").explode().unique())) #uncomment to create a new sequence\n",
        "\n",
        "#fixed sequence to account for saved model\n",
        "unique_emotions = ['Disgust', 'Hope', 'Fear', 'Anger', 'Sadness', 'Joy', 'Surprise', 'Neutral']\n",
        "\n",
        "unique_emotions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "t5jknit__N8x"
      },
      "outputs": [],
      "source": [
        "occurences = data[\"emotion\"].str.split(\"/\").explode().value_counts()\n",
        "occurences"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0wRJElatTN3J"
      },
      "source": [
        "## Spell Check"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FO4mLXB8LtnH"
      },
      "outputs": [],
      "source": [
        "#fix spelling using huggingface spell correction model\n",
        "def fix_spelling_iterate(df):\n",
        "  '''\n",
        "    This function is applied on a dataframe. It corrects the spelling of essay column\n",
        "  '''\n",
        "  #take in essay\n",
        "  essay = df['essay']\n",
        "\n",
        "  #get length of essay\n",
        "  length_essay = len(essay)\n",
        "\n",
        "  #fix spelling by calling the spell check function\n",
        "  df['essay'] = fix_spelling(essay,max_length=length_essay)\n",
        "\n",
        "  #return the row\n",
        "  return df\n",
        "\n",
        "#if spell check flag is set\n",
        "if spell_check:\n",
        "\n",
        "  #instantiate huggingface pipeline\n",
        "  fix_spelling = pipeline(\"text2text-generation\",model=\"oliverguhr/spelling-correction-english-base\")\n",
        "\n",
        "  #apply the function on training and development data\n",
        "  data = data.progress_apply(fix_spelling_iterate, axis=1)\n",
        "  dev_data = dev_data.progress_apply(fix_spelling_iterate, axis=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "23A4T2kfTUbd"
      },
      "source": [
        "##Paraphrasing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KsrNceuJ4hPi"
      },
      "outputs": [],
      "source": [
        "#check count of each label combination\n",
        "frequency_list = data.groupby(['emotion']).size().to_dict()\n",
        "frequency_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iWKoO1hx_aqO"
      },
      "outputs": [],
      "source": [
        "#if paraphrasing flag is set\n",
        "if use_paraphrasing:\n",
        "  if paraphrase_selective:\n",
        "    #create a dictionary from the training data\n",
        "    paraphrase_threshold = 40\n",
        "    data_dict = data.to_dict(orient=\"records\")\n",
        "  else:\n",
        "    data_dict = []\n",
        "    paraphrase_threshold = 400000\n",
        "\n",
        "  #instantiate paraphrasing model and shift it to gpu\n",
        "  model = BartForConditionalGeneration.from_pretrained('eugenesiow/bart-paraphrase')\n",
        "  device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "  model = model.to(device)\n",
        "\n",
        "  #instantiate the tokenizer\n",
        "  tokenizer = BartTokenizer.from_pretrained('eugenesiow/bart-paraphrase')\n",
        "\n",
        "  #define the paraphrasing function\n",
        "  def paraphrase(df):\n",
        "    '''\n",
        "      This function paraphrases the essays whose label frequency is less than 40\n",
        "    '''\n",
        "\n",
        "    #get the essay and the label\n",
        "    input_sentence = df['essay']\n",
        "    emotion = df['emotion']\n",
        "\n",
        "    #check if frequency is less than 40\n",
        "    if frequency_list[emotion] < paraphrase_threshold:\n",
        "\n",
        "      #tokenize essay and paraphrase\n",
        "      batch = tokenizer(input_sentence, return_tensors='pt').to(device)\n",
        "      generated_ids = model.generate(batch['input_ids'])\n",
        "      generated_sentence = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)\n",
        "\n",
        "      #append to training data dictionary\n",
        "      data_dict.append({'essay': generated_sentence[0], 'emotion':emotion})\n",
        "\n",
        "  #apply above function to training data\n",
        "  data.progress_apply(paraphrase, axis=1)\n",
        "\n",
        "  #get new training data by converting the dictionary to dataframe\n",
        "  data = pd.DataFrame(data_dict)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eQwCb7iBTY8P"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WyCPAto8ubZ5"
      },
      "outputs": [],
      "source": [
        "#One Hot Encode the labels\n",
        "num_classes = len(unique_emotions)\n",
        "for emotion in unique_emotions:\n",
        "    data[emotion] = data[\"emotion\"].str.contains(emotion).astype(int)\n",
        "    dev_data[emotion] = dev_data[\"emotion\"].str.contains(emotion).astype(int)\n",
        "\n",
        "#drop string emotion column\n",
        "data = data.drop(columns=[\"emotion\"])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tcT7VCff7ZH8"
      },
      "outputs": [],
      "source": [
        "#get training and testing texts and labels\n",
        "train_texts= data['essay'] \n",
        "train_labels = data[unique_emotions]\n",
        "test_texts = dev_data['essay']\n",
        "test_labels= dev_data[unique_emotions]\n",
        "\n",
        "# Load tokenizer and pre-trained model\n",
        "tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)\n",
        "model = TFAutoModelForSequenceClassification.from_pretrained(model_name, num_labels=num_classes)\n",
        "\n",
        "# Compute class weights for training data\n",
        "class_weights = compute_sample_weight(class_weight='balanced', y=train_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M1MfC9kyui53"
      },
      "outputs": [],
      "source": [
        "# Tokenize train and test texts\n",
        "train_encodings = tokenizer(train_texts.to_list(), truncation=True, padding=True)\n",
        "test_encodings = tokenizer(test_texts.to_list(), truncation=True, padding=True)\n",
        "\n",
        "# Create TensorFlow datasets, add class weights to training dataset if flag is set\n",
        "if use_class_weights:\n",
        "  train_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "      dict(train_encodings),\n",
        "      train_labels.astype(np.float32),\n",
        "      class_weights\n",
        "  ))\n",
        "else:\n",
        "  train_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "      dict(train_encodings),\n",
        "      train_labels.astype(np.float32)\n",
        "  ))\n",
        "\n",
        "test_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "    dict(test_encodings),\n",
        "    test_labels.astype(np.float32)\n",
        "))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l6kCBJEqumLP"
      },
      "outputs": [],
      "source": [
        "#lr scheduler\n",
        "num_steps = len(train_dataset) * epochs // 16\n",
        "lr_scheduler = PolynomialDecay(\n",
        "    initial_learning_rate = 3e-05,\n",
        "    end_learning_rate = 0.0,\n",
        "    decay_steps = num_steps\n",
        ")\n",
        "\n",
        "\n",
        "# Define training parameters\n",
        "if schedule_lr:\n",
        "  optimizer = tf.keras.optimizers.legacy.Adam(learning_rate=lr_scheduler, decay=0.008) #, weight_decay=0.008\n",
        "else:\n",
        "  optimizer = tf.keras.optimizers.legacy.Adam(learning_rate=3e-05,decay=0.008)\n",
        "\n",
        "if use_swa:\n",
        "  optimizer = tfa.optimizers.SWA(optimizer)\n",
        "loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n",
        "metric = tf.keras.metrics.BinaryAccuracy('accuracy')\n",
        "\n",
        "# Compile model\n",
        "model.compile(optimizer=optimizer, loss=loss, metrics=[metric])\n",
        "\n",
        "#early stopping, monitoring validation loss, if flag is set\n",
        "if early_stopping:\n",
        "  early_stopping_callback = EarlyStopping(\n",
        "      monitor=\"val_loss\",\n",
        "      patience=patience,\n",
        "      restore_best_weights=best_weights\n",
        "  )\n",
        "\n",
        "  #fit the model\n",
        "  history = model.fit(train_dataset.shuffle(len(train_labels)).batch(16), epochs=epochs, batch_size=16, validation_data=test_dataset.shuffle(len(test_labels)).batch(16), callbacks=[early_stopping_callback])\n",
        "else:\n",
        "  #fit the model without early stopping\n",
        "  history = model.fit(train_dataset.shuffle(len(train_labels)).batch(16), epochs=epochs, batch_size=16, validation_data=test_dataset.shuffle(len(test_labels)).batch(16))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G29imzWxxiw1"
      },
      "outputs": [],
      "source": [
        "#plot loss \n",
        "plt.plot(history.history['loss'], label='train loss')\n",
        "plt.plot(history.history['val_loss'], label='val loss')\n",
        "\n",
        "#plot a line at the best loss\n",
        "plt.axhline(np.min(history.history['val_loss']), linestyle='--', color='r', label='best val loss')\n",
        "plt.legend()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kasACfEMyL4m"
      },
      "outputs": [],
      "source": [
        "#plot accuracy\n",
        "plt.plot(history.history['accuracy'], label='train accuracy')\n",
        "plt.plot(history.history['val_accuracy'], label='val accuracy')\n",
        "plt.legend()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JHug3H4IyMQj"
      },
      "outputs": [],
      "source": [
        "#plot learning rate\n",
        "plt.plot(lr_scheduler(tf.range(0, num_steps)))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hRkMRr7iUK7H"
      },
      "source": [
        "##Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Sr7rhxCj0rt4"
      },
      "outputs": [],
      "source": [
        "#finding the best threshold\n",
        "def predict_on_test_dataset(test_dataset, thresholds):\n",
        "  '''\n",
        "    This function takes in the test dataset (here, dev data) and calculates F1 and accuracy. \n",
        "    Test different thresholds for logits\n",
        "  '''\n",
        "\n",
        "  #iterate over thresholds\n",
        "  for threshold in thresholds:\n",
        "\n",
        "    #predict using trained model\n",
        "    y_pred = model.predict(test_dataset.batch(16))\n",
        "\n",
        "    #based on threshold, convert logits to 1\n",
        "    y_pred = tf.where(np.array(list(y_pred.logits)) > threshold, 1,0)\n",
        "    y_pred = y_pred.numpy()\n",
        "    y_true = test_labels.to_numpy()\n",
        "\n",
        "    #calculate accuracy\n",
        "    accuracy = np.sum(y_pred == y_true) / y_true.size\n",
        "\n",
        "    #print accuracy and F1 macro\n",
        "    print(threshold, f1_score(y_true, y_pred, average=\"macro\",zero_division=1), accuracy)\n",
        "\n",
        "#instantiate a list of thresholds\n",
        "#Try different values\n",
        "thresholds = [-0.3,-0.25,-0.2,-0.15, -0.1, -0.075]\n",
        "\n",
        "#call the function\n",
        "predict_on_test_dataset(test_dataset, thresholds)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gKtTCceu0Cha"
      },
      "outputs": [],
      "source": [
        "#Get the string version of predictions\n",
        "def get_predicted_emotions(data, threshold=-0.075):\n",
        "    '''\n",
        "      This function takes in the data and finds the emotion of the essay using the trained model\n",
        "    '''\n",
        "\n",
        "    #get the essay column\n",
        "    essays = data['essay']\n",
        "\n",
        "    #get the encodings\n",
        "    encodings = tokenizer(essays.tolist(), truncation=True, padding=True)\n",
        "\n",
        "    #create tensorflow dataset\n",
        "    dataset = tf.data.Dataset.from_tensor_slices(dict(encodings))\n",
        "\n",
        "    #get predictions\n",
        "    y_pred = model.predict(dataset.batch(16))\n",
        "    \n",
        "    #create matrix based on threshold\n",
        "    matrix = tf.where(y_pred.logits>threshold, 1, 0)\n",
        "    y_pred = []\n",
        "\n",
        "    #iterate over rows and find corresponding emotions\n",
        "    for row in matrix:\n",
        "      emotions = []\n",
        "      for i in range(len(row)):\n",
        "        if row[i] == 1:\n",
        "          emotions.append(unique_emotions[i])\n",
        "          emotions.sort()\n",
        "      y_pred.append(\"/\".join(emotions))\n",
        "\n",
        "    #create a dataframe with the essays and the predicted emotions\n",
        "    df = pd.DataFrame({'essay': essays, 'emotion': y_pred})\n",
        "\n",
        "    #return the dataframe\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PtHpF5DG_jq-"
      },
      "outputs": [],
      "source": [
        "#if paraphrasing flag is set\n",
        "if use_paraphrasing:\n",
        "  if paraphrase_selective:\n",
        "    #create a dictionary from the training data\n",
        "    paraphrase_threshold = 40\n",
        "    data_dict = data.to_dict(orient=\"records\")\n",
        "  else:\n",
        "    data_dict = []\n",
        "    paraphrase_threshold = 400000\n",
        "\n",
        "  #instantiate paraphrasing model and shift it to gpu\n",
        "  model_para = BartForConditionalGeneration.from_pretrained('eugenesiow/bart-paraphrase')\n",
        "  device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "  model_para = model_para.to(device)\n",
        "\n",
        "  #instantiate the tokenizer\n",
        "  tokenizer_para = BartTokenizer.from_pretrained('eugenesiow/bart-paraphrase')\n",
        "\n",
        "  #define the paraphrasing function\n",
        "  def paraphrase(df):\n",
        "    '''\n",
        "      This function paraphrases the essays whose label frequency is less than 40\n",
        "    '''\n",
        "\n",
        "    #get the essay and the label\n",
        "    input_sentence = df['essay']\n",
        "    emotion = df['emotion']\n",
        "\n",
        "    #check if frequency is less than 40\n",
        "\n",
        "    #tokenize essay and paraphrase\n",
        "    batch = tokenizer_para(input_sentence, return_tensors='pt').to(device)\n",
        "    generated_ids = model_para.generate(batch['input_ids'])\n",
        "    generated_sentence = tokenizer_para.batch_decode(generated_ids, skip_special_tokens=True)\n",
        "\n",
        "    #append to training data dictionary\n",
        "    data_dict.append({'essay': generated_sentence[0], 'emotion':emotion})\n",
        "\n",
        "  #apply above function to training data\n",
        "  dev_data.progress_apply(paraphrase, axis=1)\n",
        "\n",
        "  #get new training data by converting the dictionary to dataframe\n",
        "  dev_data = pd.DataFrame(data_dict)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u70Loq9Y12IQ"
      },
      "outputs": [],
      "source": [
        "#get predictions on the dev data\n",
        "dev_predictions = get_predicted_emotions(dev_data)\n",
        "dev_predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4hvoJ6qX4MM3"
      },
      "outputs": [],
      "source": [
        "#if no emotion is assigned, assign neutral to that row\n",
        "dev_predictions['emotion'] = dev_predictions['emotion'].replace('', 'Neutral')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YJsd-HkougsH"
      },
      "outputs": [],
      "source": [
        "#save to csv\n",
        "dev_predictions['emotion'].to_csv('predictions_EMO.tsv', sep=\"\\t\", header=False, index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-kiQ7c5pVSkl"
      },
      "source": [
        "## Saving the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ul8pWu0l2tAo"
      },
      "outputs": [],
      "source": [
        "#log in to huggingface (You'll need a token)\n",
        "notebook_login()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gLcpXCxCaQGH"
      },
      "outputs": [],
      "source": [
        "#save the model\n",
        "model.save_pretrained(\"adityapatkar/WASSA_EMO\", push_to_hub=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mnQmlbZOVpnI"
      },
      "outputs": [],
      "source": [
        "#save the tokenizer\n",
        "tokenizer.push_to_hub(\"adityapatkar/WASSA_EMO\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D3flp45WpHq4"
      },
      "source": [
        "## Prediction using huggingface hub model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Cq2wrP5JpHR4"
      },
      "outputs": [],
      "source": [
        "#get the model and tokenizer from huggingface hub\n",
        "model_name = 'adityapatkar/WASSA_EMO'\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name) #add arg: use_auth_token=\"Add_Your_Auth_Token\"\n",
        "model = TFAutoModelForSequenceClassification.from_pretrained(model_name, num_labels=8) #add arg: use_auth_token=\"Add_Your_Auth_Token\"\n",
        "\n",
        "#get the test set\n",
        "test_set = pd.read_csv(\"WASSA23_essay_level_test.tsv\", sep=\"\\t\")\n",
        "\n",
        "unique_emotions = ['Disgust', 'Hope', 'Fear', 'Anger', 'Sadness', 'Joy', 'Surprise', 'Neutral']\n",
        "\n",
        "unique_emotions\n",
        "\n",
        "#find predictions on the test set\n",
        "test_predictions = get_predicted_emotions(test_set)\n",
        "\n",
        "#see the predictions\n",
        "test_predictions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oIecK7-nVtTw"
      },
      "outputs": [],
      "source": [
        "#if no emotion is assigned, assign neutral to that row\n",
        "test_predictions['emotion'] = test_predictions['emotion'].replace('', 'Neutral')\n",
        "\n",
        "#save to csv\n",
        "test_predictions['emotion'].to_csv('predictions_EMO.tsv', sep=\"\\t\", header=False, index=False)\n",
        "test_predictions.to_csv('predictions_EMO_Essay.tsv', sep=\"\\t\", header=False, index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0qTGEe_h_IIW"
      },
      "source": [
        "## Predicting if multiple emotions exist"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "45KL3PJW_HHQ"
      },
      "outputs": [],
      "source": [
        "#classifier for predicting if multiple emotions are present in an essay\n",
        "\n",
        "data = pd.read_csv(\"/content/WASSA23_essay_level_with_labels_train.tsv\", sep=\"\\t\")\n",
        "data['multiple_emotions'] = data['emotion'].apply(lambda x: 1 if len(x.split(\"/\")) > 1 else 0)\n",
        "dev_data['multiple_emotions'] = dev_data['emotion'].apply(lambda x: 1 if len(x.split(\"/\")) > 1 else 0)\n",
        "\n",
        "train_texts = data['essay']\n",
        "train_labels = data['multiple_emotions']\n",
        "test_texts = dev_data['essay']\n",
        "test_labels = dev_data['multiple_emotions']\n",
        "\n",
        "# Load tokenizer and pre-trained model\n",
        "tokenizer = AutoTokenizer.from_pretrained('roberta-base')\n",
        "model = TFAutoModelForSequenceClassification.from_pretrained('roberta-base', num_labels=2)\n",
        "\n",
        "# Tokenize train and test texts\n",
        "train_encodings = tokenizer(train_texts.tolist(), truncation=True, padding=True)\n",
        "test_encodings = tokenizer(test_texts.tolist(), truncation=True, padding=True)\n",
        "\n",
        "# Create TensorFlow datasets\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "    dict(train_encodings),\n",
        "    train_labels.astype(np.float32)\n",
        "\n",
        "))  \n",
        "test_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "    dict(test_encodings),\n",
        "    test_labels.astype(np.float32)\n",
        "))\n",
        "\n",
        "optimizer = tf.keras.optimizers.AdamW(learning_rate=2e-5)\n",
        "loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)\n",
        "metric = tf.keras.metrics.BinaryAccuracy('accuracy')\n",
        "\n",
        "# Compile model\n",
        "model.compile(optimizer=optimizer, loss=loss, metrics=[metric])\n",
        "\n",
        "# Train model\n",
        "model.fit(train_dataset.shuffle(len(train_labels)).batch(16), epochs=5, batch_size=16, validation_data=test_dataset.shuffle(len(test_labels)).batch(16))\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Hg0VGg6WAtvk"
      },
      "outputs": [],
      "source": [
        "def multiple_emotions_present(essay):\n",
        "    '''\n",
        "    Takes in an essay and returns 0 if no multiple emotions are present and 1 if multiple emotions are present\n",
        "    '''\n",
        "    encodings = tokenizer(essay, truncation=True, padding=True)\n",
        "    \n",
        "    #create 1 row dataset\n",
        "    dataset = tf.data.Dataset.from_tensor_slices(dict(encodings))\n",
        "\n",
        "    #predict: send as batch of 1\n",
        "    y_pred = model.predict(dataset.batch(1))\n",
        "    y_pred = tf.nn.softmax(y_pred.logits)\n",
        "    #get argmax of each row\n",
        "    y_pred = tf.argmax(y_pred, axis=1)\n",
        "    return y_pred[0].numpy()  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HmhxxG6gAwhZ"
      },
      "outputs": [],
      "source": [
        "dev_data['essay'][203]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zE9IMrDRA3Hh"
      },
      "outputs": [],
      "source": [
        "essay = \"After reading the article, you can't help but feel bad for the people that were involved in the train crash. IT was a freak accident and something you can't control. It 's just sad because they had to leave all of their family behind and their lives will most likely never be the same. While reading it, i felt really worried because it could happen to anyone.\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UWA1QE5wA6uO"
      },
      "outputs": [],
      "source": [
        "multiple_emotions_present(essay)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7FGM64aWjlqt"
      },
      "source": [
        "# Training on GoEmotions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NTSiwvyvjov2"
      },
      "outputs": [],
      "source": [
        "#import libraries\n",
        "!pip install transformers\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import torch\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "from tensorflow.keras.optimizers.schedules import PolynomialDecay\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.utils.class_weight import compute_class_weight, compute_sample_weight\n",
        "from transformers import AutoTokenizer, TFAutoModelForSequenceClassification, BartForConditionalGeneration, BartTokenizer, pipeline\n",
        "from huggingface_hub import notebook_login\n",
        "from tqdm.notebook import tqdm\n",
        "tqdm.pandas()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R1PAgO48jtdr"
      },
      "outputs": [],
      "source": [
        "#Parameters for easy toggling\n",
        "use_paraphrasing = False\n",
        "paraphrase_selective = False\n",
        "use_class_weights = True\n",
        "model_name='roberta-large'\n",
        "tokenizer_name = 'roberta-large'\n",
        "epochs=15\n",
        "spell_check = False\n",
        "early_stopping = True\n",
        "patience = 10\n",
        "best_weights = True\n",
        "schedule_lr = False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qEdtGka8jxgy"
      },
      "outputs": [],
      "source": [
        "# Load data\n",
        "\n",
        "#training data\n",
        "data = pd.read_csv(\"goemotions.csv\").iloc[0:5000]\n",
        "\n",
        "#development data\n",
        "dev_data = pd.read_csv(\"WASSA23_essay_level_dev.tsv\", sep=\"\\t\")\n",
        "\n",
        "#gold standard dev labels\n",
        "df = pd.read_csv('/content/goldstandard_dev.tsv', sep=\"\\t\", names=['1', '2', 'emotion',1,2,3,4,5,6,7,8,9])\n",
        "dev_data['emotion'] = df['emotion']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "K-uBwm5Wj2R_"
      },
      "outputs": [],
      "source": [
        "#create a list of unique emotions\n",
        "#unique_emotions = list(set(data[\"emotion\"].str.split(\"/\").explode().unique())) #uncomment to create a new sequence\n",
        "\n",
        "#fixed sequence to account for saved model\n",
        "unique_emotions = ['Disgust', 'Hope', 'Fear', 'Anger', 'Sadness', 'Joy', 'Surprise', 'Neutral']\n",
        "\n",
        "unique_emotions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bgu3h_cIj6R-"
      },
      "outputs": [],
      "source": [
        "#One Hot Encode the labels\n",
        "num_classes = len(unique_emotions)\n",
        "for emotion in unique_emotions:\n",
        "    dev_data[emotion] = dev_data[\"emotion\"].str.contains(emotion).astype(int)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z2BpcUTNj_kQ"
      },
      "outputs": [],
      "source": [
        "#get training and testing texts and labels\n",
        "train_texts= data['essay'] \n",
        "train_labels = data[unique_emotions]\n",
        "test_texts = dev_data['essay']\n",
        "test_labels= dev_data[unique_emotions]\n",
        "\n",
        "# Load tokenizer and pre-trained model\n",
        "tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)\n",
        "model = TFAutoModelForSequenceClassification.from_pretrained(model_name, num_labels=num_classes)\n",
        "\n",
        "# Compute class weights for training data\n",
        "class_weights = compute_sample_weight(class_weight='balanced', y=train_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WA_yoVbgkD0a"
      },
      "outputs": [],
      "source": [
        "# Tokenize train and test texts\n",
        "train_encodings = tokenizer(train_texts.to_list(), truncation=True, padding=True)\n",
        "test_encodings = tokenizer(test_texts.to_list(), truncation=True, padding=True)\n",
        "\n",
        "# Create TensorFlow datasets, add class weights to training dataset if flag is set\n",
        "if use_class_weights:\n",
        "  train_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "      dict(train_encodings),\n",
        "      train_labels.astype(np.float32),\n",
        "      class_weights\n",
        "  ))\n",
        "else:\n",
        "  train_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "      dict(train_encodings),\n",
        "      train_labels.astype(np.float32)\n",
        "  ))\n",
        "\n",
        "test_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "    dict(test_encodings),\n",
        "    test_labels.astype(np.float32)\n",
        "))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0s4nNgw6kG-u"
      },
      "outputs": [],
      "source": [
        "#lr scheduler\n",
        "num_steps = len(train_dataset) * epochs // 16\n",
        "lr_scheduler = PolynomialDecay(\n",
        "    initial_learning_rate = 3e-05,\n",
        "    end_learning_rate = 0.0,\n",
        "    decay_steps = num_steps\n",
        ")\n",
        "\n",
        "\n",
        "# Define training parameters\n",
        "if schedule_lr:\n",
        "  optimizer = tf.keras.optimizers.AdamW(learning_rate=lr_scheduler) #, weight_decay=0.008\n",
        "else:\n",
        "  optimizer = tf.keras.optimizers.AdamW(learning_rate=2e-05)\n",
        "loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n",
        "metric = tf.keras.metrics.BinaryAccuracy('accuracy')\n",
        "\n",
        "# Compile model\n",
        "model.compile(optimizer=optimizer, loss=loss, metrics=[metric])\n",
        "\n",
        "#early stopping, monitoring validation loss, if flag is set\n",
        "if early_stopping:\n",
        "  early_stopping_callback = EarlyStopping(\n",
        "      monitor=\"val_loss\",\n",
        "      patience=patience,\n",
        "      restore_best_weights=best_weights\n",
        "  )\n",
        "\n",
        "  #fit the model\n",
        "  history = model.fit(train_dataset.shuffle(len(train_labels)).batch(16), epochs=epochs, batch_size=16, validation_data=test_dataset.shuffle(len(test_labels)).batch(16), callbacks=[early_stopping_callback])\n",
        "else:\n",
        "  #fit the model without early stopping\n",
        "  history = model.fit(train_dataset.shuffle(len(train_labels)).batch(16), epochs=epochs, batch_size=16, validation_data=test_dataset.shuffle(len(test_labels)).batch(16))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-ial6xzHo-we"
      },
      "outputs": [],
      "source": [
        "#Parameters for easy toggling\n",
        "use_paraphrasing = False\n",
        "paraphrase_selective = False\n",
        "use_class_weights = True\n",
        "model_name='roberta-large'\n",
        "tokenizer_name = 'roberta-large'\n",
        "epochs=10\n",
        "spell_check = False\n",
        "early_stopping = True\n",
        "patience = 7\n",
        "best_weights = True\n",
        "schedule_lr = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NzvQXcYuoQ5X"
      },
      "outputs": [],
      "source": [
        "# Load data\n",
        "\n",
        "#training data\n",
        "data = pd.read_csv(\"/content/WASSA23_essay_level_with_labels_train.tsv\", sep=\"\\t\")\n",
        "data = data[['emotion', 'essay']]\n",
        "\n",
        "#One Hot Encode the labels\n",
        "num_classes = len(unique_emotions)\n",
        "for emotion in unique_emotions:\n",
        "    data[emotion] = data[\"emotion\"].str.contains(emotion).astype(int)\n",
        "\n",
        "#drop string emotion column\n",
        "data = data.drop(columns=[\"emotion\"])\n",
        "\n",
        "#get training and testing texts and labels\n",
        "train_texts= data['essay'] \n",
        "train_labels = data[unique_emotions]\n",
        "\n",
        "# Compute class weights for training data\n",
        "class_weights = compute_sample_weight(class_weight='balanced', y=train_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6EKYfZQ0or4K"
      },
      "outputs": [],
      "source": [
        "# Tokenize train and test texts\n",
        "train_encodings = tokenizer(train_texts.to_list(), truncation=True, padding=True)\n",
        "\n",
        "# Create TensorFlow datasets, add class weights to training dataset if flag is set\n",
        "if use_class_weights:\n",
        "  train_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "      dict(train_encodings),\n",
        "      train_labels.astype(np.float32),\n",
        "      class_weights\n",
        "  ))\n",
        "else:\n",
        "  train_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "      dict(train_encodings),\n",
        "      train_labels.astype(np.float32)\n",
        "  ))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rY4BlteSo0kq"
      },
      "outputs": [],
      "source": [
        "#lr scheduler\n",
        "num_steps = len(train_dataset) * epochs // 16\n",
        "lr_scheduler = PolynomialDecay(\n",
        "    initial_learning_rate = 3e-05,\n",
        "    end_learning_rate = 0.0,\n",
        "    decay_steps = num_steps\n",
        ")\n",
        "\n",
        "\n",
        "# Define training parameters\n",
        "if schedule_lr:\n",
        "  optimizer = tf.keras.optimizers.AdamW(learning_rate=lr_scheduler) #, weight_decay=0.008\n",
        "else:\n",
        "  optimizer = tf.keras.optimizers.AdamW(learning_rate=2e-05)\n",
        "loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n",
        "metric = tf.keras.metrics.BinaryAccuracy('accuracy')\n",
        "\n",
        "#early stopping, monitoring validation loss, if flag is set\n",
        "if early_stopping:\n",
        "  early_stopping_callback = EarlyStopping(\n",
        "      monitor=\"val_loss\",\n",
        "      patience=patience,\n",
        "      restore_best_weights=best_weights\n",
        "  )\n",
        "\n",
        "  #fit the model\n",
        "  history = model.fit(train_dataset.shuffle(len(train_labels)).batch(16), epochs=epochs, batch_size=16, validation_data=test_dataset.shuffle(len(test_labels)).batch(16), callbacks=[early_stopping_callback])\n",
        "else:\n",
        "  #fit the model without early stopping\n",
        "  history = model.fit(train_dataset.shuffle(len(train_labels)).batch(16), epochs=epochs, batch_size=16, validation_data=test_dataset.shuffle(len(test_labels)).batch(16))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S_z1Kjncq7oF"
      },
      "outputs": [],
      "source": [
        "history = model.fit(train_dataset.shuffle(len(train_labels)).batch(16), epochs=epochs, batch_size=16, validation_data=test_dataset.shuffle(len(test_labels)).batch(16), callbacks=[early_stopping_callback])"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "0wRJElatTN3J",
        "23A4T2kfTUbd",
        "hRkMRr7iUK7H",
        "0qTGEe_h_IIW",
        "7FGM64aWjlqt"
      ],
      "toc_visible": true,
      "provenance": []
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}