7. Transformers w Positional Embedding + Emotion.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "HW_7.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/asliakalin/NLP/blob/master/7.%20Transformers%20w%20Positional%20Embedding%20%2B%20Emotion.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cE4-tSuL0G64"
      },
      "source": [
        "import csv\n",
        "import torch\n",
        "import numpy as np\n",
        "from torchtext.data.utils import get_tokenizer\n",
        "import torch.nn as nn\n",
        "from torch.nn import TransformerEncoder, TransformerEncoderLayer, TransformerDecoder, TransformerDecoderLayer\n",
        "import math\n",
        "from tqdm import tqdm\n",
        "import random\n",
        "import time"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xqTbFUId0MSX",
        "outputId": "a6293747-1036-4c1b-c5de-d3965a6657ab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "# if this cell prints \"Running on cpu\", you must switch runtime environments\n",
        "# go to Runtime > Change runtime type > Hardware accelerator > GPU\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(\"Running on {}\".format(device))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running on cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f-CW1Dvi9FYx"
      },
      "source": [
        "# Data processing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W2fVAg4f0QKz",
        "outputId": "6a836df6-158f-44c1-92e4-ed72753a7c26",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 667
        }
      },
      "source": [
        "#This gets the EmpatheticDialogues corpus\n",
        "!wget https://dl.fbaipublicfiles.com/parlai/empatheticdialogues/empatheticdialogues.tar.gz\n",
        "!tar -xvf empatheticdialogues.tar.gz\n",
        "#This gets the GLoVe embeddings, which we will use to bootstrap our model\n",
        "!wget http://nlp.stanford.edu/data/glove.6B.zip\n",
        "!unzip glove*.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-05-03 09:22:49--  https://dl.fbaipublicfiles.com/parlai/empatheticdialogues/empatheticdialogues.tar.gz\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.74.142, 104.22.75.142, 2606:4700:10::6816:4a8e, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.74.142|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 28022709 (27M) [application/gzip]\n",
            "Saving to: ‘empatheticdialogues.tar.gz’\n",
            "\n",
            "empatheticdialogues 100%[===================>]  26.72M  7.49MB/s    in 3.6s    \n",
            "\n",
            "2020-05-03 09:22:53 (7.49 MB/s) - ‘empatheticdialogues.tar.gz’ saved [28022709/28022709]\n",
            "\n",
            "empatheticdialogues/\n",
            "empatheticdialogues/test.csv\n",
            "empatheticdialogues/train.csv\n",
            "empatheticdialogues/valid.csv\n",
            "--2020-05-03 09:22:56--  http://nlp.stanford.edu/data/glove.6B.zip\n",
            "Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:80... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://nlp.stanford.edu/data/glove.6B.zip [following]\n",
            "--2020-05-03 09:22:56--  https://nlp.stanford.edu/data/glove.6B.zip\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip [following]\n",
            "--2020-05-03 09:22:57--  http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip\n",
            "Resolving downloads.cs.stanford.edu (downloads.cs.stanford.edu)... 171.64.64.22\n",
            "Connecting to downloads.cs.stanford.edu (downloads.cs.stanford.edu)|171.64.64.22|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 862182613 (822M) [application/zip]\n",
            "Saving to: ‘glove.6B.zip’\n",
            "\n",
            "glove.6B.zip        100%[===================>] 822.24M  2.14MB/s    in 6m 29s  \n",
            "\n",
            "2020-05-03 09:29:27 (2.11 MB/s) - ‘glove.6B.zip’ saved [862182613/862182613]\n",
            "\n",
            "Archive:  glove.6B.zip\n",
            "  inflating: glove.6B.50d.txt        \n",
            "  inflating: glove.6B.100d.txt       \n",
            "  inflating: glove.6B.200d.txt       \n",
            "  inflating: glove.6B.300d.txt       \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o8shZmBg6Yxl"
      },
      "source": [
        "PAD_INDEX = 0             # reserved for padding words\n",
        "UNKNOWN_INDEX = 1         # reserved for unknown words\n",
        "START_DECODE = 2          # special symbol to denote decoding should start\n",
        "END_DECODE = 3            # special symbol to indicate decoding is ending. This is how the model indicates the sequence is done.\n",
        "\n",
        "def read_embeddings(filename, vocab_size=10000):\n",
        "  \"\"\"\n",
        "  Utility function, loads in the `vocab_size` most common embeddings from `filename`\n",
        "  \n",
        "  Arguments:\n",
        "  - filename:     path to file\n",
        "                  automatically infers correct embedding dimension from filename\n",
        "  - vocab_size:   maximum number of embeddings to load\n",
        "\n",
        "  Returns \n",
        "  - embeddings:   torch.FloatTensor matrix of size (vocab_size x word_embedding_dim)\n",
        "  - vocab:        dictionary mapping word (str) to index (int) in embedding matrix\n",
        "  \"\"\"\n",
        "\n",
        "  # get the embedding size from the first embedding\n",
        "  with open(filename, encoding=\"utf-8\") as file:\n",
        "    word_embedding_dim = len(file.readline().split(\" \")) - 1\n",
        "\n",
        "  vocab = {}\n",
        "  vocab[\"PAD_INDEX\"] = 0\n",
        "  vocab[\"UNKNOWN_INDEX\"] = 1\n",
        "  vocab[\"START_DECODE\"] = 2\n",
        "  vocab[\"END_DECODE\"] = 3\n",
        "\n",
        "  embeddings = np.zeros((vocab_size, word_embedding_dim))\n",
        "\n",
        "  with open(filename, encoding=\"utf-8\") as file:\n",
        "    for idx, line in enumerate(file):\n",
        "      if idx + 4 >= vocab_size:\n",
        "        break\n",
        "\n",
        "      cols = line.rstrip().split(\" \")\n",
        "      val = np.array(cols[1:])\n",
        "      word = cols[0]\n",
        "      embeddings[idx + 4] = val\n",
        "      vocab[word] = idx + 4\n",
        "  \n",
        "  # a FloatTensor is a multidimensional matrix\n",
        "  # that contains 32-bit floats in every entry\n",
        "  # https://pytorch.org/docs/stable/tensors.html\n",
        "  return torch.FloatTensor(embeddings), vocab"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LbNf4hcpnXIp"
      },
      "source": [
        "# Let's load in a spacy tokenizer to process our conversation data\n",
        "tokenizer = get_tokenizer(\"spacy\")\n",
        "\n",
        "class Dataset():\n",
        "  '''\n",
        "  This is a Dataset object, similar to the one used in HW4.\n",
        "\n",
        "  It serves two purposes- reading data and creating batches.\n",
        "\n",
        "  read_data():\n",
        "    Inputs: \n",
        "      filename \n",
        "      emotions_list- list of emotions to include in this dataset\n",
        "    Outputs: \n",
        "      emotions-     list of emotions associated with the dataset\n",
        "      past_turns-   list of past turns associated with the dataset.  \n",
        "                    This is input to our model.\n",
        "      responses-    list of responses associated with the dataset.  This is what \n",
        "                    we will train our model to generate.\n",
        "\n",
        "  get_batches():\n",
        "    Inputs: batch_size- size of batches we want to create \n",
        "            vocab-      our vocabulary, used to replace unknown words\n",
        "            emotset-    set of emotions to id, used to create emotion IDs\n",
        "    Outputs:  \n",
        "      batched_past_turn_idx:      indices of the words in the past turn\n",
        "      batched_past_lengths:       lengths of the past turns (since we are padding these)\n",
        "      batched_response_idx:       indices of words in the response\n",
        "      batched_past_resp_lengths:  lengths of the response (since we pad those)\n",
        "      batched_emotions:           emotions associated with the conversations\n",
        "\n",
        "  '''\n",
        "  def __init__(self, filename, emotions_list=None):\n",
        "    if emotions_list is not None:\n",
        "      self.emotions_list = ['context'] + emotions_list\n",
        "    else:\n",
        "      self.emotions_list = None\n",
        "    self.emotions, self.past_turns, self.responses = self.read_data(filename)\n",
        "\n",
        "  def read_data(self, filename):\n",
        "    past_turns = []\n",
        "    responses = []\n",
        "    emotions = []\n",
        "\n",
        "    raw_data = {}\n",
        "\n",
        "    with open(filename, encoding='utf8') as f:\n",
        "      csvreader = csv.reader(f, delimiter=',')\n",
        "      for row in csvreader:\n",
        "        convo_num = row[0]\n",
        "        emotion = row[2]  \n",
        "        utterance = row[5]\n",
        "        #if we receive an emotions_list, we need to make sure the emotion is relevant to the Dataset we create.\n",
        "        if self.emotions_list is None or emotion in self.emotions_list:\n",
        "          if convo_num not in raw_data:\n",
        "            raw_data[convo_num] = {}\n",
        "            raw_data[convo_num][\"emotion\"] = emotion\n",
        "            raw_data[convo_num][\"convo\"] = []\n",
        "          raw_data[convo_num][\"convo\"].append(utterance.replace(\"_comma_\", \",\"))\n",
        "\n",
        "      for key in raw_data:\n",
        "        for i,turn in enumerate(raw_data[key][\"convo\"]):\n",
        "          #we want to grab every other response\n",
        "          if i % 2 == 0 and i != 0:\n",
        "            emotions.append(raw_data[key][\"emotion\"])\n",
        "            past_turns.append(tokenizer(raw_data[key][\"convo\"][i-1].lower()))\n",
        "            responses.append(tokenizer(raw_data[key][\"convo\"][i].lower()))\n",
        "    \n",
        "    return emotions, past_turns, responses\n",
        "\n",
        "  def get_batches(self, batch_size, vocab, emotset):\n",
        "    # randomly shuffle the data\n",
        "    np.random.seed(159) # don't change this, for reproducibility\n",
        "    shuffle = np.random.permutation(range(len(self.past_turns)))\n",
        "    \n",
        "    #grabs the relevant data from the random permutation\n",
        "    past_turns = [self.past_turns[i] for i in shuffle]\n",
        "    emotions = [self.emotions[i] for i in shuffle]\n",
        "    responses = [self.responses[i] for i in shuffle]\n",
        "\n",
        "    #stores the id's of past_turn words\n",
        "    batched_past_turn_idx = []\n",
        "    #stores the id's of response words\n",
        "    batched_response_idx = []\n",
        "    #stores the lengths of past_turns for masking\n",
        "    batched_past_lengths = []\n",
        "    #stores the lengths of responses for masking\n",
        "    batched_past_resp_lengths = []\n",
        "    #stores the emotions associated with a batch\n",
        "    batched_emotions = []\n",
        "\n",
        "    #creates batches\n",
        "    N = len(past_turns)\n",
        "    if N % batch_size == 0:\n",
        "      num_batches = N // batch_size\n",
        "    else:\n",
        "      num_batches = N // batch_size + 1\n",
        "\n",
        "    for b in range(num_batches):\n",
        "      start = b * batch_size\n",
        "      stop = min((b+1) * batch_size, len(past_turns))\n",
        "      #calculates the max lengths of response and past turn sequences for this batch\n",
        "      max_resp_seq_len = max([len(s) for s in responses[start:stop]])\n",
        "      max_past_seq_len = max([len(s) for s in past_turns[start:stop]])\n",
        "\n",
        "      #creates the vectors for the past_turn and responses\n",
        "      past_turn_idx = np.zeros((stop-start, max_past_seq_len))\n",
        "      response_idx = np.zeros((stop-start, max_resp_seq_len + 2))\n",
        "      emotion_idx = np.empty((stop-start, 1))\n",
        "      past_lengths = np.zeros((stop-start))\n",
        "      resp_lengths = np.zeros((stop-start))\n",
        "      for i in range(start, stop):\n",
        "        #gathers the corresponding data\n",
        "        past_turn = past_turns[i]\n",
        "        response = responses[i]\n",
        "        emotion = emotions[i]\n",
        "        #gets ID for corresponding emotion\n",
        "        emotion_idx[i - start] = emotset[emotion]\n",
        "\n",
        "        #We start the response with START_DECODE to indicate to the model that decoding should start\n",
        "        response_idx[i - start][0] = START_DECODE\n",
        "\n",
        "        #this captures the lengths \n",
        "        past_lengths[i - start] = len(past_turn)\n",
        "        resp_lengths[i - start] = len(response)\n",
        "\n",
        "        #this gets the vocabulary IDs for each word in the past_turn and response\n",
        "        #UNKNOWN_INDEX is used if the word is out of vocabulary\n",
        "        for j in range(len(past_turn)):\n",
        "          if past_turn[j] in vocab:\n",
        "            past_turn_idx[i - start][j] = vocab[past_turn[j]]\n",
        "          else:\n",
        "            past_turn_idx[i - start][j] = UNKNOWN_INDEX      \n",
        "        for j in range(len(response)):\n",
        "          if response[j] in vocab:\n",
        "            response_idx[i - start][j + 1] = vocab[response[j]]\n",
        "          else:\n",
        "            response_idx[i - start][j + 1] = UNKNOWN_INDEX\n",
        "            \n",
        "        #we want to end the response with END_DECODE so the model learns to predict the end of an utterance\n",
        "        response_idx[i - start][len(response)] = END_DECODE\n",
        "      batched_past_turn_idx.append(past_turn_idx)\n",
        "      batched_response_idx.append(response_idx)\n",
        "      batched_past_lengths.append(past_lengths)\n",
        "      batched_past_resp_lengths.append(resp_lengths)\n",
        "      batched_emotions.append(emotion_idx)\n",
        "    return batched_past_turn_idx, batched_past_lengths, batched_response_idx, batched_past_resp_lengths, batched_emotions"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uhjaUWnh4xZy"
      },
      "source": [
        "'''\n",
        "Helper function to extract a set of emotions from a dataset and associate them \n",
        "with an ID.\n",
        "\n",
        "Arguments-\n",
        "  emotion_file:   Data file we want to extract emotions from\n",
        "\n",
        "Returns:\n",
        "  emotset:        Dictionary of emotions\n",
        "'''\n",
        "def read_emotions(emotion_file):\n",
        "  emotset = {}\n",
        "  with open(emotion_file, encoding='utf8') as f:\n",
        "    csvreader = csv.reader(f, delimiter=',')\n",
        "    counter = 0\n",
        "    for row in csvreader:\n",
        "      emotion = row[2]\n",
        "      if emotion not in emotset:\n",
        "        emotset[emotion] = counter\n",
        "        counter += 1\n",
        "  \n",
        "  return emotset"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s5SeNJvK5Q_e"
      },
      "source": [
        "# this loads the 10,000 most common word 300-dimensional embeddings\n",
        "vocab_size = 10000\n",
        "embeddings, vocab = read_embeddings('glove.6B.300d.txt', vocab_size)\n",
        "\n",
        "# read the files\n",
        "emotset = read_emotions('empatheticdialogues/train.csv')\n",
        "train_dataset = Dataset('empatheticdialogues/train.csv')\n",
        "dev_dataset = Dataset('empatheticdialogues/valid.csv')\n",
        "test_dataset = Dataset('empatheticdialogues/test.csv')\n",
        "\n",
        "BATCH_SIZE = 32\n",
        "\n",
        "train_batched_past_turn_idx, train_batched_past_lengths, train_batched_response_idx, train_batched_past_resp_lengths, train_batched_emotions = train_dataset.get_batches(BATCH_SIZE, vocab, emotset)\n",
        "dev_batched_past_turn_idx, dev_batched_past_lengths, dev_batched_response_idx, dev_batched_past_resp_lengths, dev_batched_emotions= dev_dataset.get_batches(BATCH_SIZE, vocab, emotset)\n",
        "test_batched_past_turn_idx, test_batched_past_lengths, test_batched_response_idx, test_batched_past_resp_lengths, test_batched_emotions= test_dataset.get_batches(BATCH_SIZE, vocab, emotset)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rQgF4Ni89KGL"
      },
      "source": [
        "# Transformer Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ik4JzhLS83N-"
      },
      "source": [
        "'''\n",
        "This class is used to create transformer-style positional encodings.  \n",
        "Reference: https://github.com/pytorch/pytorch/issues/24826\n",
        "Note: these are different than the categorical positional encodings discussed in \n",
        "class for Information Extraction.\n",
        "'''\n",
        "class TransformerPositionalEncoding(nn.Module):\n",
        "    def __init__(self, d_model, dropout=0.1, max_len=160):\n",
        "        super(TransformerPositionalEncoding, self).__init__()\n",
        "        self.dropout = nn.Dropout(p=dropout)\n",
        "        #import pdb; pdb.set_trace()\n",
        "        pe = torch.zeros(max_len, d_model)\n",
        "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
        "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
        "        pe[:, 0::2] = torch.sin(position * div_term)\n",
        "        pe[:, 1::2] = torch.cos(position * div_term)\n",
        "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
        "        self.register_buffer('pe', pe)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = x + self.pe[:x.size(0), :]\n",
        "        return self.dropout(x)\n",
        "\n",
        "'''\n",
        "This is the TransformerGenerator class, where the generaiton model is set up \n",
        "and the model structure is defined.\n",
        "\n",
        "Please fill in your solution where you see \"...\" \n",
        "'''\n",
        "class TransformerGenerator(nn.Module):\n",
        "  def __init__(self, embeddings, ntoken, ninp, nhead, nhid, nlayers, dropout=0.5):\n",
        "    super().__init__()\n",
        "    self.embed = nn.Embedding.from_pretrained(embeddings, freeze=False)\n",
        "    #this should be a call to nn.Transformer() with the necessary parameters.\n",
        "    self.model = nn.Transformer(d_model = ninp, nhead=nhead, num_encoder_layers=nlayers, num_decoder_layers=nlayers, dropout=dropout)\n",
        "    # this should be the linear layer to produce the model’s output \n",
        "    # (the top grey Linear box in the Transformer diagram above).\n",
        "    self.out = nn.Linear(ninp, ntoken)\n",
        "    # this should be a TransformerPositionalEncoding for the input\n",
        "    self.pos_encoder = TransformerPositionalEncoding(ninp, dropout)\n",
        "    #this should be a TransformerPositionalEncoding for the output\n",
        "    self.pos_decoder = TransformerPositionalEncoding(ninp, dropout)\n",
        "\n",
        "\n",
        "  def forward(self, past_turn, past_turn_lengths, response, response_lengths):\n",
        "    past_turn = torch.LongTensor(past_turn).to(device)\n",
        "    response = torch.LongTensor(response).to(device)\n",
        "    past_turn_lengths = torch.LongTensor(past_turn_lengths)\n",
        "    response_lengths = torch.LongTensor(response_lengths)\n",
        "    \n",
        "    #print(\"input shapes: past turn, past lenghts, response, response lengths:\")\n",
        "    #print(past_turn.shape, past_turn_lengths.shape, response.shape, response_lengths.shape)\n",
        "    \n",
        "    # This should be a mask for the past_turn input. The value of this should be True for positions greater than the length of the input and False for positions less than the length. In other words, True values represent a PAD token for a position in the source sequence.\n",
        "    #print(past_turn.shape[0], past_turn.shape[1], past_turn_lengths.shape, past_turn[:5])\n",
        "    #print(max(past_turn_lengths))\n",
        "    src_masks = []\n",
        "    for sentence in range(past_turn.shape[1]):\n",
        "      length = past_turn_lengths[sentence].item()\n",
        "      s = []\n",
        "      for i in range(past_turn.shape[0]):\n",
        "        if i < length:\n",
        "          s.append(False)\n",
        "        else: \n",
        "          s.append(True)\n",
        "      #print(length,s)\n",
        "      src_masks.append(s)\n",
        "    #print(np.array(src_masks).shape)\n",
        "    src_masks = torch.LongTensor(src_masks).bool()\n",
        "    #print(src_masks.shape)\n",
        "    #print(src_masks)\n",
        "    \n",
        "    # This is similar to src_masks but for the response.\n",
        "    tgt_masks = []\n",
        "    for sentence in range(response.shape[1]):\n",
        "      length = response_lengths[sentence].item()\n",
        "      r = []\n",
        "      for i in range(response.shape[0]):\n",
        "        if i < length:\n",
        "          r.append(False)\n",
        "        else:\n",
        "          r.append(True)\n",
        "      #print(length,r)\n",
        "      tgt_masks.append(r)\n",
        "\n",
        "    #print(np.array(tgt_masks).shape)\n",
        "    tgt_masks = torch.LongTensor(tgt_masks).bool()\n",
        "    #print(tgt_masks.shape)\n",
        "    \n",
        "    # This should be the output of the input embedding and transformer-style positional encoding layers.\n",
        "    src = self.pos_encoder.forward(self.embed(past_turn))\n",
        "    \n",
        "    # This should be the output of the output embedding and positional encoding layers. \n",
        "    tgt = self.pos_decoder.forward(self.embed(response))\n",
        "\n",
        "    #Ensures decoder doesn't peek at the future tokens\n",
        "    cheater_mask = self.model.generate_square_subsequent_mask(sz = len(tgt)).to(device)\n",
        "    output = self.model(src, tgt, tgt_mask = cheater_mask, src_key_padding_mask=src_masks.to(device), tgt_key_padding_mask = tgt_masks.to(device))\n",
        "    output = self.out(output)\n",
        "    \n",
        "    return output\n",
        "\n",
        "\n",
        "'''\n",
        "This is a function which is used to evaluate a model on a development dataset.\n",
        "This method does not update the model; rather, it is used to evaluate a model's responses\n",
        "on a dataset.\n",
        "\n",
        "Arguments:\n",
        "  model:        Model to evaluate\n",
        "  dev_dataset:  Dataset we want to evaluate with\n",
        "  batch_size:   batch size for dev dataset\n",
        "  vocab:        Vocabulary for the dataset\n",
        "  emotset:      Set of emotions for the dataset\n",
        "\n",
        "Returns:\n",
        "  avg_loss:     The average loss for the model on the dev_dataset.\n",
        "'''\n",
        "def evaluate_on_data(model, dev_dataset, batch_size, vocab,  emotset):\n",
        "    loss_function = nn.CrossEntropyLoss(ignore_index=vocab[\"PAD_INDEX\"])\n",
        "    dev_batched_past_turn_idx, dev_batched_past_lengths, dev_batched_response_idx, dev_batched_resp_lengths, dev_batched_emotions= dev_dataset.get_batches(BATCH_SIZE, vocab, emotset)\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "      device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "      total_loss = 0\n",
        "      for b in range(len(dev_batched_past_turn_idx)):\n",
        "          # have to transpose since model expects them in a certain format\n",
        "          src = dev_batched_past_turn_idx[b].transpose([1, 0])\n",
        "          tgt = dev_batched_response_idx[b].transpose([1, 0])\n",
        "\n",
        "          #calls the model on the current batch's input\n",
        "          logits = model.forward(src, dev_batched_past_lengths[b], tgt[:-1,:], dev_batched_resp_lengths[b])\n",
        "          # move labels to GPU memory\n",
        "          labels = torch.LongTensor(dev_batched_response_idx[b].transpose([1,0])).to(device)\n",
        "          # compute the loss with respect to true words\n",
        "          loss = loss_function(logits.view(-1, 10000), labels[1:,:].reshape(-1))\n",
        "          total_loss += loss\n",
        "      avg_loss = total_loss / float(len(dev_batched_past_turn_idx))\n",
        "      avg_loss = float(avg_loss.detach().cpu().numpy())\n",
        "      return avg_loss\n",
        "\n",
        "'''\n",
        "This is the function used to train a model.\n",
        "\n",
        "Arguments:\n",
        "  model:          model we want to train\n",
        "  train_dataset:  dataset we want to train the model with\n",
        "  dev_dataset:    dataset we want to evaluate model with during training\n",
        "  batch_size:     batch size for training\n",
        "  vocab:          vocabulary for the dataset\n",
        "  emotset:        emotion set for the dataset\n",
        "  lr:             learning rate we want to use\n",
        "  num_epochs:     epochs we want to train our model for\n",
        "  eval_every:     how often we want to evaluate on the dev dataset\n",
        "'''\n",
        "def run_training(model, train_dataset, dev_dataset, batch_size, vocab,  emotset,\n",
        "                         lr=1e-4, num_epochs=100, eval_every=5):\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "    if str(device) == 'cpu':\n",
        "      print(\"Training only supported in GPU environment\")\n",
        "      return\n",
        "\n",
        "\n",
        "    # clear unreferenced data/models from GPU memory \n",
        "    torch.cuda.empty_cache()\n",
        "    # move model to GPU memory\n",
        "    model.to(device)\n",
        "\n",
        "    # set the optimizer (Adam) and loss function (CrossEnt)\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
        "    loss_function = nn.CrossEntropyLoss(ignore_index=vocab[\"PAD_INDEX\"])\n",
        "\n",
        "    # batch training and dev data\n",
        "    train_batched_past_turn_idx, train_batched_past_lengths, train_batched_response_idx, train_batched_resp_lengths, train_batched_emotions = train_dataset.get_batches(BATCH_SIZE, vocab, emotset)\n",
        "    #dev_batched_past_turn_idx, dev_batched_past_lengths, dev_batched_response_idx, dev_batched_resp_lengths, dev_batched_emotions= dev_dataset.get_batches(BATCH_SIZE, vocab, emotset)\n",
        "\n",
        "    t0 = time.time()\n",
        "    print(\"**** TRAINING *****\")\n",
        "    for i in range(num_epochs):\n",
        "      if i % eval_every == 0:\n",
        "      #  # Run on Dev data\n",
        "         dev_loss = evaluate_on_data(model, dev_dataset, batch_size, vocab,  emotset)\n",
        "         print(\"-------------------------------\")\n",
        "         print(\"Dev Loss: {}\".format(dev_loss))\n",
        "         print(\"-------------------------------\")\n",
        "\n",
        "      # sets the model in train mode\n",
        "      model.train()\n",
        "      total_loss = 0\n",
        "      for b in range(len(train_batched_past_turn_idx)):\n",
        "\n",
        "        # have to transpose since model expects them in a certain format\n",
        "        src = train_batched_past_turn_idx[b].transpose([1, 0])\n",
        "        tgt = train_batched_response_idx[b].transpose([1, 0])\n",
        "\n",
        "        #calls the model on the current batch's input\n",
        "        logits = model.forward(src, train_batched_past_lengths[b], tgt[:-1,:], train_batched_resp_lengths[b])\n",
        "        # move labels to GPU memory\n",
        "        labels = torch.LongTensor(train_batched_response_idx[b].transpose([1,0])).to(device)\n",
        "        # compute the loss with respect to true words\n",
        "        loss = loss_function(logits.view(-1, 10000), labels[1:,:].reshape(-1))\n",
        "        total_loss += loss\n",
        "\n",
        "        # propagate gradients backward\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        # set model gradients to zero before performing next forward pass\n",
        "        model.zero_grad()\n",
        "\n",
        "      seconds_elapsed = time.time()-t0\n",
        "      mins = int(np.floor(seconds_elapsed/60))\n",
        "      secs = int(seconds_elapsed - (60*mins))\n",
        "      print(\"Epoch {} | Train Loss: {} | Time: {} mins, {} secs\".format(i, total_loss / float(len(train_batched_past_turn_idx)),mins,secs))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4e1LiLFn-mZ4"
      },
      "source": [
        "def set_seed(seed):\n",
        "  \"\"\"\n",
        "  Sets random seeds and sets model in deterministic\n",
        "  training mode. Ensures reproducible results\n",
        "  \"\"\"\n",
        "  torch.manual_seed(seed)\n",
        "  torch.backends.cudnn.deterministic = True\n",
        "  torch.backends.cudnn.benchmark = False\n",
        "  np.random.seed(seed)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FRELt-cKm5RI"
      },
      "source": [
        "# sets the random seed – DO NOT change this\n",
        "# this ensures deterministic results that are comparable with the staff values\n",
        "set_seed(159)\n",
        "\n",
        "'''Do NOT change these parameters'''\n",
        "#Number of vocabulary words we have\n",
        "VOCAB_SIZE = 10000\n",
        "#Size of our word embeddings.  We embed each word before passing into the transformer layer,\n",
        "#so the transformer needs to know how large these embeddings will be\n",
        "NINP = 300\n",
        "#The number of heads we want our transformer model to have\n",
        "NHEAD = 6\n",
        "#The size of hidden dimensions we want our transformer to have\n",
        "NHIDDEN = 200\n",
        "#The number of layers we want our transformer to have\n",
        "NLAYERS = 1\n",
        "#Dropout rate\n",
        "DROPOUT = 0.4"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hsv_Rtks35gS",
        "outputId": "141a70df-b87c-4b92-eba8-80a0944e826d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        }
      },
      "source": [
        "'''\n",
        "NOTE: do NOT run this cell if you are loading a pre-trained model.\n",
        "'''\n",
        "\n",
        "#This is the call which initializes the model\n",
        "model = TransformerGenerator(embeddings, VOCAB_SIZE, NINP, NHEAD, NHIDDEN, NLAYERS, DROPOUT)\n",
        "\n",
        "# This call trains the model.  If you have implemented Q1 correctly, the loss should decrease from ~5.6 to ~3.8\n",
        "# Sanity check: if this function fails, your Q1 code is probably incorrect. \n",
        "run_training(model, train_dataset, dev_dataset, BATCH_SIZE, vocab, emotset, \n",
        "                   lr=1e-4, num_epochs=25, eval_every=5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "**** TRAINING *****\n",
            "-------------------------------\n",
            "Dev Loss: 9.19412899017334\n",
            "-------------------------------\n",
            "Epoch 0 | Train Loss: 5.538825988769531 | Time: 0 mins, 29 secs\n",
            "Epoch 1 | Train Loss: 4.734216690063477 | Time: 0 mins, 58 secs\n",
            "Epoch 2 | Train Loss: 4.550368785858154 | Time: 1 mins, 26 secs\n",
            "Epoch 3 | Train Loss: 4.452611446380615 | Time: 1 mins, 55 secs\n",
            "Epoch 4 | Train Loss: 4.3852858543396 | Time: 2 mins, 23 secs\n",
            "-------------------------------\n",
            "Dev Loss: 4.441157817840576\n",
            "-------------------------------\n",
            "Epoch 5 | Train Loss: 4.332594394683838 | Time: 2 mins, 53 secs\n",
            "Epoch 6 | Train Loss: 4.290670394897461 | Time: 3 mins, 21 secs\n",
            "Epoch 7 | Train Loss: 4.257047176361084 | Time: 3 mins, 50 secs\n",
            "Epoch 8 | Train Loss: 4.2250142097473145 | Time: 4 mins, 18 secs\n",
            "Epoch 9 | Train Loss: 4.197795867919922 | Time: 4 mins, 47 secs\n",
            "-------------------------------\n",
            "Dev Loss: 4.3237528800964355\n",
            "-------------------------------\n",
            "Epoch 10 | Train Loss: 4.173142910003662 | Time: 5 mins, 17 secs\n",
            "Epoch 11 | Train Loss: 4.14968204498291 | Time: 5 mins, 45 secs\n",
            "Epoch 12 | Train Loss: 4.129739761352539 | Time: 6 mins, 13 secs\n",
            "Epoch 13 | Train Loss: 4.1082048416137695 | Time: 6 mins, 42 secs\n",
            "Epoch 14 | Train Loss: 4.088576316833496 | Time: 7 mins, 10 secs\n",
            "-------------------------------\n",
            "Dev Loss: 4.27132511138916\n",
            "-------------------------------\n",
            "Epoch 15 | Train Loss: 4.072922706604004 | Time: 7 mins, 40 secs\n",
            "Epoch 16 | Train Loss: 4.056135177612305 | Time: 8 mins, 9 secs\n",
            "Epoch 17 | Train Loss: 4.039005756378174 | Time: 8 mins, 37 secs\n",
            "Epoch 18 | Train Loss: 4.020671844482422 | Time: 9 mins, 5 secs\n",
            "Epoch 19 | Train Loss: 4.0067572593688965 | Time: 9 mins, 34 secs\n",
            "-------------------------------\n",
            "Dev Loss: 4.243836879730225\n",
            "-------------------------------\n",
            "Epoch 20 | Train Loss: 3.992591142654419 | Time: 10 mins, 4 secs\n",
            "Epoch 21 | Train Loss: 3.9791131019592285 | Time: 10 mins, 32 secs\n",
            "Epoch 22 | Train Loss: 3.9639205932617188 | Time: 11 mins, 1 secs\n",
            "Epoch 23 | Train Loss: 3.950558662414551 | Time: 11 mins, 29 secs\n",
            "Epoch 24 | Train Loss: 3.938959836959839 | Time: 11 mins, 58 secs\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MRm_VBOrOHPe"
      },
      "source": [
        "**If you want to save your trained model so you don't have to train it again for #2, please run the following cell.  You will need to download the model file and import this to Colab the next time you'd like to load it.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ss4X5RHFxQIu"
      },
      "source": [
        "#Now, let's save this model so you won't have to run it again for #2.\n",
        "torch.save(model.state_dict(), \"./model\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vh4jVENRxfRA",
        "outputId": "6f5579e3-772e-4d81-8732-ae77704e2768",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 846
        }
      },
      "source": [
        "#NOTE: if you have a saved version of the model, un-comment and run this code to load your model back in.\n",
        "model = TransformerGenerator(embeddings, VOCAB_SIZE, NINP, NHEAD, NHIDDEN, NLAYERS, DROPOUT)\n",
        "model.load_state_dict(torch.load(\"./model\"))\n",
        "model.eval()\n",
        "model.to(device)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TransformerGenerator(\n",
              "  (embed): Embedding(10000, 300)\n",
              "  (model): Transformer(\n",
              "    (encoder): TransformerEncoder(\n",
              "      (layers): ModuleList(\n",
              "        (0): TransformerEncoderLayer(\n",
              "          (self_attn): MultiheadAttention(\n",
              "            (out_proj): Linear(in_features=300, out_features=300, bias=True)\n",
              "          )\n",
              "          (linear1): Linear(in_features=300, out_features=2048, bias=True)\n",
              "          (dropout): Dropout(p=0.4, inplace=False)\n",
              "          (linear2): Linear(in_features=2048, out_features=300, bias=True)\n",
              "          (norm1): LayerNorm((300,), eps=1e-05, elementwise_affine=True)\n",
              "          (norm2): LayerNorm((300,), eps=1e-05, elementwise_affine=True)\n",
              "          (dropout1): Dropout(p=0.4, inplace=False)\n",
              "          (dropout2): Dropout(p=0.4, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (norm): LayerNorm((300,), eps=1e-05, elementwise_affine=True)\n",
              "    )\n",
              "    (decoder): TransformerDecoder(\n",
              "      (layers): ModuleList(\n",
              "        (0): TransformerDecoderLayer(\n",
              "          (self_attn): MultiheadAttention(\n",
              "            (out_proj): Linear(in_features=300, out_features=300, bias=True)\n",
              "          )\n",
              "          (multihead_attn): MultiheadAttention(\n",
              "            (out_proj): Linear(in_features=300, out_features=300, bias=True)\n",
              "          )\n",
              "          (linear1): Linear(in_features=300, out_features=2048, bias=True)\n",
              "          (dropout): Dropout(p=0.4, inplace=False)\n",
              "          (linear2): Linear(in_features=2048, out_features=300, bias=True)\n",
              "          (norm1): LayerNorm((300,), eps=1e-05, elementwise_affine=True)\n",
              "          (norm2): LayerNorm((300,), eps=1e-05, elementwise_affine=True)\n",
              "          (norm3): LayerNorm((300,), eps=1e-05, elementwise_affine=True)\n",
              "          (dropout1): Dropout(p=0.4, inplace=False)\n",
              "          (dropout2): Dropout(p=0.4, inplace=False)\n",
              "          (dropout3): Dropout(p=0.4, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (norm): LayerNorm((300,), eps=1e-05, elementwise_affine=True)\n",
              "    )\n",
              "  )\n",
              "  (out): Linear(in_features=300, out_features=10000, bias=True)\n",
              "  (pos_encoder): TransformerPositionalEncoding(\n",
              "    (dropout): Dropout(p=0.4, inplace=False)\n",
              "  )\n",
              "  (pos_decoder): TransformerPositionalEncoding(\n",
              "    (dropout): Dropout(p=0.4, inplace=False)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MfRZScz17wLC"
      },
      "source": [
        "# Decoding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qZcBsYZgAl89"
      },
      "source": [
        "id2vocab = {}\n",
        "\n",
        "def id2string(vocab, response):\n",
        "\n",
        "    \"\"\"\n",
        "    id2string function, takes a vocabulary and response and translates the response to a list of strings using the vocab.\n",
        "    \n",
        "    Arguments:\n",
        "    - vocab:        vocabulary, keys are strings and values are IDs\n",
        "    - response:     list of IDs we want to translate\n",
        "\n",
        "    Returns \n",
        "    - str_response:   list of strings containing the words represented by the response's IDs\n",
        "    \"\"\"\n",
        "    str_response = []\n",
        "    for w in vocab:\n",
        "      id2vocab[vocab[w]] = w\n",
        "    \n",
        "    for i in response:\n",
        "      str_response.append(id2vocab[i])\n",
        "    return str_response\n",
        "\n",
        "    \n",
        "    \n",
        "def decode(model, prev_turn, prev_lengths, vocab, max_len, random_top_k=False):\n",
        "  #, batched_resps, batched_resp_lens, vocab, max_len):\n",
        "    \"\"\"\n",
        "    Decode function, takes a trained model and past_turn and returns the model's generated response\n",
        "    \n",
        "    Arguments:\n",
        "    - model:        trained model that we want to evaluate\n",
        "    - prev_turn:    The previous turn we want to generate a model response for\n",
        "    - prev_lengths: The length of the prev_turn\n",
        "    - max_len:      The maximum decoded sequence length\n",
        "    - random_top_k: Flag specifying whether to use topK decoding\n",
        "\n",
        "    Returns \n",
        "    - id2string(vocab, prediction):   list of strings indicating the words produced \n",
        "                                      by the model, calculated using id2string helper function. \n",
        "    \"\"\"\n",
        "\n",
        "    prediction = []\n",
        "    seq = [2]\n",
        "    curr_length = 1\n",
        "\n",
        "    #torch.Size([25, 32]) torch.Size([32]) torch.Size([43, 32]) torch.Size([32])\n",
        "    #You should call forward(), determine the most probable token from the model's probability distribution, \n",
        "    #append that token to seq, update the response length variable, and call forward() again.  \n",
        "    #The forward() should stop when END_DECODE is predicted or MAX_LEN is reached.\n",
        "    prev_turn = torch.LongTensor(prev_turn).transpose(0, 1)\n",
        "    prev_lengths = torch.LongTensor(prev_lengths)\n",
        "\n",
        "    token = None\n",
        "    while (token != vocab[\"END_DECODE\"]):\n",
        "      seq_tensor = torch.LongTensor(seq).unsqueeze(1)\n",
        "      if curr_length == max_len:\n",
        "        if token != vocab[\"END_DECODE\"]:\n",
        "          seq.append(vocab[\"END_DECODE\"])\n",
        "        break;\n",
        "      distr = model.forward(prev_turn, prev_lengths, seq_tensor, torch.LongTensor(curr_length))\n",
        "      if random_top_k:\n",
        "        options = torch.topk(distr[curr_length-1][0], k=5, dim=0)[1]\n",
        "        pick = random.randint(0, 4)\n",
        "        token = options[pick].item()\n",
        "        #print(options,pick,token)\n",
        "      else:\n",
        "        token = torch.max(distr[curr_length-1][0], 0)[1].item()\n",
        "      seq.append(token)\n",
        "      curr_length += 1\n",
        "    #print(seq)\n",
        "    prediction = seq[1:]\n",
        "    return id2string(vocab, prediction)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oZRfpe9nzrgW"
      },
      "source": [
        "### Let's test the base decode() function by evaluating it with some model responses to dev set data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "emS1i7NB806j"
      },
      "source": [
        "'''\n",
        "This function runs a trained model to respond to random examples from a dataset\n",
        "\n",
        "Arguments:\n",
        "  dataset:      Dataset we want to evaluate the model with\n",
        "  model:        The model we want to evaluate\n",
        "  random_top_k: Whether we want to use topk decoding\n",
        "'''\n",
        "def generate_5_responses(dataset, model, random_top_k=False):\n",
        "  dev_batched_past_turn_idx, dev_batched_past_lengths, dev_batched_response_idx, dev_batched_resp_lengths, dev_batched_emotions= dataset.get_batches(BATCH_SIZE, vocab, emotset)\n",
        "  for _ in range(5):\n",
        "    rand_batch = random.randint(0, len(dev_batched_past_turn_idx)-1)\n",
        "    rand_item = random.randint(0, len(dev_batched_past_turn_idx[0])-1)\n",
        "    print(\"Past response: \")\n",
        "    print(id2string(vocab, [x for x in dev_batched_past_turn_idx[rand_batch][rand_item] if x != PAD_INDEX]))\n",
        "    print(\"Model Response: \")\n",
        "    model_resp = decode(model=model, prev_turn=[dev_batched_past_turn_idx[rand_batch][rand_item]], prev_lengths=[dev_batched_past_lengths[rand_batch][rand_item]], vocab=vocab, max_len=20, random_top_k=random_top_k)\n",
        "    print(model_resp)\n",
        "    print(\"Gold Response: \")\n",
        "    print(id2string(vocab, [x for x in dev_batched_response_idx[rand_batch][rand_item][1:] if x != PAD_INDEX]))\n",
        "    print()\n",
        "    print(\"---------------------------------\")\n",
        "    print()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J5-as-xwYkXu",
        "outputId": "0aaeab6f-b57a-4227-c125-a4300c581103",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 768
        }
      },
      "source": [
        "generate_5_responses(dev_dataset, model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Past response: \n",
            "['that', 'sounds', 'UNKNOWN_INDEX', ',', 'did', 'you', 'wash', 'it', 'or', 'throw', 'it', 'away', '?']\n",
            "Model Response: \n",
            "['no', ',', 'i', 'did', \"n't\", 'know', 'what', 'i', 'did', \"n't\", 'know', 'what', 'i', 'did', \"n't\", 'know', 'what', 'i', 'was', 'END_DECODE']\n",
            "Gold Response: \n",
            "['i', 'washed', 'it', 'and', 'it', 'came', 'out', 'surprisingly', '.', 'but', 'at', 'that', 'time', ',', 'i', 'felt', 'so', 'sick', 'seeing', 'what', 'was', 'happening', 'to', 'my', 'jacket', 'that', 'i', 'thought', 'i', 'was', 'going', 'to', 'UNKNOWN_INDEX', 'next', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['what', 'made', 'you', 'realize', 'that', '?', 'you', 'ca', \"n't\", 'live', 'in', 'fear']\n",
            "Model Response: \n",
            "['i', 'do', \"n't\", 'know', 'what', 'i', 'have', 'to', 'be', 'UNKNOWN_INDEX', 'END_DECODE']\n",
            "Gold Response: \n",
            "['yes', ',', 'but', 'if', 'you', 'do', \"n't\", 'run', 'away', 'from', 'it', ',', 'there', 'wo', 'nt', 'be', 'any', 'fear', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['well', 'maybe', 'you', 'guys', 'should', 'talk']\n",
            "Model Response: \n",
            "['i', 'am', 'so', 'UNKNOWN_INDEX', 'i', \"'m\", 'not', 'sure', 'i', \"'m\", 'not', 'sure', 'i', \"'m\", 'not', 'sure', 'END_DECODE']\n",
            "Gold Response: \n",
            "['i', 'told', 'him', 'i', 'was', 'disappointed', 'in', 'his', 'actions', ',', 'but', 'i', 'still', 'love', 'him', '.', 'we', 'all', 'make', 'mistakes', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['i', 'need', 'some', 'good', 'rest', ',', 'myself', '.', 'sleep', 'is', 'so', 'important', '!']\n",
            "Model Response: \n",
            "['i', 'know', ',', 'i', \"'m\", 'not', 'sure', 'i', \"'m\", 'not', 'sure', 'if', 'i', \"'m\", 'not', 'sure', 'i', \"'m\", 'not', 'END_DECODE']\n",
            "Gold Response: \n",
            "['yes', 'it', 'is', 'but', 'i', 'always', 'have', 'a', 'hard', 'time', 'getting', 'off', 'the', 'UNKNOWN_INDEX', 'to', 'go', 'to', 'bed', 'sometimes', 'i', 'just', 'fall', 'UNKNOWN_INDEX', 'on', 'the', 'UNKNOWN_INDEX', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['never', 'know', 'who', 'was', '?']\n",
            "Model Response: \n",
            "['he', 'was', 'a', 'UNKNOWN_INDEX', 'UNKNOWN_INDEX', '.', 'i', 'was', 'so', 'UNKNOWN_INDEX', 'END_DECODE']\n",
            "Gold Response: \n",
            "['the', 'women', 'that', 'hit', 'my', 'car', 'and', 'left', 'before', 'the', 'police', 'came', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M2R6gNzvz9zS"
      },
      "source": [
        "### Now, let's compare the base decode() method to the Random-Top5 decoding method. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZmqcOyzR0ITO",
        "outputId": "d2cf86b5-ea9a-4f39-ed48-cb32f88287b1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 768
        }
      },
      "source": [
        "generate_5_responses(dev_dataset, model, random_top_k=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Past response: \n",
            "['that', \"'s\", 'terrible', '!', 'did', 'the', 'dog', 'survive', '?']\n",
            "Model Response: \n",
            "['no', ',', 'she', 'is', 'UNKNOWN_INDEX', '.', 'i', 'was', \"n't\", 'UNKNOWN_INDEX', 'UNKNOWN_INDEX', ',', 'but', 'he', \"'s\", 'UNKNOWN_INDEX', '.', 'it', \"'s\", 'END_DECODE']\n",
            "Gold Response: \n",
            "['yes', ',', 'the', 'dog', 'turned', 'out', 'to', 'be', 'fine', ',', 'kid', 'said', 'he', 'was', 'training', 'he', 'her', 'to', 'be', 'stronger', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['that', 'sounds', 'like', 'a', 'UNKNOWN_INDEX', '.', 'hopefully', ',', 'you', 'did', 'not', 'lose', 'too', 'much', '.']\n",
            "Model Response: \n",
            "['yes', '.', 'he', \"'s\", 'UNKNOWN_INDEX', 'and', 'got', 'me', 'out', 'the', 'whole', 'UNKNOWN_INDEX', 'UNKNOWN_INDEX', 'END_DECODE']\n",
            "Gold Response: \n",
            "['UNKNOWN_INDEX', ',', 'i', \"'m\", 'a', 'little', 'careful', 'when', 'i', 'go', 'out', 'there', '.', 'UNKNOWN_INDEX', 'it', 'would', 'have', 'been', 'nice', 'to', 'join', 'in', 'on', 'the', 'winning', 'though', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['oh', 'no', 'do', 'you', 'feel', 'ready', 'for', 'it', '?']\n",
            "Model Response: \n",
            "['UNKNOWN_INDEX', 'no', 'i', 'have', 'a', 'few', 'times', 'and', 'my', 'own', 'own', 'UNKNOWN_INDEX', 'END_DECODE']\n",
            "Gold Response: \n",
            "['i', \"'m\", 'super', 'ready', 'for', 'it', '!', 'i', 'spent', 'so', 'much', 'time', 'studying', 'over', 'the', 'last', 'week', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['it', \"'s\", 'hard', 'to', 'trust', 'a', 'UNKNOWN_INDEX', '.', 'i', 'have', 'heard', 'a', 'lot', 'of', 'stories', 'about', 'people', 'being', 'UNKNOWN_INDEX', 'the', 'wrong', 'way', '.', 'please', 'tell', 'me', 'you', 'did', \"n't\", 'do', 'it', '.']\n",
            "Model Response: \n",
            "['it', \"'s\", 'UNKNOWN_INDEX', ',', 'but', 'i', \"'m\", 'just', 'not', 'expecting', 'to', 'get', 'a', 'few', 'times', 'END_DECODE']\n",
            "Gold Response: \n",
            "['i', 'was', 'very', 'nervous', 'and', 'UNKNOWN_INDEX', 'to', 'do', 'it', '.', 'i', 'actually', 'did', 'and', 'it', 'was', 'nt', 'that', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['nice', '!', 'do', 'you', 'have', 'any', 'UNKNOWN_INDEX', '?']\n",
            "Model Response: \n",
            "['yes', 'but', 'but', 'my', 'friend', \"'s\", 'a', 'lot', 'to', 'be', 'END_DECODE']\n",
            "Gold Response: \n",
            "['in', 'a', 'way', 'yes', ',', 'yes', 'i', 'do', 'the', 'power', 'of', 'my', 'will', 'and', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BCkPZwaP7yLT"
      },
      "source": [
        "# Effect of Emotions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4dWUtkoJDgEP",
        "outputId": "ca285f1a-d4e6-48d3-a494-499c6ba82f73",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "#Here is where we divide up the emotions in the emotset into positive and negative emotions, of equal sizes.\n",
        "positive_emotions = ['anticipating', 'caring', 'confident', 'content', 'excited', 'faithful', 'grateful', 'hopeful', 'impressed', 'joyful', 'nostalgic', 'prepared', 'proud', 'sentimental','surprised','trusting']\n",
        "negative_emotions = ['afraid', 'angry', 'annoyed', 'anxious', 'apprehensive', 'ashamed','devastated','disappointed','disgusted', 'embarrassed','furious','guilty','jealous','lonely','sad','terrified']\n",
        "len(positive_emotions), len(negative_emotions)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(16, 16)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CKMtZHLfDgSP"
      },
      "source": [
        "# creates the positive and negative datasets, specifying the relevant emotions_list\n",
        "train_dataset_positive = Dataset('empatheticdialogues/train.csv', emotions_list=positive_emotions)\n",
        "dev_dataset_positive = Dataset('empatheticdialogues/valid.csv', emotions_list=positive_emotions)\n",
        "test_dataset_positive = Dataset('empatheticdialogues/test.csv', emotions_list=positive_emotions)\n",
        "\n",
        "train_dataset_negative = Dataset('empatheticdialogues/train.csv', emotions_list=negative_emotions)\n",
        "dev_dataset_negative = Dataset('empatheticdialogues/valid.csv', emotions_list=negative_emotions)\n",
        "test_dataset_negative = Dataset('empatheticdialogues/test.csv', emotions_list=negative_emotions)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pLyS6YVuLO4Q",
        "outputId": "3213d05f-8ae4-4c74-84ee-d8d40bcca336",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "'''\n",
        "NOTE: these models train quicker than the Q1 model, as they are trained on 1/2 of the data.\n",
        "'''\n",
        "\n",
        "#we need to reset these embeddings or they will be shared among all 3 models.\n",
        "positive_embeddings, _ = read_embeddings('glove.6B.300d.txt', vocab_size)\n",
        "negative_embeddings, _ = read_embeddings('glove.6B.300d.txt', vocab_size)\n",
        "\n",
        "#This is the call which initializes the model\n",
        "positive_model = TransformerGenerator(positive_embeddings, VOCAB_SIZE, NINP, NHEAD, NHIDDEN, NLAYERS, DROPOUT)\n",
        "negative_model = TransformerGenerator(negative_embeddings, VOCAB_SIZE, NINP, NHEAD, NHIDDEN, NLAYERS, DROPOUT)\n",
        "\n",
        "print(\"Positive model\")\n",
        "run_training(positive_model, train_dataset_positive, dev_dataset_positive, BATCH_SIZE, vocab, emotset, \n",
        "                   lr=1e-4, num_epochs=25, eval_every=5)\n",
        "print(\"Negative model\")\n",
        "run_training(negative_model, train_dataset_negative, dev_dataset_negative, BATCH_SIZE, vocab, emotset, \n",
        "                   lr=1e-4, num_epochs=25, eval_every=5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Positive model\n",
            "**** TRAINING *****\n",
            "-------------------------------\n",
            "Dev Loss: 9.43320369720459\n",
            "-------------------------------\n",
            "Epoch 0 | Train Loss: 6.134271144866943 | Time: 0 mins, 21 secs\n",
            "Epoch 1 | Train Loss: 5.487732887268066 | Time: 0 mins, 41 secs\n",
            "Epoch 2 | Train Loss: 5.279111862182617 | Time: 1 mins, 1 secs\n",
            "Epoch 3 | Train Loss: 5.097773551940918 | Time: 1 mins, 21 secs\n",
            "Epoch 4 | Train Loss: 4.97153902053833 | Time: 1 mins, 41 secs\n",
            "-------------------------------\n",
            "Dev Loss: 4.955872535705566\n",
            "-------------------------------\n",
            "Epoch 5 | Train Loss: 4.870154857635498 | Time: 2 mins, 2 secs\n",
            "Epoch 6 | Train Loss: 4.7883477210998535 | Time: 2 mins, 22 secs\n",
            "Epoch 7 | Train Loss: 4.7170939445495605 | Time: 2 mins, 42 secs\n",
            "Epoch 8 | Train Loss: 4.658893585205078 | Time: 3 mins, 2 secs\n",
            "Epoch 9 | Train Loss: 4.607816696166992 | Time: 3 mins, 22 secs\n",
            "-------------------------------\n",
            "Dev Loss: 4.67985725402832\n",
            "-------------------------------\n",
            "Epoch 10 | Train Loss: 4.561574459075928 | Time: 3 mins, 43 secs\n",
            "Epoch 11 | Train Loss: 4.5221943855285645 | Time: 4 mins, 3 secs\n",
            "Epoch 12 | Train Loss: 4.486129283905029 | Time: 4 mins, 23 secs\n",
            "Epoch 13 | Train Loss: 4.457493305206299 | Time: 4 mins, 43 secs\n",
            "Epoch 14 | Train Loss: 4.425626277923584 | Time: 5 mins, 2 secs\n",
            "-------------------------------\n",
            "Dev Loss: 4.582110404968262\n",
            "-------------------------------\n",
            "Epoch 15 | Train Loss: 4.398252487182617 | Time: 5 mins, 24 secs\n",
            "Epoch 16 | Train Loss: 4.372033596038818 | Time: 5 mins, 43 secs\n",
            "Epoch 17 | Train Loss: 4.347303867340088 | Time: 6 mins, 3 secs\n",
            "Epoch 18 | Train Loss: 4.325154781341553 | Time: 6 mins, 23 secs\n",
            "Epoch 19 | Train Loss: 4.303622245788574 | Time: 6 mins, 43 secs\n",
            "-------------------------------\n",
            "Dev Loss: 4.522642135620117\n",
            "-------------------------------\n",
            "Epoch 20 | Train Loss: 4.28161096572876 | Time: 7 mins, 4 secs\n",
            "Epoch 21 | Train Loss: 4.262352466583252 | Time: 7 mins, 24 secs\n",
            "Epoch 22 | Train Loss: 4.241543292999268 | Time: 7 mins, 44 secs\n",
            "Epoch 23 | Train Loss: 4.221987247467041 | Time: 8 mins, 4 secs\n",
            "Epoch 24 | Train Loss: 4.204221248626709 | Time: 8 mins, 24 secs\n",
            "Negative model\n",
            "**** TRAINING *****\n",
            "-------------------------------\n",
            "Dev Loss: 9.483702659606934\n",
            "-------------------------------\n",
            "Epoch 0 | Train Loss: 6.063594818115234 | Time: 0 mins, 22 secs\n",
            "Epoch 1 | Train Loss: 5.383795261383057 | Time: 0 mins, 43 secs\n",
            "Epoch 2 | Train Loss: 5.166941165924072 | Time: 1 mins, 4 secs\n",
            "Epoch 3 | Train Loss: 4.99073600769043 | Time: 1 mins, 25 secs\n",
            "Epoch 4 | Train Loss: 4.861337184906006 | Time: 1 mins, 46 secs\n",
            "-------------------------------\n",
            "Dev Loss: 4.842034816741943\n",
            "-------------------------------\n",
            "Epoch 5 | Train Loss: 4.764353275299072 | Time: 2 mins, 8 secs\n",
            "Epoch 6 | Train Loss: 4.687539100646973 | Time: 2 mins, 29 secs\n",
            "Epoch 7 | Train Loss: 4.618865966796875 | Time: 2 mins, 50 secs\n",
            "Epoch 8 | Train Loss: 4.5620951652526855 | Time: 3 mins, 11 secs\n",
            "Epoch 9 | Train Loss: 4.5154547691345215 | Time: 3 mins, 32 secs\n",
            "-------------------------------\n",
            "Dev Loss: 4.5692362785339355\n",
            "-------------------------------\n",
            "Epoch 10 | Train Loss: 4.47127628326416 | Time: 3 mins, 55 secs\n",
            "Epoch 11 | Train Loss: 4.435220718383789 | Time: 4 mins, 16 secs\n",
            "Epoch 12 | Train Loss: 4.402187347412109 | Time: 4 mins, 37 secs\n",
            "Epoch 13 | Train Loss: 4.372888565063477 | Time: 4 mins, 58 secs\n",
            "Epoch 14 | Train Loss: 4.340875625610352 | Time: 5 mins, 19 secs\n",
            "-------------------------------\n",
            "Dev Loss: 4.460622787475586\n",
            "-------------------------------\n",
            "Epoch 15 | Train Loss: 4.315242290496826 | Time: 5 mins, 41 secs\n",
            "Epoch 16 | Train Loss: 4.29177713394165 | Time: 6 mins, 2 secs\n",
            "Epoch 17 | Train Loss: 4.267686367034912 | Time: 6 mins, 23 secs\n",
            "Epoch 18 | Train Loss: 4.249639987945557 | Time: 6 mins, 44 secs\n",
            "Epoch 19 | Train Loss: 4.229484558105469 | Time: 7 mins, 6 secs\n",
            "-------------------------------\n",
            "Dev Loss: 4.405383110046387\n",
            "-------------------------------\n",
            "Epoch 20 | Train Loss: 4.20750617980957 | Time: 7 mins, 28 secs\n",
            "Epoch 21 | Train Loss: 4.192046165466309 | Time: 7 mins, 49 secs\n",
            "Epoch 22 | Train Loss: 4.173384666442871 | Time: 8 mins, 10 secs\n",
            "Epoch 23 | Train Loss: 4.155624866485596 | Time: 8 mins, 31 secs\n",
            "Epoch 24 | Train Loss: 4.141120433807373 | Time: 8 mins, 52 secs\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tmNYBkRWynJL"
      },
      "source": [
        "### Let's decode the *positive* model on some *positive* data to see the types of responses it produces."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZxhvkxSu8u_0",
        "outputId": "6e331791-dd94-4b89-c7ae-f9f85da7cf89",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 768
        }
      },
      "source": [
        "generate_5_responses(dataset=dev_dataset_positive, model=positive_model, random_top_k=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Past response: \n",
            "['you', 'are', 'so', 'lucky', 'to', 'find', 'someone', 'special', 'online', '.', 'that', 'can', 'turn', 'out', 'so', 'badly', 'sometimes', 'for', 'some', 'people', '.']\n",
            "Model Response: \n",
            "['yeah', 'it', 'was', ',', 'and', 'UNKNOWN_INDEX', 'i', \"'m\", 'so', 'much', 'so', 'END_DECODE']\n",
            "Gold Response: \n",
            "['oh', 'i', 'know', '.', 'i', 'got', 'lucky', ',', 'we', 'found', 'out', 'we', 'lived', 'really', 'close', 'to', 'each', 'other', 'so', 'we', 'met', 'up', 'and', 'it', 'has', 'been', 'great', 'ever', 'since', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['i', 'have', 'the', 'same', 'feeling', '.', 'i', 'miss', 'her', 'also']\n",
            "Model Response: \n",
            "['it', 'is', 'a', 'lot', ',', 'and', 'i', 'do', 'END_DECODE']\n",
            "Gold Response: \n",
            "['she', 'passed', 'last', 'year', '.', 'she', 'is', 'in', 'a', 'better', 'place', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['i', 'would', 'hope', 'so', 'too', 'or', 'else', 'the', 'relationship', 'would', 'be', 'UNKNOWN_INDEX', '.']\n",
            "Model Response: \n",
            "['UNKNOWN_INDEX', 'UNKNOWN_INDEX', '.', 'we', 'are', 'going', 'through', 'the', 'UNKNOWN_INDEX', '.', 'i', \"'m\", 'glad', 'END_DECODE']\n",
            "Gold Response: \n",
            "['yes', 'that', 's', 'true', ',', 'to', 'say', 'the', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['i', 'was', 'going', 'to', 'say', ',', 'yeah', ',', 'that', \"'s\", 'the', 'kind', 'of', 'thing', 'i', 'would', 'love', 'to', 'preserve', 'in', 'a', 'frame', 'or', 'something', 'for', 'my', 'office', '.', 'how', 'cool', '!']\n",
            "Model Response: \n",
            "['that', 'is', 'UNKNOWN_INDEX', '!', 'we', \"'re\", 'glad', 'that', 'i', \"'m\", 'glad', 'i', 'think', 'that', 'END_DECODE']\n",
            "Gold Response: \n",
            "['yeah', ',', 'it', \"'s\", 'a', 'nice', 'little', 'UNKNOWN_INDEX', 'to', 'remind', 'me', 'of', 'good', 'times', '.', 'UNKNOWN_INDEX', 'it', 'was', 'a', 'great', 'concert', 'and', 'experience', 'for', 'all', 'of', 'us', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['ready', 'for', 'all', 'those', 'ice', 'UNKNOWN_INDEX', '?', 'those', 'amazing', 'crazy', 'party', '?', 'what', 'are', 'you', 'hoping', 'to', 'do', 'this', 'year', '?']\n",
            "Model Response: \n",
            "['i', 'do', 'you', 'UNKNOWN_INDEX', '.', 'but', 'we', 'are', 'in', 'END_DECODE']\n",
            "Gold Response: \n",
            "['i', 'am', 'really', 'hoping', 'i', 'can', 'make', 'some', 'good', 'friends', ',', 'and', 'just', 'learn', 'about', 'everything', 'i', 'do', \"n't\", 'know', 'a', 'lot', 'about', 'you', 'know', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KCGDPLI5y02x"
      },
      "source": [
        "### Let's decode the *negative* model on some *negative* data to see the types of responses it produces."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JpRmXr7NyvQd",
        "outputId": "d61a32a7-a567-4be4-b76f-b4a8163bdb33",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 768
        }
      },
      "source": [
        "generate_5_responses(dataset=dev_dataset_negative, model=negative_model, random_top_k=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Past response: \n",
            "['is', 'there', 'something', 'that', 'is', 'making', 'you', 'nervous', 'about', 'the', 'upcoming', 'week', '?']\n",
            "Model Response: \n",
            "['yeah', 'it', 'just', 'just', 'END_DECODE']\n",
            "Gold Response: \n",
            "['very', 'busy', 'and', 'lots', 'of', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['you', \"'re\", 'welcome', '.', 'it', \"'s\", 'always', 'nerve', 'UNKNOWN_INDEX', 'when', 'family', 'is', 'sick', 'like', 'that', '.']\n",
            "Model Response: \n",
            "['i', 'hope', 'they', 'do', 'not', 'get', 'to', 'the', 'same', 'time', 'END_DECODE']\n",
            "Gold Response: \n",
            "['END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['oh', 'no', '!', '!', 'how', 'did', 'you', 'dry', 'them', '?']\n",
            "Model Response: \n",
            "['i', 'had', 'to', 'the', 'police', ',', 'i', 'did', \"n't\", 'get', 'a', 'bad', 'UNKNOWN_INDEX', '.', 'UNKNOWN_INDEX', 'UNKNOWN_INDEX', 'they', \"'re\", 'so', 'END_DECODE']\n",
            "Gold Response: \n",
            "['they', 'are', 'still', 'air', 'UNKNOWN_INDEX', 'what', 'makes', 'it', 'worse', 'i', 'sat', 'in', 'the', 'sets', 'and', 'it', 'looked', 'like', 'i', 'wet', 'myself', 'UNKNOWN_INDEX', 'a', 'little', 'embarrassing', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['i', 'wonder', 'what', 'could', 'have', 'changed', '?', 'either', 'way', ',', 'i', \"'d\", 'be', 'pretty', 'UNKNOWN_INDEX', 'off', '.']\n",
            "Model Response: \n",
            "['no', ',', 'they', 'have', 'to', 'be', 'a', 'time', 'and', 'UNKNOWN_INDEX', 'and', 'i', 'was', 'a', 'bit', '.', 'and', 'it', 'is', 'END_DECODE']\n",
            "Gold Response: \n",
            "['i', 'am', '!', 'but', 'whatever', ',', 'nothing', 'we', 'can', 'do', 'but', 'have', 'the', 'city', 'come', 'pick', 'it', 'up', 'i', 'guess', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['do', 'you', 'think', 'you', 'will', 'ever', 'UNKNOWN_INDEX', 'line', 'again', '?']\n",
            "Model Response: \n",
            "['it', 'is', 'a', 'little', '.', 'it', \"'s\", 'hard', '.', 'i', 'do', 'that', 'will', 'do', 'nt', 'be', 'UNKNOWN_INDEX', 'for', 'them', 'END_DECODE']\n",
            "Gold Response: \n",
            "['oh', 'yes', '.', 'it', 'was', 'a', 'great', 'experience', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zZCCUs8hy30V"
      },
      "source": [
        "### Let's decode *both* models on some *positive* data to see the types of responses it produces."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Znget6A-yvZd",
        "outputId": "805c505b-19a4-4648-c1f9-4ba48db9a5cb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 931
        }
      },
      "source": [
        "dev_batched_past_turn_idx, dev_batched_past_lengths, dev_batched_response_idx, dev_batched_resp_lengths, dev_batched_emotions= dev_dataset_positive.get_batches(BATCH_SIZE, vocab, emotset)\n",
        "for _ in range(5):\n",
        "  rand_batch = random.randint(0, len(dev_batched_past_turn_idx)-1)\n",
        "  rand_item = random.randint(0, len(dev_batched_past_turn_idx[0])-1)\n",
        "  print(\"Past response: \")\n",
        "  print(id2string(vocab, [x for x in dev_batched_past_turn_idx[rand_batch][rand_item] if x != PAD_INDEX]))\n",
        "  print(\"POSITIVE model Response: \")\n",
        "  model_resp = decode(model=positive_model, prev_turn=[dev_batched_past_turn_idx[rand_batch][rand_item]], prev_lengths=[dev_batched_past_lengths[rand_batch][rand_item]], vocab=vocab, max_len=20, random_top_k=True)\n",
        "  print(model_resp)\n",
        "  print(\"NEGATIVE model Response: \")\n",
        "  model_resp = decode(model=negative_model, prev_turn=[dev_batched_past_turn_idx[rand_batch][rand_item]], prev_lengths=[dev_batched_past_lengths[rand_batch][rand_item]], vocab=vocab, max_len=20, random_top_k=True)\n",
        "  print(model_resp)\n",
        "  print(\"Gold Response: \")\n",
        "  print(id2string(vocab, [x for x in dev_batched_response_idx[rand_batch][rand_item][1:] if x != PAD_INDEX]))\n",
        "  print()\n",
        "  print(\"---------------------------------\")\n",
        "  print()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Past response: \n",
            "['surprise', 'gifts', 'are', 'always', 'fun', '.']\n",
            "POSITIVE model Response: \n",
            "['i', 'am', 'so', 'happy', '.', 'but', 'UNKNOWN_INDEX', 'we', 'are', 'going', 'in', 'a', 'couple', 'of', 'the', 'other', 'END_DECODE']\n",
            "NEGATIVE model Response: \n",
            "['yes', '!', 'i', 'do', \"n't\", 'like', 'the', 'same', 'one', 'is', 'a', 'lot', 'to', 'get', 'in', 'front', 'and', 'they', 'get', 'END_DECODE']\n",
            "Gold Response: \n",
            "['very', '!', 'i', 'decided', 'since', 'it', 'was', 'surprise', 'money', 'i', 'would', 'waste', 'it', 'on', 'something', 'fun', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['nice', '!', 'were', 'they', 'UNKNOWN_INDEX', 'UNKNOWN_INDEX', '?']\n",
            "POSITIVE model Response: \n",
            "['yes', 'i', 'did', '!', 'UNKNOWN_INDEX', 'i', 'am', 'just', 'just', 'all', 'going', 'back', 'to', 'do', 'it', '.', 'UNKNOWN_INDEX', 'UNKNOWN_INDEX', 'END_DECODE']\n",
            "NEGATIVE model Response: \n",
            "['i', 'had', 'a', 'bit', '.', 'UNKNOWN_INDEX', 'UNKNOWN_INDEX', 'they', 'was', 'so', 'bad', 'END_DECODE']\n",
            "Gold Response: \n",
            "['for', 'the', 'UNKNOWN_INDEX', '.', 'we', 'are', 'all', 'crossing', 'our', 'fingers', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['oh', 'UNKNOWN_INDEX', '.', 'give', 'me', 'the', 'details', '.']\n",
            "POSITIVE model Response: \n",
            "['yeah', 'it', 'will', 'have', \"n't\", 'know', 'what', 'i', 'was', 'so', 'UNKNOWN_INDEX', 'to', 'see', 'that', 'END_DECODE']\n",
            "NEGATIVE model Response: \n",
            "['no', ',', 'he', 'UNKNOWN_INDEX', '.', 'they', 'UNKNOWN_INDEX', 'my', 'best', 'friends', 'END_DECODE']\n",
            "Gold Response: \n",
            "['UNKNOWN_INDEX', ',', 'as', 'i', 'pulled', 'over', 'i', 'thought', 'it', 'was', 'someone', 'down', 'on', 'his', 'luck', '.', 'UNKNOWN_INDEX', ',', 'he', 'was', 'crazy', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['i', 'do', 'too', '.', 'i', 'have', 'a', 'box', 'filled', 'with', 'them', '.', 'how', 'old', 'is', 'your', 'oldest', '?']\n",
            "POSITIVE model Response: \n",
            "['yes', '.', 'he', \"'s\", 'a', 'great', 'time', '!', 'END_DECODE']\n",
            "NEGATIVE model Response: \n",
            "['yes', '.', 'he', 'was', 'really', 'i', \"'m\", 'afraid', 'for', 'a', 'little', 'lot', '.', 'UNKNOWN_INDEX', '.', 'it', 'makes', 'a', 'UNKNOWN_INDEX', 'END_DECODE']\n",
            "Gold Response: \n",
            "['i', 'have', 'some', 'old', 'ones', 'from', 'the', 'korean', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['sometimes', 'we', 'believe', 'is', 'the', 'things', 'that', 'happen', 'to', 'us', '...', 'they', 'are', 'many']\n",
            "POSITIVE model Response: \n",
            "['yes', 'i', 'think', 'so', 'they', 'have', 'been', 'so', 'i', \"'m\", 'trying', 'for', 'him', 'to', 'be', 'UNKNOWN_INDEX', 'and', 'have', 'END_DECODE']\n",
            "NEGATIVE model Response: \n",
            "['it', \"'s\", 'a', 'little', 'END_DECODE']\n",
            "Gold Response: \n",
            "['what', 'sort', 'of', 'things', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cjUfvnSwzEPb"
      },
      "source": [
        "### Let's decode *both* models on some *negative* data to see the types of responses it produces."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ccSl8SkyveX",
        "outputId": "eb808c1d-f94b-45e8-c958-bdb53d8245b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 931
        }
      },
      "source": [
        "dev_batched_past_turn_idx, dev_batched_past_lengths, dev_batched_response_idx, dev_batched_resp_lengths, dev_batched_emotions= dev_dataset_negative.get_batches(BATCH_SIZE, vocab, emotset)\n",
        "for _ in range(5):\n",
        "  rand_batch = random.randint(0, len(dev_batched_past_turn_idx)-1)\n",
        "  rand_item = random.randint(0, len(dev_batched_past_turn_idx[0])-1)\n",
        "  print(\"Past response: \")\n",
        "  print(id2string(vocab, [x for x in dev_batched_past_turn_idx[rand_batch][rand_item] if x != PAD_INDEX]))\n",
        "  print(\"POSITIVE model Response: \")\n",
        "  model_resp = decode(model=positive_model, prev_turn=[dev_batched_past_turn_idx[rand_batch][rand_item]], prev_lengths=[dev_batched_past_lengths[rand_batch][rand_item]], vocab=vocab, max_len=20, random_top_k=True)\n",
        "  print(model_resp)\n",
        "  print(\"NEGATIVE model Response: \")\n",
        "  model_resp = decode(model=negative_model, prev_turn=[dev_batched_past_turn_idx[rand_batch][rand_item]], prev_lengths=[dev_batched_past_lengths[rand_batch][rand_item]], vocab=vocab, max_len=20, random_top_k=True)\n",
        "  print(model_resp)\n",
        "  print(\"Gold Response: \")\n",
        "  print(id2string(vocab, [x for x in dev_batched_response_idx[rand_batch][rand_item][1:] if x != PAD_INDEX]))\n",
        "  print()\n",
        "  print(\"---------------------------------\")\n",
        "  print()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Past response: \n",
            "['i', 'am', 'sorry', 'to', 'hear', 'UNKNOWN_INDEX', 'you', 'get', 'hurt', '.']\n",
            "POSITIVE model Response: \n",
            "['i', \"'m\", 'going', 'to', 'see', 'it', ',', 'i', 'am', 'really', 'good', 'END_DECODE']\n",
            "NEGATIVE model Response: \n",
            "['i', 'am', 'so', 'UNKNOWN_INDEX', 'about', 'me', '.', 'UNKNOWN_INDEX', 'END_DECODE']\n",
            "Gold Response: \n",
            "['no', 'it', 'was', 'a', 'minor', 'accident', ',', 'but', 'car', 'got', 'damaged', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['where', 'was', 'she', '?']\n",
            "POSITIVE model Response: \n",
            "['she', \"'s\", 'going', 'through', 'to', 'her', 'and', 'her', 'birthday', 'with', 'me', '.', 'UNKNOWN_INDEX', 'END_DECODE']\n",
            "NEGATIVE model Response: \n",
            "['i', 'got', 'her', '.', 'i', 'had', 'been', 'so', '.', 'it', 'was', 'so', 'much', 'to', 'my', 'house', 'and', 'it', 'END_DECODE']\n",
            "Gold Response: \n",
            "['in', 'a', 'spare', 'bedroom', 'with', 'her', 'ex', ',', 'laying', 'on', 'the', 'bed', 'and', 'UNKNOWN_INDEX', 'him', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['well', 'i', 'guess', 'if', 'that', 'is', 'something', 'you', 'enjoy', '.', 'UNKNOWN_INDEX', 'i', 'ca', \"n't\", 'say', 'that', 'i', 'have', 'UNKNOWN_INDEX', 'of', 'friends', '.', 'UNKNOWN_INDEX', 'mostly', 'my', 'friends', 'are', 'family']\n",
            "POSITIVE model Response: \n",
            "['i', \"'m\", 'sure', 'so', 'proud', 'END_DECODE']\n",
            "NEGATIVE model Response: \n",
            "['UNKNOWN_INDEX', 'are', 'really', '.', 'i', 'do', '.', 'and', 'the', 'best', 'thing', 'will', 'try', 'END_DECODE']\n",
            "Gold Response: \n",
            "['well', 'in', 'that', 'case', ',', 'i', 'have', 'a', 'few', 'friends', '.', 'outside', 'of', 'family', ',', 'i', 'do', \"n't\", 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['i', 'do', \"n't\", 'like', 'them', 'at', 'all', '.', 'i', \"'ve\", 'seen', 'too', 'many', 'videos', 'on', 'the', 'internet', 'where', 'people', 'get', 'injured', 'or', 'they', 'break', 'easily']\n",
            "POSITIVE model Response: \n",
            "['yeah', 'it', 'is', 'so', ',', 'i', \"'m\", 'going', 'a', 'good', 'time', ',', 'i', 'have', 'been', 'the', 'UNKNOWN_INDEX', 'and', 'it', 'END_DECODE']\n",
            "NEGATIVE model Response: \n",
            "['it', 'will', 'not', '.', 'i', 'am', 'UNKNOWN_INDEX', 'so', 'upset', 'about', 'the', 'END_DECODE']\n",
            "Gold Response: \n",
            "['yes', ',', 'i', 'remember', 'in', '2007', 'a', 'teen', 'was', 'UNKNOWN_INDEX', 'at', 'six', 'flags', 'in', 'atlanta', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n",
            "Past response: \n",
            "['it', 'must', 'have', 'been', 'awful', 'to', 'see', 'that', 'in', 'someone', 'you', 'care', 'about', '.', 'i', \"'m\", 'glad', 'she', \"'s\", 'not', 'giving', 'up', '.']\n",
            "POSITIVE model Response: \n",
            "['it', \"'s\", 'very', 'hard', 'with', 'my', 'first', 'year', 'and', 'she', 'does', '.', 'UNKNOWN_INDEX', 'she', 'had', 'UNKNOWN_INDEX', 'she', 'UNKNOWN_INDEX', ',', 'END_DECODE']\n",
            "NEGATIVE model Response: \n",
            "['she', 'has', 'been', 'very', 'END_DECODE']\n",
            "Gold Response: \n",
            "['yeah', 'that', 's', 'what', 'i', 'said', '.', 'its', 'like', 'the', 'world', 'ended', 'when', 'she', 'found', 'out', ',', 'but', 'she', 'is', 'doing', 'better', 'now', 'END_DECODE']\n",
            "\n",
            "---------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H2Gy5akr0kTq"
      },
      "source": [
        "### Now, let's see which model does better when evaluated on the other's development set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sSnaRk490oq9"
      },
      "source": [
        "def compare_positive_and_negative_data(positive_model, negative_model, dev_dataset_positive, dev_dataset_negative):\n",
        "  '''\n",
        "  This method compares the positive model on the negative dataset and the negative model on the positive dataset.\n",
        "  You should use evaluate_on_data to get the training loss for each model.\n",
        "\n",
        "  Arguments:\n",
        "  positive_model:       model trained on positive data\n",
        "  negative_model:       model trained on negative data\n",
        "  dev_dataset_positive: dev dataset for positive data\n",
        "  dev_dataset_negative: dev dataset for negative data\n",
        "\n",
        "  Returns:\n",
        "  positive_model_negative_data:   result of evaluating the positive model on the negative dev dataset\n",
        "  negative_model_positive_data:   result of evaluating the negative model on the positive dev dataset\n",
        "  '''\n",
        "  #Your code goes here. \n",
        "  positive_model_negative_data = evaluate_on_data(positive_model, dev_dataset_positive, BATCH_SIZE, vocab,  emotset)\n",
        "  negative_model_positive_data = evaluate_on_data(negative_model, dev_dataset_positive, BATCH_SIZE, vocab,  emotset)\n",
        "\n",
        "\n",
        "  return positive_model_negative_data, negative_model_positive_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EBORbA8oD2Xc",
        "outputId": "a66aff26-92b4-43b2-cc22-1a76f187a049",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "positive_model_negative_data, negative_model_positive_data = compare_positive_and_negative_data(positive_model,  \n",
        "                                                                                                negative_model, \n",
        "                                                                                                dev_dataset_positive, \n",
        "                                                                                                dev_dataset_negative)\n",
        "print(\"Positive model loss on negative data:\", positive_model_negative_data)\n",
        "print(\"Negative model loss on positive data:\", negative_model_positive_data)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Positive model loss on negative data: 4.492856979370117\n",
            "Negative model loss on positive data: 4.664223670959473\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AazxFndM_-j2"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}