Text_Processing_RNN_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eaTI-Dn1RTK6",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "eaTI-Dn1RTK6",
    "outputId": "a461354a-74ed-414e-e344-14d8da7091cc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "zsh:1: command not found: pip\n",
      "zsh:1: command not found: pip\n",
      "zsh:1: command not found: pip\n",
      "zsh:1: command not found: pip\n",
      "zsh:1: command not found: pip\n",
      "zsh:1: command not found: pip\n"
     ]
    }
   ],
   "source": [
    "# importing required packages\n",
    "!pip install nltk==3.6.1\n",
    "!pip install numpy==1.18.5\n",
    "!pip install pandas==1.3.0\n",
    "!pip install torch==1.9.0\n",
    "!pip install tqdm==4.59.0\n",
    "!pip install scikit_learn==1.0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6ccf15cd",
   "metadata": {
    "id": "6ccf15cd"
   },
   "outputs": [],
   "source": [
    "# importing the required libraries\n",
    "import re\n",
    "import torch\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5547d608",
   "metadata": {
    "id": "5547d608"
   },
   "outputs": [],
   "source": [
    "# defineing configuration file paths\n",
    "lr = 0.0001\n",
    "input_size = 50\n",
    "num_epochs = 5\n",
    "hidden_size = 50\n",
    "label_col = \"Product\"\n",
    "tokens_path = \"Output/tokens.pkl\"\n",
    "labels_path = \"Output/labels.pkl\"\n",
    "data_path = \"Input/complaints.csv\"\n",
    "rnn_model_path = \"Output/model_rnn.pth\"\n",
    "lstm_model_path = \"Output/model_lstm.pth\"\n",
    "vocabulary_path = \"Output/vocabulary.pkl\"\n",
    "embeddings_path = \"Output/embeddings.pkl\"\n",
    "glove_vector_path = \"Input/glove.6B.50d.txt\"\n",
    "text_col_name = \"Consumer complaint narrative\"\n",
    "label_encoder_path = \"Output/label_encoder.pkl\"\n",
    "product_map = {'Vehicle loan or lease': 'vehicle_loan',\n",
    "               'Credit reporting, credit repair services, or other personal consumer reports': 'credit_report',\n",
    "               'Credit card or prepaid card': 'card',\n",
    "               'Money transfer, virtual currency, or money service': 'money_transfer',\n",
    "               'virtual currency': 'money_transfer',\n",
    "               'Mortgage': 'mortgage',\n",
    "               'Payday loan, title loan, or personal loan': 'loan',\n",
    "               'Debt collection': 'debt_collection',\n",
    "               'Checking or savings account': 'savings_account',\n",
    "               'Credit card': 'card',\n",
    "               'Bank account or service': 'savings_account',\n",
    "               'Credit reporting': 'credit_report',\n",
    "               'Prepaid card': 'card',\n",
    "               'Payday loan': 'loan',\n",
    "               'Other financial service': 'others',\n",
    "               'Virtual currency': 'money_transfer',\n",
    "               'Student loan': 'loan',\n",
    "               'Consumer Loan': 'loan',\n",
    "               'Money transfers': 'money_transfer'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9935336a",
   "metadata": {
    "id": "9935336a"
   },
   "outputs": [],
   "source": [
    "# defineing function for saving a file\n",
    "def save_file(name, obj):\n",
    "    \"\"\"\n",
    "    Function to save an object as pickle file\n",
    "    \"\"\"\n",
    "    with open(name, 'wb') as f:\n",
    "        pickle.dump(obj, f)\n",
    "\n",
    "# defineing function for loading a file\n",
    "def load_file(name):\n",
    "    \"\"\"\n",
    "    Function to load a pickle object\n",
    "    \"\"\"\n",
    "    return pickle.load(open(name, \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "531f3f24",
   "metadata": {
    "id": "531f3f24"
   },
   "outputs": [],
   "source": [
    "# reading glove file\n",
    "with open(glove_vector_path, \"rt\") as f:\n",
    "    emb = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "b60b0e62",
   "metadata": {
    "id": "b60b0e62"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lenght of words in embeddings 400000\n"
     ]
    }
   ],
   "source": [
    "# length of embeddings\n",
    "print('lenght of words in embeddings',len(emb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e27f3bfd",
   "metadata": {
    "id": "e27f3bfd"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the 0.418 0.24968 -0.41242 0.1217 0.34527 -0.044457 -0.49688 -0.17862 -0.00066023 -0.6566 0.27843 -0.14767 -0.55677 0.14658 -0.0095095 0.011658 0.10204 -0.12792 -0.8443 -0.12181 -0.016801 -0.33279 -0.1552 -0.23131 -0.19181 -1.8823 -0.76746 0.099051 -0.42125 -0.19526 4.0071 -0.18594 -0.52287 -0.31681 0.00059213 0.0074449 0.17778 -0.15897 0.012041 -0.054223 -0.29871 -0.15749 -0.34758 -0.045637 -0.44251 0.18785 0.0027849 -0.18411 -0.11514 -0.78581\\n'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check first record\n",
    "emb[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3d5698d0",
   "metadata": {
    "id": "3d5698d0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split the first record and check for vocabulary\n",
    "emb[0].split()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "89c86e32",
   "metadata": {
    "id": "89c86e32"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['0.418',\n",
       " '0.24968',\n",
       " '-0.41242',\n",
       " '0.1217',\n",
       " '0.34527',\n",
       " '-0.044457',\n",
       " '-0.49688',\n",
       " '-0.17862',\n",
       " '-0.00066023',\n",
       " '-0.6566',\n",
       " '0.27843',\n",
       " '-0.14767',\n",
       " '-0.55677',\n",
       " '0.14658',\n",
       " '-0.0095095',\n",
       " '0.011658',\n",
       " '0.10204',\n",
       " '-0.12792',\n",
       " '-0.8443',\n",
       " '-0.12181',\n",
       " '-0.016801',\n",
       " '-0.33279',\n",
       " '-0.1552',\n",
       " '-0.23131',\n",
       " '-0.19181',\n",
       " '-1.8823',\n",
       " '-0.76746',\n",
       " '0.099051',\n",
       " '-0.42125',\n",
       " '-0.19526',\n",
       " '4.0071',\n",
       " '-0.18594',\n",
       " '-0.52287',\n",
       " '-0.31681',\n",
       " '0.00059213',\n",
       " '0.0074449',\n",
       " '0.17778',\n",
       " '-0.15897',\n",
       " '0.012041',\n",
       " '-0.054223',\n",
       " '-0.29871',\n",
       " '-0.15749',\n",
       " '-0.34758',\n",
       " '-0.045637',\n",
       " '-0.44251',\n",
       " '0.18785',\n",
       " '0.0027849',\n",
       " '-0.18411',\n",
       " '-0.11514',\n",
       " '-0.78581']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split the first record and check for embeddings\n",
    "emb[0].split()[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "242f889d",
   "metadata": {
    "id": "242f889d"
   },
   "outputs": [],
   "source": [
    "#separating Embeddings for vocabulary\n",
    "vocabulary, embeddings = [], []\n",
    "\n",
    "for item in emb:\n",
    "    vocabulary.append(item.split()[0])\n",
    "    embeddings.append(item.split()[1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "8bfab44e",
   "metadata": {
    "id": "8bfab44e"
   },
   "outputs": [],
   "source": [
    "#Converting embeddings to numpy float array\n",
    "embeddings = np.array(embeddings, dtype=np.float32)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f867fce-6752-43c3-8270-a50b08bbe733",
   "metadata": {
    "id": "99dbd776"
   },
   "source": [
    "embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "977e10c1-bf08-40b2-a156-e142a4c483c3",
   "metadata": {
    "id": "85da0e52"
   },
   "outputs": [],
   "source": [
    "#padding embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bddbc45a-1488-4713-848a-927343d68e72",
   "metadata": {
    "id": "85da0e52"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Output/voc'"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocabulary[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "428314b9",
   "metadata": {
    "id": "428314b9"
   },
   "outputs": [],
   "source": [
    "embeddings = np.vstack([np.ones(50, dtype=np.float32), np.mean(embeddings, axis=0),\n",
    "                            embeddings])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "5cac06f0",
   "metadata": {
    "id": "5cac06f0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "400002 (400002, 50)\n"
     ]
    }
   ],
   "source": [
    "print(len(vocabulary), embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "b173a0ea",
   "metadata": {
    "id": "b173a0ea"
   },
   "outputs": [],
   "source": [
    "#saving to files\n",
    "save_file(embeddings_path, embeddings)\n",
    "save_file(vocabulary_path, vocabulary)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f58d0479",
   "metadata": {
    "id": "f58d0479"
   },
   "source": [
    "## To process text data\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c71c7dfc",
   "metadata": {
    "id": "c71c7dfc"
   },
   "outputs": [],
   "source": [
    "#read data\n",
    "data = pd.read_csv(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "216c4a4d",
   "metadata": {
    "id": "216c4a4d"
   },
   "outputs": [],
   "source": [
    "#drop empty rows\n",
    "data.dropna(subset=[text_col_name], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "64801543",
   "metadata": {
    "id": "64801543"
   },
   "outputs": [],
   "source": [
    "#replace duplicate labels\n",
    "data.replace({label_col: product_map}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "900db623",
   "metadata": {
    "id": "900db623"
   },
   "outputs": [],
   "source": [
    "#Encoding the label columns\n",
    "label_encoder = LabelEncoder()\n",
    "label_encoder.fit(data[label_col])\n",
    "labels = label_encoder.transform(data[label_col])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "edd80694",
   "metadata": {
    "id": "edd80694"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "52c367c8",
   "metadata": {
    "id": "52c367c8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['card', 'credit_report', 'debt_collection', 'loan',\n",
       "       'money_transfer', 'mortgage', 'others', 'savings_account',\n",
       "       'vehicle_loan'], dtype=object)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_encoder.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "f7ccce6f",
   "metadata": {
    "id": "f7ccce6f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1             vehicle_loan\n",
       "7            credit_report\n",
       "8            credit_report\n",
       "10           credit_report\n",
       "13           credit_report\n",
       "                ...       \n",
       "2326240               card\n",
       "2326241    debt_collection\n",
       "2326242           mortgage\n",
       "2326243      credit_report\n",
       "2326244      credit_report\n",
       "Name: Product, Length: 809343, dtype: object"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[label_col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "0544906f",
   "metadata": {
    "id": "0544906f"
   },
   "outputs": [],
   "source": [
    "#save labels to files\n",
    "save_file(labels_path, labels)\n",
    "save_file(label_encoder_path, label_encoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "06b2e29d",
   "metadata": {
    "id": "06b2e29d"
   },
   "outputs": [],
   "source": [
    "#processing text column\n",
    "input_text = data[text_col_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d28db548-196d-48ae-84ff-bf6540334bde",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_text = [i.lower() for i in tqdm(input_text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f43ed8f-2332-4ee2-a573-733e93f89bad",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_text = [re.sub(r\"[^\\w\\d'\\s]+\", \" \", i) for i in tqdm(input_text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a7f5198-c116-44fc-8f65-ab8584f1627b",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_text = [re.sub(\"\\d+\", \"\", i) for i in tqdm(input_text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2fcf5ea-7341-4023-a1f4-52bae3cd68ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_text = [re.sub(r'[x]{2,}', \"\", i) for i in tqdm(input_text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a39d0a0-86ce-41b7-ba96-5f6f648b2c39",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_text = [re.sub(' +', ' ', i) for i in tqdm(input_text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c20bf33-9350-4fb9-a57b-a57d00173ac3",
   "metadata": {},
   "outputs": [],
   "source": [
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "934505ad",
   "metadata": {
    "id": "934505ad"
   },
   "source": [
    "### Tokenize the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbe99b31-3dfa-4983-8415-c6b468a79ec5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Tokenize text\n",
    "tokens = [word_tokenize(t) for t in tqdm(input_text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f2a621a-ae0f-47b5-8bbe-c65933101737",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Take first 20 tokens\n",
    "tokens = [i[:20] if len(i) > 19 else ['<pad>'] * (20 - len(i)) + i for i in tqdm(tokens)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "288a4e01",
   "metadata": {
    "id": "288a4e01"
   },
   "outputs": [],
   "source": [
    "#Function definition to convert tokens to integer indices from vocabulary\n",
    "def token_index(tokens, vocabulary, missing='<unk>'):\n",
    "    \"\"\"\n",
    "    :param tokens: List of word tokens\n",
    "    :param vocabulary: All words in the embeddings\n",
    "    :param missing: Token for words not present in the vocabulary\n",
    "    :return: List of integers representing the word tokens\n",
    "    \"\"\"\n",
    "    idx_token = []\n",
    "    for text in tqdm(tokens):\n",
    "        idx_text = []\n",
    "        for token in text:\n",
    "            if token in vocabulary:\n",
    "                idx_text.append(vocabulary.index(token))\n",
    "            else:\n",
    "                idx_text.append(vocabulary.index(missing))\n",
    "        idx_token.append(idx_text)\n",
    "    return idx_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "618f32df",
   "metadata": {
    "id": "618f32df"
   },
   "outputs": [],
   "source": [
    "#vocabulary = \"Output/vocabulary.pkl\"\n",
    "#tokens = \"Output/tokens.pkl\"\n",
    "tokens = token_index(tokens, vocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "c51657a5",
   "metadata": {
    "id": "c51657a5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "e7fe16b6",
   "metadata": {
    "id": "e7fe16b6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'O'"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bc30185-2040-47d7-8d14-5de8864b00fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3843f11f-3a50-4b1d-bc20-4d1e7f0e8442",
   "metadata": {},
   "outputs": [],
   "source": [
    "vocabulary[tokens[0][0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "ab83d5fb",
   "metadata": {
    "id": "ab83d5fb"
   },
   "outputs": [],
   "source": [
    "#save tokens\n",
    "save_file(tokens_path, tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8936a733",
   "metadata": {
    "id": "8936a733"
   },
   "outputs": [],
   "source": [
    "#Create Pytorch Dataset\n",
    "class TextDataset(torch.utils.data.Dataset):\n",
    "\n",
    "    def __init__(self, tokens, embeddings, labels):\n",
    "        \"\"\"\n",
    "        :param tokens: List of word tokens\n",
    "        :param embeddings: Word embeddings (from glove)\n",
    "        :param labels: List of labels\n",
    "        \"\"\"\n",
    "        self.tokens = tokens\n",
    "        self.embeddings = embeddings\n",
    "        self.labels = labels\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.tokens)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.labels[idx], self.embeddings[self.tokens[idx], :]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f1c8577",
   "metadata": {
    "id": "8f1c8577"
   },
   "source": [
    "## Create Models\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f4953da5",
   "metadata": {
    "id": "f4953da5"
   },
   "outputs": [],
   "source": [
    "#Build RNN\n",
    "class RNNNetwork(torch.nn.Module):\n",
    "\n",
    "    def __init__(self, input_size, hidden_size, num_classes):\n",
    "        \"\"\"\n",
    "        :param input_size: Size of embedding\n",
    "        :param hidden_size: Hidden vector size\n",
    "        :param num_classes: Number of classes in the dataset\n",
    "        \"\"\"\n",
    "        super(RNNNetwork, self).__init__()\n",
    "        # RNN Layer\n",
    "        self.rnn = torch.nn.RNN(input_size=input_size,\n",
    "                                hidden_size=hidden_size,\n",
    "                                batch_first=True)\n",
    "        # Linear Layer\n",
    "        self.linear = torch.nn.Linear(hidden_size, num_classes)\n",
    "\n",
    "    def forward(self, input_data):\n",
    "        _, hidden = self.rnn(input_data)\n",
    "        output = self.linear(hidden)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a1383a39",
   "metadata": {
    "id": "a1383a39"
   },
   "outputs": [],
   "source": [
    "#Build LSTM\n",
    "class LSTMNetwork(torch.nn.Module):\n",
    "\n",
    "    def __init__(self, input_size, hidden_size, num_classes):\n",
    "        \"\"\"\n",
    "        :param input_size: Size of embedding\n",
    "        :param hidden_size: Hidden vector size\n",
    "        :param num_classes: Number of classes in the dataset\n",
    "        \"\"\"\n",
    "        super(LSTMNetwork, self).__init__()\n",
    "        # LSTM Layer\n",
    "        self.rnn = torch.nn.LSTM(input_size=input_size,\n",
    "                                 hidden_size=hidden_size,\n",
    "                                 batch_first=True)\n",
    "        # Linear Layer\n",
    "        self.linear = torch.nn.Linear(hidden_size, num_classes)\n",
    "\n",
    "    def forward(self, input_data):\n",
    "        _, (hidden, _) = self.rnn(input_data)\n",
    "        output = self.linear(hidden[-1])\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f289dabc",
   "metadata": {
    "id": "f289dabc"
   },
   "outputs": [],
   "source": [
    "#Train function definition\n",
    "def train(train_loader, valid_loader, model, criterion, optimizer, device,\n",
    "          num_epochs, model_path):\n",
    "    \"\"\"\n",
    "    Function to train the model\n",
    "    :param train_loader: Data loader for train dataset\n",
    "    :param valid_loader: Data loader for validation dataset\n",
    "    :param model: Model object\n",
    "    :param criterion: Loss function\n",
    "    :param optimizer: Optimizer\n",
    "    :param device: CUDA or CPU\n",
    "    :param num_epochs: Number of epochs\n",
    "    :param model_path: Path to save the model\n",
    "    \"\"\"\n",
    "    best_loss = 1e8\n",
    "    for i in range(num_epochs):\n",
    "        print(f\"Epoch {i+1} of {num_epochs}\")\n",
    "        valid_loss, train_loss = [], []\n",
    "        model.train()\n",
    "        # Train loop\n",
    "        for batch_labels, batch_data in tqdm(train_loader):\n",
    "            # Move data to GPU if available\n",
    "            batch_labels = batch_labels.to(device)\n",
    "            batch_labels = batch_labels.type(torch.LongTensor)\n",
    "            batch_data = batch_data.to(device)\n",
    "            # Forward pass\n",
    "            batch_output = model(batch_data)\n",
    "            batch_output = torch.squeeze(batch_output)\n",
    "            # Calculate loss\n",
    "            loss = criterion(batch_output, batch_labels)\n",
    "            train_loss.append(loss.item())\n",
    "            optimizer.zero_grad()\n",
    "            # Backward pass\n",
    "            loss.backward()\n",
    "            # Gradient update step\n",
    "            optimizer.step()\n",
    "        model.eval()\n",
    "        # Validation loop\n",
    "        for batch_labels, batch_data in tqdm(valid_loader):\n",
    "            # Move data to GPU if available\n",
    "            batch_labels = batch_labels.to(device)\n",
    "            batch_labels = batch_labels.type(torch.LongTensor)\n",
    "            batch_data = batch_data.to(device)\n",
    "            # Forward pass\n",
    "            batch_output = model(batch_data)\n",
    "            batch_output = torch.squeeze(batch_output)\n",
    "            # Calculate loss\n",
    "            loss = criterion(batch_output, batch_labels)\n",
    "            valid_loss.append(loss.item())\n",
    "        t_loss = np.mean(train_loss)\n",
    "        v_loss = np.mean(valid_loss)\n",
    "        print(f\"Train Loss: {t_loss}, Validation Loss: {v_loss}\")\n",
    "        if v_loss < best_loss:\n",
    "            best_loss = v_loss\n",
    "            # Save model if validation loss improves\n",
    "            torch.save(model.state_dict(), model_path)\n",
    "        print(f\"Best Validation Loss: {best_loss}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6b9e405f",
   "metadata": {
    "id": "6b9e405f"
   },
   "outputs": [],
   "source": [
    "#Test function definition\n",
    "def test(test_loader, model, criterion, device):\n",
    "    \"\"\"\n",
    "    Function to test the model\n",
    "    :param test_loader: Data loader for test dataset\n",
    "    :param model: Model object\n",
    "    :param criterion: Loss function\n",
    "    :param device: CUDA or CPU\n",
    "    \"\"\"\n",
    "    model.eval()\n",
    "    test_loss = []\n",
    "    test_accu = []\n",
    "    for batch_labels, batch_data in tqdm(test_loader):\n",
    "        # Move data to device\n",
    "        batch_labels = batch_labels.to(device)\n",
    "        batch_labels = batch_labels.type(torch.LongTensor)\n",
    "        batch_data = batch_data.to(device)\n",
    "        # Forward pass\n",
    "        batch_output = model(batch_data)\n",
    "        batch_output = torch.squeeze(batch_output)\n",
    "        # Calculate loss\n",
    "        loss = criterion(batch_output, batch_labels)\n",
    "        test_loss.append(loss.item())\n",
    "        batch_preds = torch.argmax(batch_output, axis=1)\n",
    "        # Move predictions to CPU\n",
    "        if torch.cuda.is_available():\n",
    "            batch_labels = batch_labels.cpu()\n",
    "            batch_preds = batch_preds.cpu()\n",
    "        # Compute accuracy\n",
    "        test_accu.append(accuracy_score(batch_labels.detach().numpy(),\n",
    "                                        batch_preds.detach().numpy()))\n",
    "    test_loss = np.mean(test_loss)\n",
    "    test_accu = np.mean(test_accu)\n",
    "    print(f\"Test Loss: {test_loss}, Test Accuracy: {test_accu}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4163f818",
   "metadata": {
    "id": "4163f818"
   },
   "source": [
    "## Train RNN Model\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c9aaae86",
   "metadata": {
    "id": "c9aaae86"
   },
   "outputs": [],
   "source": [
    "#Load files\n",
    "tokens = load_file(tokens_path)\n",
    "labels = load_file(labels_path)\n",
    "vocabulary = load_file(vocabulary_path)\n",
    "embeddings = load_file(embeddings_path)\n",
    "label_encoder = load_file(label_encoder_path)\n",
    "num_classes = len(label_encoder.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7916b320",
   "metadata": {
    "id": "7916b320"
   },
   "outputs": [],
   "source": [
    "#Split data into train, validation and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(tokens, labels,\n",
    "                                                    test_size=0.2)\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train,\n",
    "                                                      test_size=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "af22498f",
   "metadata": {
    "id": "af22498f"
   },
   "outputs": [],
   "source": [
    "#Create PyTorch datasets\n",
    "train_dataset = TextDataset(X_train, embeddings, y_train)\n",
    "valid_dataset = TextDataset(X_valid, embeddings, y_valid)\n",
    "test_dataset = TextDataset(X_test, embeddings, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "030610ea",
   "metadata": {
    "id": "030610ea"
   },
   "source": [
    "### Create data loaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a0dcc6dd",
   "metadata": {
    "id": "a0dcc6dd"
   },
   "outputs": [],
   "source": [
    "#Create data loaders\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=16,\n",
    "                                           shuffle=True, drop_last=True)\n",
    "valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=16)\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7aaacda1",
   "metadata": {
    "id": "7aaacda1"
   },
   "outputs": [],
   "source": [
    "#rnn model object\n",
    "model = RNNNetwork(input_size, hidden_size, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ca465cd6",
   "metadata": {
    "id": "ca465cd6"
   },
   "outputs": [],
   "source": [
    "#gpu set up\n",
    "if torch.cuda.is_available():\n",
    "    model = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d4becc10",
   "metadata": {
    "id": "d4becc10"
   },
   "outputs": [],
   "source": [
    "#loss function \n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f188838e-365b-4fec-b4aa-85384c6347c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#train RNN\n",
    "train(train_loader, valid_loader, model, criterion, optimizer,\n",
    "      device, num_epochs, rnn_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5c6e465-24cb-4344-9f5a-3d8d0c3c4d1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#testing RNN\n",
    "test(test_loader, model, criterion, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "a5e3ceb0",
   "metadata": {
    "id": "a5e3ceb0"
   },
   "outputs": [],
   "source": [
    "#LSTM model object\n",
    "model = LSTMNetwork(input_size, hidden_size, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "92a8e560",
   "metadata": {
    "id": "92a8e560"
   },
   "outputs": [],
   "source": [
    "#gpu set up\n",
    "if torch.cuda.is_available():\n",
    "    model = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc0de382-3d89-4c54-857c-5043295b0394",
   "metadata": {},
   "outputs": [],
   "source": [
    "#loss function\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55a10fde-63fb-4578-a5b3-cd6653728b42",
   "metadata": {},
   "outputs": [],
   "source": [
    "#train LSTM\n",
    "train(train_loader, valid_loader, model, criterion, optimizer,\n",
    "      device, num_epochs, rnn_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7826134-5140-4d17-82db-424d14004d28",
   "metadata": {},
   "outputs": [],
   "source": [
    "#testing LSTM\n",
    "test(test_loader, model, criterion, device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a74eb71",
   "metadata": {
    "id": "8a74eb71"
   },
   "source": [
    "## Predict on input text\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "ff13ab78",
   "metadata": {
    "id": "ff13ab78"
   },
   "outputs": [],
   "source": [
    "input_text = '''I am a victim of Identity Theft & currently have an Experian account that \n",
    "I can view my Experian Credit Report and getting notified when there is activity on \n",
    "my Experian Credit Report. For the past 3 days I've spent a total of approximately 9 \n",
    "hours on the phone with Experian. Every time I call I get transferred repeatedly and \n",
    "then my last transfer and automated message states to press 1 and leave a message and \n",
    "someone would call me. Every time I press 1 I get an automatic message stating than you \n",
    "before I even leave a message and get disconnected. I call Experian again, explain what \n",
    "is happening and the process begins again with the same end result. I was trying to have \n",
    "this issue attended and resolved informally but I give up after 9 hours. There are hard \n",
    "hit inquiries on my Experian Credit Report that are fraud, I didn't authorize, or recall \n",
    "and I respectfully request that Experian remove the hard hit inquiries immediately just \n",
    "like they've done in the past when I was able to speak to a live Experian representative \n",
    "in the United States. The following are the hard hit inquiries : \n",
    "\n",
    "BK OF XXXX XX/XX/XXXX \n",
    "XXXX XXXX XXXX  XX/XX/XXXX XXXX  XXXX XXXX  XX/XX/XXXX XXXX  XX/XX/XXXX XXXX  XXXX \n",
    "XX/XX/XXXX'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "1dded0d3",
   "metadata": {
    "id": "1dded0d3"
   },
   "outputs": [],
   "source": [
    "#process text input\n",
    "input_text = input_text.lower()\n",
    "input_text = re.sub(r\"[^\\w\\d'\\s]+\", \" \", input_text)\n",
    "input_text = re.sub(\"\\d+\", \"\", input_text)\n",
    "input_text = re.sub(r'[x]{2,}', \"\", input_text)\n",
    "input_text = re.sub(' +', ' ', input_text)\n",
    "tokens = word_tokenize(input_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "7ba95fe0",
   "metadata": {
    "id": "7ba95fe0"
   },
   "outputs": [],
   "source": [
    "tokens = ['<pad>']*(20-len(tokens))+tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "821fb5de",
   "metadata": {
    "id": "821fb5de"
   },
   "outputs": [],
   "source": [
    "#tokenize input text\n",
    "idx_token = []\n",
    "for token in tokens:\n",
    "    if token in vocabulary:\n",
    "        idx_token.append(vocabulary.index(token))\n",
    "    else:\n",
    "        idx_token.append(vocabulary.index('<unk>'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "1d6cc01c",
   "metadata": {
    "id": "1d6cc01c"
   },
   "outputs": [],
   "source": [
    "#get embedding tokens\n",
    "token_emb = embeddings[idx_token,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "780d42e4",
   "metadata": {
    "id": "780d42e4"
   },
   "outputs": [],
   "source": [
    "#convert to torch tensor\n",
    "inp = torch.from_numpy(token_emb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "f15e636d",
   "metadata": {
    "id": "f15e636d"
   },
   "outputs": [],
   "source": [
    "inp = inp.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "5c3865b5",
   "metadata": {
    "id": "5c3865b5"
   },
   "outputs": [],
   "source": [
    "#create batch\n",
    "inp = torch.unsqueeze(inp, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "92a17e27",
   "metadata": {
    "id": "92a17e27"
   },
   "outputs": [],
   "source": [
    "#load encoder for label\n",
    "label_encoder = load_file(label_encoder_path)\n",
    "num_classes = len(label_encoder.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "07dd7955",
   "metadata": {
    "id": "07dd7955"
   },
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cdbe841b",
   "metadata": {
    "id": "cdbe841b"
   },
   "source": [
    "### RNN prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e16fd00-edf2-4c59-b8bb-50b9b63e16aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create model object\n",
    "model = RNNNetwork(input_size, hidden_size, num_classes)\n",
    "\n",
    "# Load trained weights\n",
    "model.load_state_dict(torch.load(rnn_model_path))\n",
    "\n",
    "# Move the model to GPU if available\n",
    "if torch.cuda.is_available():\n",
    "    model = model.cuda()\n",
    "    \n",
    "# Forward pass\n",
    "out = torch.squeeze(model(inp))\n",
    "\n",
    "# Find predicted class\n",
    "prediction = label_encoder.classes_[torch.argmax(out)]\n",
    "print(f\"Predicted  Class: {prediction}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1426d72f",
   "metadata": {
    "id": "1426d72f"
   },
   "source": [
    "### LSTM prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "290e2616",
   "metadata": {
    "id": "290e2616"
   },
   "outputs": [],
   "source": [
    "# Create model object\n",
    "model = LSTMNetwork(input_size, hidden_size, num_classes)\n",
    "\n",
    "# Load trained weights\n",
    "model.load_state_dict(torch.load(lstm_model_path))\n",
    "\n",
    "# Move the model to GPU if available\n",
    "if torch.cuda.is_available():\n",
    "    model = model.cuda()\n",
    "    \n",
    "# Forward pass\n",
    "out = torch.squeeze(model(inp))\n",
    "\n",
    "# Find predicted class\n",
    "prediction = label_encoder.classes_[torch.argmax(out)]\n",
    "print(f\"Predicted  Class: {prediction}\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "99dbd776",
    "42b3b07a",
    "f567f06f",
    "86dffa9a",
    "a70b83e2",
    "6b1714dc",
    "fc16fc63",
    "e2fd241b",
    "54188fc7",
    "03cce62f",
    "7fa2608b",
    "6dea776b",
    "934505ad",
    "5c312593",
    "be6561df",
    "ab9c8712",
    "8b77dedc",
    "550fc77e",
    "e752fd29",
    "2ad46f7e",
    "171cc1be",
    "030610ea",
    "af02c4c2",
    "7c88a3f8",
    "1deebde4",
    "7f90ad48",
    "678d4e28",
    "d09d91b1",
    "82fbbab5",
    "540cb058",
    "de584626",
    "e91e281c",
    "ebed4640",
    "e2cc919d",
    "cdbe841b",
    "1426d72f"
   ],
   "name": "RNN.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}