GLove+Bilstm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data import TwitterP\n",
    "import numpy as np\n",
    "from tqdm import trange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data has apparently already been downloaded and unpacked.\n"
     ]
    }
   ],
   "source": [
    "trainT, validationT, testT = TwitterP.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=trainT.append(validationT)\n",
    "df=df.append(testT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5459/5459 [01:04<00:00, 84.73it/s] \n"
     ]
    }
   ],
   "source": [
    "Negative_tweets=[]\n",
    "Positive_tweets=[]\n",
    "n_private=0\n",
    "for i in trange(1,len(df)):\n",
    "    timeline = TwitterP.get_username_profile(df['username'].iloc[i])\n",
    "    if df[\"Positive\"][i]==1:\n",
    "        if (timeline.empty):\n",
    "            n_private+=1\n",
    "        else:\n",
    "            Positive_tweets=np.append(Positive_tweets,timeline.tweet)\n",
    "    if df[\"Negative\"][i]==1:\n",
    "        if (timeline.empty):\n",
    "            n_private+=1\n",
    "        else:\n",
    "            Negative_tweets=np.append(Negative_tweets,timeline.tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "PL = [1] * len(Positive_tweets) \n",
    "NL = [0] * len(Negative_tweets) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(223014, 187360)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Positive_tweets), len(Negative_tweets) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=np.append(Positive_tweets[10001:15000],Negative_tweets[10001:15000])\n",
    "y=np.append(PL[10001:15000],NL[10001:15000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we shuffle our sorted data, to make it random first\n",
    "indices = np.arange(x.shape[0])\n",
    "np.random.shuffle(indices)\n",
    "x = x[indices]\n",
    "y = y[indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "d = {'Tweet':x,'label':y}\n",
    "df = pd.DataFrame(d)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('test_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=np.append(Positive_tweets[0:10000],Negative_tweets[0:10000])\n",
    "y=np.append(PL[0:10000],NL[0:10000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "# NumPy is a very widely-used library in Python for mathematical operations.\n",
    "# It supports many complex mathematical functions and can work on large arrays and matrices.\n",
    "\n",
    "embeddings_index = {}\n",
    "with open(os.path.join('glove.6B.100d.txt')) as f:\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        coefs = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = coefs\n",
    "\n",
    "print(\"Found {} word vectors.\".format(len(embeddings_index)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.067077,  0.34271 ,  0.31389 ,  0.45675 , -1.0441  , -0.30359 ,\n",
       "       -0.72071 ,  0.024742,  1.0052  , -0.30277 , -0.61816 ,  0.1482  ,\n",
       "        0.55835 , -0.74295 ,  0.10714 ,  0.22635 , -0.99951 , -0.54721 ,\n",
       "        0.73511 ,  0.40552 ,  0.71583 ,  0.37093 ,  0.017737,  0.56344 ,\n",
       "       -0.086712, -0.19697 ,  0.16806 , -0.67181 , -0.16957 ,  1.233   ,\n",
       "        0.40425 ,  0.39286 ,  0.09856 , -0.39293 , -0.56219 ,  0.25364 ,\n",
       "        0.19299 ,  0.17292 ,  0.40808 , -0.32074 , -0.082906, -0.59409 ,\n",
       "        0.10728 , -0.28854 ,  0.11224 ,  0.30465 ,  0.75957 ,  0.91237 ,\n",
       "        0.54494 , -0.89268 , -0.66143 , -0.24641 ,  0.053331,  0.20658 ,\n",
       "       -0.021297, -0.97389 ,  0.55199 , -0.34674 ,  0.58749 , -0.023662,\n",
       "        0.55734 ,  0.55996 , -0.63015 , -0.15025 , -0.15092 ,  0.11918 ,\n",
       "        1.0616  , -0.29991 ,  0.38754 , -0.48744 ,  0.29452 , -0.42888 ,\n",
       "       -0.031931,  0.49391 ,  0.31056 ,  0.18853 ,  0.28281 ,  0.1583  ,\n",
       "        0.046716, -0.66058 ,  0.055853,  0.95369 , -1.4047  ,  0.47653 ,\n",
       "       -0.98774 , -0.86063 , -0.29713 , -0.17217 , -1.1155  , -0.60824 ,\n",
       "       -0.37497 ,  0.23848 ,  0.46777 ,  0.099564, -0.41541 ,  0.97339 ,\n",
       "        0.28276 ,  0.12338 , -0.323   , -0.69511 ], dtype=float32)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# to see what word embeddings look like\n",
    "embeddings_index[\"depressed\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.layers import Dense, Input, LSTM, Embedding, Bidirectional, Dropout\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.initializers import Constant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setting some parameters\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 200 # determines how many words of every tweet will be processed\n",
    "# if the tweet length is smaller than this, the tweet will be padded until it has a length equivalent to this\n",
    "\n",
    "MAX_NUM_WORDS = 70000 # how many unique words in the training text to be tokenised\n",
    "EMBEDDING_DIM = 100 # number of dimensions in the word embeddings\n",
    "TRAINING_SPLIT = 0.8 # proportion of data to be used to training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59179 distinct words in original text\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer(num_words=MAX_NUM_WORDS)\n",
    "tokenizer.fit_on_texts(x)\n",
    "sequences = tokenizer.texts_to_sequences(x)\n",
    "word_index = tokenizer.word_index\n",
    "\n",
    "print(\"{} distinct words in original text\".format(len(word_index)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD4CAYAAADmWv3KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXgV5dk/8O/NoqgIgiwii4Bg3epKKSq4UVe08Fp9i/1pacUfWhXX1mKtr7ZvW1EqCtQNl4K4Qd1AUTAEUERBwr6FLIAQCEnYEgIh6/P+ceaEk5OZc2Y9c2by/VwXF8lkzsx95sy555lnnkWUUiAiovBp5ncARETkDSZ4IqKQYoInIgopJngiopBigiciCqkWfgcAAB06dFA9e/b0OwwiokBZvnz5bqVUR6O/p0WC79mzJ7KysvwOg4goUETkh0R/ZxUNEVFIMcETEYUUEzwRUUglTfAi8qaIFIvIuphl7UUkQ0Rytf/bxfztMRHJE5FNInKNV4ETEVFiZkrwUwBcG7dsDIBMpVRfAJna7xCRMwEMB3CW9pqXRKS5a9ESEZFpSRO8UuprAHvjFg8FMFX7eSqAYTHL31dKVSqltgDIA9DfpViJiMgCu3XwnZVShQCg/d9JW94VwPaY9Qq0ZY2IyCgRyRKRrJKSEpthEBGREbcfsorOMt3xiJVSk5VS/ZRS/Tp2NGynTw7NXlOI/Yeq/A6DiHxgN8EXiUgXAND+L9aWFwDoHrNeNwA77YdHTuzcX4F7312Be95Z4XcoROQDuwl+FoAR2s8jAMyMWT5cRI4WkV4A+gL43lmIZFdlTR2ASKInoqYn6VAFIvIegMsBdBCRAgBPAhgLYIaIjASwDcAtAKCUWi8iMwBsAFAD4F6lVK1HsRMRUQJJE7xS6laDPw02WP/vAP7uJCgiInKOPVmJiEKKCZ6IKKSY4ImIQooJvgnQ7YhARKHHBB9ier3OiKjpYIInIgopJngiopBigqcmqbC0AnPWFfodBpGnmOCpSbr55e9w99sco4fCjQmemqQdHJ+HmgAmeCKikGKCbwIUG8ITNUlM8CEmbAhP1KQxwRMRhRQTPBFRSDHBExGFFBM8EVFIMcETEYUUE3wToDhgMFGTxAQfYsIBg4maNCZ4IqKQYoInIgopJngiopBigiciCikmeCKikGKCbwI4miRR08QEH2IcTZKoaWOCJyIKKSZ4IqKQYoInIgopJngiopBylOBF5CERWS8i60TkPRFpJSLtRSRDRHK1/9u5FSwREZlnO8GLSFcA9wPop5Q6G0BzAMMBjAGQqZTqCyBT+52IiFLMaRVNCwDHiEgLAMcC2AlgKICp2t+nAhjmcB/kENvBEzVNthO8UmoHgH8C2AagEECpUupLAJ2VUoXaOoUAOum9XkRGiUiWiGSVlJTYDYOIiAw4qaJph0hpvReAkwEcJyK3mX29UmqyUqqfUqpfx44d7YZBREQGnFTR/AzAFqVUiVKqGsBHAC4GUCQiXQBA+7/YeZhERGSVkwS/DcAAETlWRATAYAAbAcwCMEJbZwSAmc5CJCIiO1rYfaFSaqmIfABgBYAaACsBTAbQGsAMERmJyEXgFjcCJSIia2wneABQSj0J4Mm4xZWIlOaJiMhH7MlKRBRSTPAhduBwDQBAsSE8UZPEBB9if/5kLQBgZ+lhnyMhIj8wwYfY/kPVfodARD5igiciCikmeCKikGKCJyIKKSZ4amT/oSq8vmgzW9/4rPRQNT8HcsRRRycKpz9+uAZz1xfhvO4noF/P9n6H02Q99vEafL52F87u2hYDep/odzgUQCzBh5nYe1lpRaT1TVVtnYvBkFVlFZF+DNX8HMgmJngyxpoBokBjgqdGxG7Rn4jSChN8iDFNJ8cHmBRmTPBkiKmPKNiY4KkRYdGfKBSY4ImIQooJPsTEYVGc1dNEwcYET42wioaSUUrhjW+2oLiMQ1GnMyZ4MqT4mJUMbN59EP/72Qbc/fZyv0OhBJjg41TV1GFiZi4OV9f6HYpvzLSDr6tTeHFBHsoOc8x5r6VjVVltXSSo6KxhlJ6Y4OO8veQHjM/IweSvN/sdSlrLzC7GuLmb8NdPN/gdSmgFoaosDa89FIMJPk6FVnKvCEEJ3ml+SFRyrKqJjI9yqIoluKYoANceAhM86QhCyZHSA3sCp7cmmeALSyvwxjdb/A7Dc0zU5IUDh6sxaX6e32GQCU1yPPiRU7KwobAM15zVGd3aHau7ThgKJk7fg5mXh+E4kTVjv8jGrNU7/Q6DTGiSJfhoyw+95MRSrzlBPE7f5u3GiDe/x7Y9hxKuN23JD9iy+2CKonLXJyt3YG1Bqaf7OFR15PkUr+/prUmW4BNhifSIsNWv/ur1pQCAX07+znAdpRSe+GQd2h7TEqufvDpVobnmwemrAABbxw7xORJKB02yBG+GnRJqRVUtXpiXU9/CJKicDnGQ7soqkrfdLzWxDoFF+DTHBO+ilxbm4YV5uXjv+21+h0JEFK4Ev+9gFSZl5qKuzrhYoZRCwb4K3b+t21GKj1fu0Nazvv8KrW4y64d91l+chswcgkW5uz2Pww15xeWGf1NKYfLX+Sgs1T8vgmh8Rg4qa7zvy7E5gM8qpizekvQ5TDKz1xQia+telyLyTqgS/OOfrMVzGTn4Nn+P4Trrd5YZ/u2GSd8kTARmfZomLQzs1rSYeVl0nfLKYHR0+tn4r+p/jq+C+mHPIfzj82yMeiu9xlVx8ghkYmYupize6losYVFaUY2nPt2AW19b4mg79767Aje/YvwsJ104SvAicoKIfCAi2SKyUUQuEpH2IpIhIrna/+3cCjaZ8spIiaWmzrgOvDZB6d4ps1vef6gKE5PcabjBztyq5ZU1+CqnxINo0letlkkP+nSx2lV6GK9+le/6Q223ngUppfDqV8G6w8krLse0JT80Wh5tHRWUgolTTkvwEwDMUUqdDuBcABsBjAGQqZTqCyBT+71J2LTrgKn1Hv9kHcZn5GBRXvpVb/xz7qYjv/ABWkrc9fZyPP1FdqPqjnR51r1t7yE8/UU2/v9bWX6HYtqQiYvwxCfrGi2/+eVvAbh38Ut3thO8iLQBcCmANwBAKVWllNoPYCiAqdpqUwEMcxqkWclKQOWVNXh+Xo5n+0905xDrkFZ6qDW5fip9m59+Fx27yitrMD4jBzW1DY+zUd70qz65vL5fhqr//+WF+Sgs9X6s9UNVNRj/5ab6hLdzf0Wju4noXe9B7Q45Ta47CVVq72f8l5sa3JnVaO8lDGNNmeGkHXxvACUA/i0i5wJYDuABAJ2VUoUAoJQqFJFOei8WkVEARgFAjx49HIShu23d5ePmZGPhpqZV/WBVTpHzZxDp4p9zN2HKt1vRo71+b+V0lV9yEM/MyU7JviZm5uGVr/LRsU0r3D7gFIycmoWNhWW4/sdd0D1gx03PxPl5OFhViyduONPvUHzhpIqmBYALALyslDofwEFYqI5RSk1WSvVTSvXr2LGjgzDMydq6FzOyCnT/NvXbra48XDXLTC/JwtIKvLzQ/XpZKxJN+BF7Db32ha89j3P19v24a1oWFluo1oqO6V9da+9OqaqmDs99uQmHqmowa/XORq0mPl9biO9iHuhv2FmG911oIpvKzzx6jFZv34/py7bVl3brtBiqa+swPsO7u95UaMpzOzgpwRcAKFBKLdV+/wCRBF8kIl200nsXAMVOg3RDoifeT85aj+OOau54H2a/l1u1Jlo1tcYvuGvacqwpKMU1Z3VG746tHcfmpexdB7Bq+36c38O75+lDX1wMAJi7vsjTXpqxn+GMrO2YND8PVbV1ePWryPwAsfu+550VDZZdP3ERAGB4f2t3pPFngd2699JDDTtnWdnOB8sL8MHygkZ3O5+s3IHP1hTaC4h8Z7sEr5TaBWC7iPxIWzQYwAYAswCM0JaNADDTUYQmvb5oc32bbCvfj+17I8n2YFXiq3zBvkN45at8w78rpbB0i7V2sYl6jJbXl6Qiv+8ur8SEebm2S3efry1MWL8+ZfGWRncxVnaVaNX52UVYkN34Ol9TGykh22nRUFVTh3/O3WRrPPoDJvcXrZeurD5yBxDdX8mBSsv71XOwsgabS6J3dBL3vzV2plics25Xwr9Xxdz9BGXoiuUO+6EUHziMSZn2v2vpxOlYNKMBvCMiRwHYDOC3iFw0ZojISADbANzicB9JHa6uxd9mb7T12hFvfq+7PP7LcseUZcgpKsfPzz0ZJ59wTKP180u8fUD36AdrMD+7GAN6t8dPe59o6jWx14/40ma8pz7dgGNa2r+LKU8wddsdU7J09/3Jqp2YND8PZRXV+MvQsw1fr3eLPX3ZNvxrQR5qlcIfrz3dZtTm7D9UVf/zpPl5+OO1p+PhGatc2fbEzNxGy+yW4O08P9mVZNJsvaa26Z72fqG1lImyGu/D01fjm7zdGHRaR5zX/QT3AvOBo2aSSqlVWj36OUqpYUqpfUqpPUqpwUqpvtr/nnX3ipYAncwLabY0t0+7/TVqK12R5A5AT8LvcdxZGS051npYqkjUsuBwdS3Gzc2OSbYNo28mgqqaOoybm21Yqi4ua1gyitaNV9bUYWNhGd5Z2rjdspFKndK1kWSHzKhns56XF+ZjT3mlaz149S5edlupGB33yV/nm+65uW1vw/W8aKo57butyCky16TYjNcXbcZWGy2gjJ7NHIx+1zzup5IKge7JOmt1pAT4rActDuJLLtFb8n8t0J/o4N/fWp9AJNGXR5lYx2uxiXHKt1vx4oJ8vGYwV61IpFT94oJ8TMzUP0aj31uJ5zJydHsTXzdhER7/uHG7ZSfMHrvf/lv/Lu7Idhpu6LGP1toNyRS7g729vqjhOSgi2FNeiX98no3b3lhq8Koksdh6VWJPzFyPq5//2pVtHayswd9mb8R/v2q9V+nMVenR49xLgU7w0Stw/EMgEWBjYRne1nqyRcd/t8Kovt2og0SlwfK6OoXnM3Kwp7xxna2Z77HZL1he8QFMWWztIvOfrO14PiMH87OLkq4bfd/PZeTgyZmNE7HgyDEwOkYrtkXqRt9fZtzSJL5uu7q2rmHnK038hBPFBw7bekax92AVkhXU4j8nO22oK6oid0CVNbX4eGVBwnFM4uvFD5m8O9Q77tH3ZvVZhV7VTLRxQDq1g49+dIl6IRvFG98/IowCneCj9L5w101YhD9rPdke/c8a74MwSBJLtuzBhMxc/PHDxjEkGkrAaqK6cdJiPPXpBkuv+cMHazAhM7e+jjyR2HCmfqdTlRLzVowe9lVrrYbeXmKc4H//n9UNfv945Q68rjO94pq4SS0emr4Kz8/LwWqLk138Wae3Y7xmLtxGvbQwDy8uyMc7S7bhoemrdVt1RXcT3wb+5YX+TY+XLr1pnQh+RYt9oUjwiewqPYw56/VbCoybu8l2G2mzovV4h3XqimvqFMbNzcaBBHcY8bfruUXlWouXA/h3TIndbKlyd3klxmfkuD4OjkDqx1k3s+2imId7sReP+HFwzH4+0VJuTtGB+jbdZnwRV1qu0tlffI6zU/+u1ya/55jZDe4elAIm6Tx0Nbo7TAU74xl5rexwNcbNzW5UAi+vrNGtrl2UW5K0tdCW3QcNqx8BYMf+Cry4IM+w4PVt/m58tib9qnxCOaNT7Ek5+r0Vhuu5Oq+kje/BrNU78enqnSirqMH/DmvYisQoRT45az0AoFXLZjhcXYffXtLL0j7/9NFafLmhCAN6tbcecIz4kp1IpNcgAMzfVIy/JHn96PdW4qbzuzqKIVb0ovLoB0fulG7t393ydt7WGaDKzVJs/LaWxVTV5BYdwHNOOhUleqZj8XpefxeWfvkdz3yRjXeWbsOpHVvj6rNOql8e7bkcb/veCtz99vJGrbhiP4tfvvodig9U4taf6vdhGPVWFtbvjPTw7dXhuEZ//9VrkWccN5xzso135J3Ql+D1Ss5mbdhZhkvGzq9vK59Qki9QeWUNnpmT3aCetDpJnTVg/P0yel9TFm9JOOhZtIXAYRNjhWdsOFI3n6yN9b/mH6lGqE3QgSsqtvXI9KztSdePZXagqGztOFhpHx5f9QPAsAe0FUYJti52zBcLWXjL7oOY/LVxv4wouxenaJ8OvZf7XeURvaOpibtTtHqnE3seFWvPfvKKy7Fy2/765Su27cOMZdvrW8nVxX1GZYerG9w1zFhm7Vz2WugTvBPXT1yEHfsrDNvKW7Fq+368vDAfHyw3lyzstoZ86tMNuG5CpIWCXmuMxXmRrvVVNcl3YCXxfpPCkTE/WnHkGEbfot67if2imuXqXZ0Ot6o8hk/+Dv/4PLtBA4L4LTu584i9E4qXymE9zLLzddF7FjRM6zEdddNL3+JRnednUePmbMJLC49caBOt64dAJ3ijL4vbbcUbzCLfoM400kIme1cZZq811507tm7YVCsaG19SM9XrXj978FJ1itonJ6u3NWvdjlK8pVX97K+oSrK2PqUUJszLrR+T/VBl8juwF+blorjMWY9bvUJCfMnZjDnrdun2ZnbiuS831XdCsxNTtYXRXKMjja6KKzRYmTVrd3klnvtyk+fzQMQKdII3+oBiTyQ36k+NbvFXF5RiQmYurn1hkelt6bXy8KOlgtVOHG4NW2CVnw/5vndpSrYbJn1TXx3w4oKG1Spm319OUTmen5eD371t/EwpXlVNXX2PW7ufiVtH/+63l+O3U5a5sq1oTEVllfXt6e2M727nvT0S18rLijEfrsGk+XlYstl4xjm3BTrBG4mvJ3NKb3MLsosx2+FT8+h29YYwjl5UVm3f78oIhW5LNBZLZU0dnv58I16Yl4PNJfq383p13U4kGioh1Z7+fKOtEQwTJfsjrbEi2432wI49N/UKCk5HUtTbplFLkl2lh/F8Rk7CJr7FB6yPcZ+9q6xBi7GSmD4lsXfXbk92r9f/wo51O0ox7but9c/NvOyNHi+UrWjcpvdxuFUaAfTHA4meAw+8HymBDejtrNWLU/HHIFH78b0Hq/Cq1uRMr1VKlJt3Luk0+fOrX29Gm2Na4t4r+vgdSn3npFS4//2V+H7LXgw+oxPO6aY/hssjM6yXgKN3yNEWY27N6ZCsx/B3LpW0b5j0DQBgYJ8OAJzNtWtVKEvw73/v3ZPsOet3uTZ357qdDUuxSilMzMzVbbWzZLN+dYFRaenlhfnYstv5w7C9B6vw7Jxs2/WGicbosX2iW3zhG3Fd+L0QP1Tvuh3W71CcXvDcrs6Kno/xsuNaaa3XzuPK6mhLE+Ntxp4PU3WaNMbv//m4ZqN2jqsRu0fr9UXG7eUT7s+H2sZQJvjYzipuHNP4fDJxfuOT3kiiL138IFcF+yowPiMHI6cuc3yVf2ZOtqMmolFPzFyHlxbmY2FOWgzrDwDYUNh4LJtEUlG6//vnDXsRx3egMhL7pffy+29n6Nv8kvJGdwClFY075Q2Z+I3pbWbFDOUb7dNhJHvXAUyIu8BES8NusJtw7Y5cG5XKZqahTPDua/iRVJtoYmhH9NmBG4nZLbO1cX7SqdFNOo7y52Vv00RVCRu1i13BvkOmqhQ27TpgugSq95l/aLKZb6yd+82P1tlw/w0/55vjhgG2avjkhsNDFO73Zs7b3eWVGDc3G4tyS/CCh3NAmxH+OngX7ovSedx/pdJ7vBCnhy6d35sbzJxbSinD4zB88hJsHTsEd01bbmp/109chNo6hTsH9bYQ5RF//czaeEcAcPHY+bb2FX/RzHI4kUd8NafZocKtGvPhWszbWNSoxZQfAl2CN/PlcKWKJu53O7MIeWXrntQ8XCww05s3RcyOrphKRvPsWrnbMEri2bsONDjX9c6/ZK1loncBVuKxM0OUVa8v2lx/F6IXQRAlaxufypmiAp3gUyX+A3nf4+7ICsr0SXD7G8572ZrhRWnHbuk8HecINWr2maxzT8NjYHxAdsRUc0xOMCiWkXSdfu5vszfiugnm+5GkGysPtu2O8+9E6BO8nQ4QbnrTwhjtdlpBpPuM8YlK204fVgVBsh6WZvNu9PlM9q4Duk1Prabv1dutD+NgllsXEzPDaQRRnVJ4dk42ipNMl+iG0Cd4qy0u9Dg5zebb7J5tdp9B/go4mWqxKdtdbm+4g1hD48Zc0WM5T7tcQp29Nv2G342nV41ldNyiR2fZ1n14aWG+o16xZoU+wQdVYan5q7udNupGdcbpYOSUZfXvKVmS+SFFzyAS+cuniZr7uXMJTpQ6//bZBk8ullYS/J7ySkd3BR8sL8DivN0o2HcI47/cBKUUakyMSpqOjAbei851EH0OkorxoMLfiibklFJYXWD9ixXfvjidZGYXY1XBflzQox027Up8BzZvo//t8/+9eKvt1zZoB2+zAKw341U8r1NlsjbtyURn8jqnW1usKSjFkDQbVz2oWII3IRXPp/JjxmyxOrBXGjYLdyxH6y0Z9Pe271Di+YBjW7UkuhOzM4tUrP2HqjFvQ/K5d2NZaUVj9Kzrf3Tm700k+rA6FS14/GbUO91NTPBp4levLQl9m28rxny01u8QXPH3JA+Sc2PGVl+wyfhuZFqCMX3MuvOt5HPvuu0tvfl7TUrThj+uKveoLX5UoKtoUpUQ7exn2pIfcMZJx5tev7KmzvYJ/e7S9BttkiKstOIK+t2K28oSzFVM5gQ6waezJxKMtpiM1QvKhyucTymXroJ+V6M3ibeRdHurfpegzY7nEzTs6JRmUvF52N2H319CL611ecz4VNuw01oT3SBfzGJb8YT4lAwcJvg0ZOkha4gz/I3/cm/kQD9cPzG4PTStcmvs9KDxc8YxM5jg04idElxZyDsLzV0fztt0olRggqe0VuRw0uggSbfSoJ83hyG+MU0pJvg0EVvVEj8RCCU2KY07bVmx95DzIQisyE7SicyuZ77IbjA4GvmHCZ4C77kMfydVcEuGxY5ITg0zMR6NHUu37MXod1c42kaQHzinE8cJXkSai8hKEflM+729iGSISK72fzvnYepL1W2c3jRlREFXnWSsl6cSjrGTWLJRNMMip/hA8pXieDn7Vzw3SvAPAIjtrjcGQKZSqi+ATO13IgqY5Q5nUHIiKHXwK7dZHwdqRpa380nEcpTgRaQbgCEAXo9ZPBTAVO3nqQCGOdlHIgE5B0wJe2sYSj9ezm0blAQNAO99n9qe4Kmcc9lpCf4FAI8CiI24s1KqEAC0/zvpvVBERolIlohklZSUOAyDiMiex0Iy7pEe2wleRG4AUKyUMjfbbxyl1GSlVD+lVL+OHTvai8HWq4jIa05Hg+RDVnc4GYvmEgA/F5HrAbQC0EZE3gZQJCJdlFKFItIFgP8DdhNRoASpiscJr69jtkvwSqnHlFLdlFI9AQwHMF8pdRuAWQBGaKuNADDTcZQGeJUnSl9ORjn18vlAOvH6XXrRDn4sgKtEJBfAVdrvRNSEKAX86WP7dduLDaa9I2tcGS5YKbUQwELt5z0ABrux3eT7TcVeiCjVUjFfaVPAnqxE5Dqn1afpMNduGDDBp5E63pJQSDg9lVP1TZi5akeK9uSPQCf4sD1k/drhxMpEZM0D76/yOwRPBTrBh02YJ+8gosbStpkkEZERp2WVkN2cGwpiM0myqam0/SVKZtV264N4UWOBTvBhq9Hwc/Q+IgqfQCf4sLU6Cde7oaaM57I5b36zxdNOXa50dPLLuh2lfodARGTbeG02sq1jh3iy/UCX4MNmf4rn5CSicGOCTyOL8/b4HQIRhQgTPBFRSAU6wYfsGSsRkasCneCJKD2xV3Z6YIInIgqpQCf4sA02RkTkpkAneCIiMsYET0Su44xM6YEJnohcl19y0O8QCAFP8HxQT0RkLNAJnoiIjDHBExGFVKATfOtWgR4Mk4jIU4FO8Kd2bO13CEREaSvQCZ4dnYiIjAU6wbMVDRGRsWAneL8DICJKY4FO8EREZIwJnogopAKd4PmMlYjIWKATPBERGbOd4EWku4gsEJGNIrJeRB7QlrcXkQwRydX+b+deuEREZJaTEnwNgEeUUmcAGADgXhE5E8AYAJlKqb4AMrXfiYgoxWwneKVUoVJqhfbzAQAbAXQFMBTAVG21qQCGOQ2SiIisc6UOXkR6AjgfwFIAnZVShUDkIgCgk8FrRolIlohklZSU2Nov28ETERlznOBFpDWADwE8qJQqM/s6pdRkpVQ/pVS/jh07Og2DiIjiOErwItISkeT+jlLqI21xkYh00f7eBUCxsxAT7N+rDRMRhYCTVjQC4A0AG5VS42P+NAvACO3nEQBm2g+PiIjscjKg+iUAbgewVkRWacv+BGAsgBkiMhLANgC3OAuRiIjssJ3glVLfwLiWZLDd7RIRkTvYk5WIKKSY4ImIQooJnogopJjgiYhCKtAJnj1ZKUj+cM2P/A6BmphAJ3iiILn3ij5+h0BNDBM8kQ9uOr+r3yFQExDoBM+hCiioLvsRx18i7wU6wRMFxZjrTvc7BGqCmOCJUuCWC7v5HQI1QUzwREQhxQRPlAKRwVeJUivQCZ7t4In80aH10X6HQCYEOsETkT8evbbpdtrq06m13yGYxgRPlAJNvYKmR/tj/Q6hSQp0gm/qXxoKjvjqRNXE6hebNwvPt7V5gJ6nBDrBEwXFwcqaBr+XVlT7FIk/VIiuaEEaU4gJnigF4gt9YUp4Tc1RLYKTNoMTKVGAxTeTbGrNJsP0fhWA31zc0+8wTGGCJ0qB+PRWVHbYlzj8EqIqeAiAuy871e8wTGGCJ/LBK1/l+x1CSv3szM5+h+CaS/p0aFTllq4CneBZi0lBddzRLfwOwRGrLUlatWjuUSSp17yZBKYFX6ATPFFQxOfDowP0oE7PjeeebGn9uy7r7VEkPglIhg/0WRaQY0whdeXpnUyv27jMF+yz12pLkmOPaoFu7Y7xKBoyEugET+SnU048FvdfaW0avqdv+jGmjxoQmDpcN00b+VO/Q3BNUCppmOCJbGrZvBkevtpap5db+/fAT3ufGJD04K5eHY7DXZeGrKomzTHBE9l0/+C+tl+brAR/cttWtrfttZn3XmL7tVbr7o28dUd/V7Zjl5t3YB/dc7F7G4vDBE9kU2utJcy0kdaTTbJb/BbNj3w127RKrxY353Y/wfZrW3AJzU8AAAp2SURBVDR3JzP26nCcK9tJBxf0aOfZtpngiSx45bYLGy2zUx+bqAQY//B2yDlHSr0f/s670p5fBp/eCU/f9GO/w7BE7+Prm4bDCAc6wbd0qTRAZNa1Z5/UaJmd2/VEL4mvp449z08+wVrVTdtjWiZd50/Xp25CcL0heG44twtu7d8jZTG4QW/ohb6dmeBddcuF3X3bN2e0CYcRF51i+TV/HnIGvnhgUNL1nvnFkVJp5zYNz5faJIONGV00TmpzJMH//urTksZgpmQ8cqC5B59Xn9kZY2O2N21kf0tNRfW0aCa44ZzE9fJ3X3Yqhv/E/He9e3vvm2O2P+6oRstGDuyN0RZbVXnNswQvIteKyCYRyRORMV7so5nFAS7+POQM1/Z9Xve2jsajaBGmwTlsuHNgL8+2fdbJbUyv2z3JRBR6HZLuHNQbZ3Q5sg+jz/KXPzlSKo0v8Z1wTOMEEdWsmeCYlkd6fraMqY+P3c59V/bVTTSxYuM00ryZYOvYIUnXm/zrfhgeU9Ie1LcjHr4q+UUmfl+x/jr07AbvT8+Y605H5zbm7ly2jh2CjIcusxSTW449qjkesdiqymueJHgRaQ7gRQDXATgTwK0icqYX+4pP2h1aNz7hb7qgK+64pBduG3AKPrrnYvz6olPwxA1nIvORxifCf+6+CI9ddzo+/N1FCfc79hfnYPSVfXBut7boorV4OK/7CZhq8HT/hV+e1+D3hX+4HAt/f3nCfZg158FBhq0K+vdqj/uu6IPvHrsSd13WG711Hk49ENMa5N07I22VX/zVBY3WO61z6wYllGOPatz9/DcX90zaumTUpb3xwM/64mptfJK37uiPNq1a4KQ2reqPZSK39u+BYeedjBd/dQH+OvQsPHVjw1PrvVEDMPv+gbjwlHYYfWUfPHmj/qk3bWR/3DbglIS9SmfdN7C+9Dju5nN01/lJz/a45/JT0b9X+/pl00cNAADccUkvXHNW43FYptzxk/qfPxs9EHdd1hsZD12K+67ogwt7tMNrv+6H+67og1GX9sZDV52GJ288E5+NHggAePbmc+q3P187h//r/K4AgKl39MeE4ZFz7W/DzkbPExtewF69/ULc2v9IafjT+wY2iu1Z7X3+uGvb+mNnNHriWSe3wf1X9qmvVrr5wm6N1ok9Z/p2at3gvIld/8PfXYwfdT4efTu1xsA+HdD1hGPwsdbCJLYn7M0XdtPtNBV9PtGqZXMM6tuhwd/i7wDO7dYWn9x7CY4/ugUG9e2A+66IxHhMy+Y4p1tb/Obinph13yWNLkixn/FpcVUyp590PADg9gEN7wqjnxsAfPWHyxvF7SXxYlxqEbkIwFNKqWu03x8DAKXU03rr9+vXT2VlZbkeR1ANfm4h8ksOYvGYK9H1BOu3my8uyMO4uZvwyFWnYbRBsv0qpwQj3vwe15zVGa/e3s/S9nuOmQ0Apkp9FBw9x8xGMwE2Px2+z/WHPQdx2biFOP2k4zHnwUv9Dsc1IrJcKWX4Bfaq/VVXANtjfi8A0KAbm4iMAjAKAHr0CNYDFq9N+W1/fLJyh+220CMH9kJZRTXuHGRctzqwTwfcc/mpGGmjquSNEf1QXcuh3sLmz0POwKC+Hf0OwxM92h+Lh686rf5Op6nwqgR/C4BrlFJ3ar/fDqC/Umq03voswRMRWZesBO/VQ9YCALGVXt0A7PRoX0REpMOrBL8MQF8R6SUiRwEYDmCWR/siIiIdntTBK6VqROQ+AHMBNAfwplJqvRf7IiIifZ4NcqGU+hzA515tn4iIEgt0T1YiIjLGBE9EFFJM8EREIcUET0QUUp50dLIchEgJgB8cbKIDgN0uheMmxmUN47KGcVkTxrhOUUoZdj9OiwTvlIhkJerN5RfGZQ3jsoZxWdMU42IVDRFRSDHBExGFVFgS/GS/AzDAuKxhXNYwLmuaXFyhqIMnIqLGwlKCJyKiOEzwREQhFegEn4qJveP2111EFojIRhFZLyIPaMufEpEdIrJK+3d9zGse0+LbJCLXxCy/UETWan+bKPGzMluPbau2vVUikqUtay8iGSKSq/3fLpVxiciPYo7JKhEpE5EH/TheIvKmiBSLyLqYZa4dHxE5WkSma8uXikhPB3GNE5FsEVkjIh+LyAna8p4iUhFz3F5JcVyufW4uxzU9JqatIrLKh+NllBv8PceUUoH8h8gwxPkAegM4CsBqAGd6vM8uAC7Qfj4eQA4ik4o/BeD3OuufqcV1NIBeWrzNtb99D+AiAALgCwDXOYxtK4AOccueBTBG+3kMgGdSHVfc57ULwCl+HC8AlwK4AMA6L44PgHsAvKL9PBzAdAdxXQ2ghfbzMzFx9YxdL247qYjLtc/Nzbji/v4cgP/x4XgZ5QZfz7Egl+D7A8hTSm1WSlUBeB/AUC93qJQqVEqt0H4+AGAjIvPPGhkK4H2lVKVSaguAPAD9RaQLgDZKqe9U5NN6C8AwD0IeCmCq9vPUmH34EddgAPlKqUQ9lj2LSyn1NYC9Ovtz6/jEbusDAIPN3GXoxaWU+lIpVaP9ugSRGdEMpSquBHw9XlHa6/8bwHuJtuFRXEa5wddzLMgJXm9i75TNqKvdHp0PYKm26D7tlvrNmNswoxi7aj/HL3dCAfhSRJZLZEJzAOislCoEIicggE4+xBU1HA2/eH4fL8Dd41P/Gi05lwI40YUY70CkFBfVS0RWishXIjIoZt+pisutz82L4zUIQJFSKjdmWcqPV1xu8PUcC3KC17typaTNp4i0BvAhgAeVUmUAXgZwKoDzABQicpuYKEYvYr9EKXUBgOsA3CsilyZYN5VxQSLTNv4cwH+0RelwvBKxE4frMYrI4wBqALyjLSoE0EMpdT6AhwG8KyJtUhiXm5+bF5/prWhYiEj58dLJDYarGuzH1diCnOB9mdhbRFoi8gG+o5T6CACUUkVKqVqlVB2A1xCpPkoUYwEa3nY7jl0ptVP7vxjAx1oMRdotX/S2tDjVcWmuA7BCKVWkxej78dK4eXzqXyMiLQC0hfkqjkZEZASAGwD8P+1WHdrt/B7t5+WI1Nuelqq4XP7c3D5eLQDcBGB6TLwpPV56uQE+n2NBTvApn9hbq+96A8BGpdT4mOVdYlb7LwDRJ/yzAAzXnn73AtAXwPfardoBERmgbfPXAGY6iOs4ETk++jMiD+nWafsfoa02ImYfKYkrRoOSld/HK4abxyd2WzcDmB9NzFaJyLUA/gjg50qpQzHLO4pIc+3n3lpcm1MYl5ufm2txaX4GIFspVV+9kcrjZZQb4Pc5luwpbDr/A3A9Ik+r8wE8noL9DUTklmgNgFXav+sBTAOwVls+C0CXmNc8rsW3CTEtPwD0Q+QLkg/gX9B6FduMqzciT+RXA1gfPRaI1M9lAsjV/m+fyri07R0LYA+AtjHLUn68ELnAFAKoRqQkNNLN4wOgFSJVUHmItILo7SCuPETqWqPnWLTlxC+0z3c1gBUAbkxxXK59bm7GpS2fAuDuuHVTebyMcoOv5xiHKiAiCqkgV9EQEVECTPBERCHFBE9EFFJM8EREIcUET0QUUkzwREQhxQRPRBRS/wfED58BT9eVPwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "seq_lengths = [len(x) for x in sequences]\n",
    "plt.plot(seq_lengths)\n",
    "max(seq_lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = max(seq_lengths)\n",
    "\n",
    "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of data tensor: (20000, 99)\n",
      "Shape of label tensor: (20000, 2)\n"
     ]
    }
   ],
   "source": [
    "labels = to_categorical(np.asarray(y))\n",
    "print('Shape of data tensor:', data.shape)\n",
    "print('Shape of label tensor:', labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99\n"
     ]
    }
   ],
   "source": [
    "print(MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# we shuffle our sorted data, to make it random first\n",
    "indices = np.arange(data.shape[0])\n",
    "np.random.shuffle(indices)\n",
    "data = data[indices]\n",
    "labels = labels[indices]\n",
    "\n",
    "# before we begin to split them\n",
    "num_training_samples = int(TRAINING_SPLIT * data.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = data[:num_training_samples]\n",
    "y_train = labels[:num_training_samples]\n",
    "x_val = data[num_training_samples:]\n",
    "y_val = labels[num_training_samples:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare embedding:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_words = min(MAX_NUM_WORDS, len(word_index)) + 1\n",
    "embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    if i > MAX_NUM_WORDS:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "\n",
    "# load pre-trained word embeddings into an Embedding layer\n",
    "# note that we set trainable = False so as to keep the embeddings fixed\n",
    "embedding_layer = Embedding(num_words,\n",
    "                            EMBEDDING_DIM,\n",
    "                            embeddings_initializer=Constant(embedding_matrix),\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Orthogonal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    }
   ],
   "source": [
    "#Model\n",
    "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sequence_input)\n",
    "x = Bidirectional(LSTM(256))(embedded_sequences)\n",
    "y = Dropout(0.5)(x)\n",
    "\n",
    "preds = Dense(labels.shape[1], activation='softmax')(y)\n",
    "model = Model(sequence_input, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 99)]              0         \n",
      "_________________________________________________________________\n",
      "embedding (Embedding)        (None, 99, 100)           5918000   \n",
      "_________________________________________________________________\n",
      "bidirectional (Bidirectional (None, 512)               731136    \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 2)                 1026      \n",
      "=================================================================\n",
      "Total params: 6,650,162\n",
      "Trainable params: 732,162\n",
      "Non-trainable params: 5,918,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Train on 16000 samples, validate on 4000 samples\n",
      "Epoch 1/50\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "checkpoint = ModelCheckpoint(\"best_weights.hdf5\", monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "earlystopping = EarlyStopping(patience=4, monitor='acc')\n",
    "callbacks = [earlystopping, checkpoint]\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=32,\n",
    "          epochs=50,\n",
    "          validation_data=(x_val, y_val),\n",
    "          callbacks=callbacks,\n",
    "          verbose=2)\n",
    "\n",
    "model.load_weights(\"best_weights.hdf5\")\n",
    "\n",
    "print(\"DONE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_tweet(text_to_predict):\n",
    "    test_sequence = tokenizer.texts_to_sequences([text_to_predict])\n",
    "    test_data = pad_sequences(test_sequence, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "    predictions = model.predict(test_data)\n",
    "    prediction = np.argmax(predictions)\n",
    "    print(\"The person who tweeted this is {}\\n\".format(classes_index[prediction]))\n",
    "    ranking = predictions.argsort()[-6:][::-1]\n",
    "    for i in reversed(range(0, 6)):\n",
    "        rank = ranking[0][i]\n",
    "        print(\"{:>10} : {:.5f}%\".format(classes_index[rank], predictions[0][rank]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_tweet(\"If I just disappear from the world right now, will anyone even care?\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}