hw3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for data\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# for deep learning\n",
    "from tensorflow.keras.layers import Embedding, Bidirectional, LSTM, Dense, Permute, multiply, Activation, Input, concatenate\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "\n",
    "# for bert language model\n",
    "import transformers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GPU optimization stuff\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "physical_devices = tf.config.list_physical_devices('GPU')\n",
    "tf.config.experimental.set_memory_growth(physical_devices[0], enable=True)\n",
    "tf.compat.v1.disable_eager_execution()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# constants\n",
    "\n",
    "DATA_PATH = 'data/'\n",
    "MODEL_PATH = 'models/'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read prepared datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                text  y  helpful_votes  \\\n",
      "0  1 coffee keurigs ive tried 15 different one in...  1              1   \n",
      "1  1 favorite coffee ever wonderful dark roast ri...  1              0   \n",
      "2                                      1 gallon cmon  0              9   \n",
      "3  1 star there 0 star liquid chocolate pulverize...  0              1   \n",
      "4  10 month old love artificial color sweetener g...  1              0   \n",
      "\n",
      "   total_votes  vine  verified_purchase  \n",
      "0            1     0                  1  \n",
      "1            0     0                  1  \n",
      "2           10     0                  0  \n",
      "3            1     0                  1  \n",
      "4            0     0                  0  \n"
     ]
    }
   ],
   "source": [
    "# split dataset\n",
    "df_train = pd.read_csv(DATA_PATH + 'train.csv')\n",
    "df_test = pd.read_csv(DATA_PATH + 'test.csv')\n",
    "\n",
    "print(df_train.head())\n",
    "\n",
    "# get X (text features)\n",
    "X_train = df_train['text'].values\n",
    "X_test = df_test['text'].values\n",
    "\n",
    "# get other numerical and categorical features\n",
    "feat_train = df_train.drop(['text', 'y'], axis=1).values\n",
    "feat_test = df_test.drop(['text', 'y'], axis=1).values\n",
    "\n",
    "# get target\n",
    "y_train = df_train[\"y\"].values\n",
    "y_test = df_test[\"y\"].values\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load prepared word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_matrix = np.genfromtxt(DATA_PATH + 'word_emb.csv', delimiter=',')\n",
    "\n",
    "MAX_LEN = 30\n",
    "\n",
    "embedding_layer = Embedding(input_dim=emb_matrix.shape[0], output_dim=emb_matrix.shape[1], weights=[\n",
    "                            emb_matrix], input_length=MAX_LEN, trainable=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Bi-directional LSTM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# code attention layer\n",
    "def attention_layer(inputs, neurons):\n",
    "    X = Permute((2, 1))(inputs)\n",
    "    X = Dense(neurons, activation=\"softmax\")(X)\n",
    "    X = Permute((2, 1), name=\"attention\")(X)\n",
    "    X = multiply([inputs, X])\n",
    "\n",
    "    return X\n",
    "\n",
    "\n",
    "# model\n",
    "def bid_lstm(input_shape, additional_input_shape=None):\n",
    "    # input\n",
    "    X_indices = Input(input_shape, name=\"text_input\")\n",
    "    inputs = X_indices\n",
    "\n",
    "    # additional input layer of other features besides text\n",
    "    if additional_input_shape:\n",
    "        feat_input = Input(additional_input_shape, name='feat_input')\n",
    "        inputs = [X_indices, feat_input]\n",
    "        feat_input = Activation('tanh')(feat_input)\n",
    "\n",
    "    # embedding\n",
    "    embeddings = embedding_layer(X_indices)\n",
    "\n",
    "    # apply attention\n",
    "    X = attention_layer(embeddings, neurons=MAX_LEN)\n",
    "\n",
    "    # 2 layers of bidirectional lstm\n",
    "    X = Bidirectional(LSTM(\n",
    "        units=MAX_LEN, dropout=0.2, return_sequences=True))(X)\n",
    "    X = Bidirectional(LSTM(units=MAX_LEN, dropout=0.2))(X)\n",
    "\n",
    "    # concatenating additional features before prediction layer\n",
    "    if 'feat_input' in locals():\n",
    "        X = concatenate([X, feat_input])\n",
    "\n",
    "    # final dense layers\n",
    "    X = Dense(64, activation='relu')(X)\n",
    "    y_out = Dense(1, activation='sigmoid')(X)\n",
    "\n",
    "    # compile\n",
    "    model = Model(inputs=inputs, outputs=y_out)\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenizing  sequences from training data\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(X_train)\n",
    "\n",
    "X_train_indices = tokenizer.texts_to_sequences(X_train)\n",
    "\n",
    "# padding sequences to uniform length\n",
    "X_train_indices = pad_sequences(X_train_indices, maxlen=MAX_LEN)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEACAYAAACznAEdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAxNklEQVR4nO2debhcRbW3318SwpAQgkBkCIKRQSFCkIBeFYhMolcGZUpQIaJyQQKIMgpImD5mAQ3iRWQSBRlEJiEyRVAI8zwK3AAhKDMYkCHJ7/ujqslm0+ek+5w+Od191vs8+zm9a69dXbtPd62qVbXWkm2CIAiCoF9vNyAIgiBoDkIhBEEQBEAohCAIgiATCiEIgiAAQiEEQRAEmVAIQRAEAdBECkHSZpIek/SEpAN6uz1BEAR9DTWDH4Kk/sDjwCbAdOAOYJzth3u1YUEQBH2IZpkhrAs8Yfsp2+8CFwBb9nKbgiAI+hTNohCWA54tnE/PZUEQBMF8olkUgqqU9b4tKwiCoA8xoLcbkJkOLF84Hw7MKAtJ2gXYBeCXJx659vd2HDd/WhcEQUuz8LLr9XYTmoZZ7z5XbQAONM+i8gDSovJGwHOkReUdbD/U0T3vvfRU7zc86DLxAw2C3qEzhdAUMwTbsyRNACYD/YEzO1MGQevznxk393YTgmC+0gqDoKaYIXSFAQOXa82GB0EQ9CJNP0PoCjHCDIKgnemNGUXLKoQgCNqPVjCrtDNdVgiSFgJuAhbM9Vxs+1BJo4BfAQsBs4Af2L5d0hLAxcA6wNm2JxTqOgrYEVjc9uCutikIgtamGWb+fVkpdXkNQZKAQbZnSloA+BuwF3A4cJLtqyV9FdjP9hhJg4C1gJHAyJJC+BzwNPCPWhVCrCEEQRDUT4+sIThpkpn5dIF8OB9DcvliZH8C228Cf5O0UpW6pgIkHVMbzTCSCIKgb9Nus4lurSHkoHR3ASsBp9q+TdIPgcmSTiB5Qn++260Mgl6m3X74QVCNbikE27OBUZKGApdKGknyJN7b9iWStgN+A2zc7ZbyIU9lwlM5mF/EjLRrhCJtLRrmhyDpUOBN4BBgqG3ndYbXbQ8pyI0HRhfXEArXZsYaQhAEQc/RI2sIkpYC3rP9mqSFSbOAY0lrBhsAU4ANgX909T06I0ZsQdD8xAyhteiOyWgZ4Jy8jtAPuND2lZJeA07J8YneJpt4ACRNIy04D5S0FbCp7YclHQfsACwiaTpwhu2J3WhbEARNQLMM3EIx1UbLhq6I4HbB/CQ6lKBdaMvQFUEwP2mWkW7QXLTbQKE7awirAn8oFI0Afmr7ZEl7ABNInspX2d5P0ibAMcBA4F1gX9s3SFoEuAj4BDAbuML2AV1tVxAEwfyinoFCKyiP7jimPQaMgvf9EZ4jbT39Eikf8hq235E0LN/yErC57Rl5e+pk5qbJPMH2jZIGAtdL+ortq7vatiAI2p9W6GBbjUaZjDYCnrT9tKTjgWNsvwNg+4X8956C/EPAQpIWtP0WcGOWeVfS3aSMaUHQNETnE/QFGqUQxgLn59erAOvlgHVvA/vYvqMkvzVwT0VpVMgObpsDpzSoXUHQEGINIWhmGjVg6bZCyGaeLYADC3UuDnyOFNn0QkkjcuwjJK1O8lfYtFTPAJJS+bntpzp4r/BUDoIAiFlbT9CIGcJXgLtt/yufTwf+mBXA7ZLmAEsCL0oaDlwK7Gj7yVI9p5OinZ7c0RvZPj3LxbbTIOjjtPusrVUT5IxjrrkI4E8kD+UpklYh7Sp6KZuDrgIOtP33YgWSjiRFRv1eA9oTBEHwAWI2URvdckzLW0afBUbYfj2XDQTOJO1Aepe0hnCDpINJZqViKItNSQrjWeBRoLKmMMn2GZ29d8QyCoIgqJ/OHNPCUzkIgrqI0XZrE57KQRA0jJ603Yey6V3mqRAknQl8DXjB9shcti0wEfgUsK7tO0v3fAx4GJho+4Rcdg0pIN4A4GZgd9uzJe0K7E7yUp4J7GL74cY8XhAErUS7LxTXQ28ox341yJwNbFYqexD4BnBTB/ecBJQ9jbezvSYpp/JSwLa5/Pe2P217FHAc8LMa2hQEQRA0mHnOEGzfJGnFUtkjUD0Hcg5r/RQpWU7xnjcK7zmQlHu5WA4wqFIeBEHQTrSCOayhawiSBgH7A5sA+1S5PhlYlzR7uLhQvjvwI5Ki2LCRbQr6Fq3wowuCZqXRi8qHASfZnllt9mD7y5IWAn5H6vivzeWnAqdK2gE4GNipwe0K+ghhgw6qEQOF2mi0QvgssE3OgDYUmCPpbduTKgK235Z0OSki6rWl+y8ATuuo8mLoCvVfjH79BjW4+UEQBH2XhioE2++rYUkTgZm2J0kaDCxq+/kcs+irpJ1GSFrZdsVZ7b/pJAdzhK5obmIUFgStTS3bTs8HxgBL5nzHhwKvAL8g7Ra6StK9tr/cSTWDgMslLQj0B24AfpWvTZC0MfAe8CphLmpZwlwTtAN9eWATnspBELQ9fbmTLxOeykEQ9ArREbcWoRCCoE2Jzjiol66GrpgIfB94MYv9xPafJW0CHEPyJ3gX2Nf2DaX6LidFRx1ZKt8GuAhYpxwKIwjamei4g2ahlhnC2cAk4NxS+UmVOEUFXgI2tz1D0khgMrBc5aKkb5DiFX0ASYsCewK31d70IGgP6lmMD+UR9CRdCl3Riew9hdOHgIUkLWj7nbz19EckP4ILS7ceQYpj9CHv5iAI5hI7ubpGKNLa6M4awgRJOwJ3Aj+2/Wrp+tbAPbYrSW+OAE4E3ioKSVoLWN72lZJCIQRB0HBiFlYbXVUIp5E6eDO3o9+5clHS6sCxpIxoSBoFrGR77+JsQ1I/UmTU8bW8adFT+ZcnHsn3dhzXxeY3jr785QmCoL2oyQ8hd+JXlheCq12TNJzkePadSu5kSbsBh5AWmgcAw4BbSOErnmTuusLSJKe3Lea1sBx+CEEQ9ATtPsjrzA+hlnwIH0LSMoXTr5PyIyBpKHAVcGBFGQDYPs32srZXBL4IPG57jO3XbS9pe8V8bSo1KIMgCIKg8XQ1dMWYbAYyMA34nyw+AVgJOETSIblsU9svNLbZQRAEPUNPLdy3wsyjZUNXDBi4XGs2PAiCoBdpy9AVsf0uCNqPVhhFtzNd9VRekxStdDDJZPTNSipMSQcC3wVmA3vanpzLpwDLAP/JVW9q+wVJ44Hjgedy+STbZzTi4YIgqI3oiAOowWQkaX3SLqBzCwrhDmAf23+VtDPwcduHSFoNOJ+UJnNZ4DpgFduzs0LYp7xgnBXCaNsT6ml4mIyCIAjqp1smow48lVcFbsqvryWFqDiEtI30guyM9n+SniAph1u70O5OCZNREAStRCvMwrq6hvAgsAVwGbAtsHwuX460dbTCdAqxjICzJM0GLgGO9NzpydZ5JvI4sLftZ7vYrqBFaIUfRxD0NbqqEHYGfi7pp8DlJIczgGpTkUqn/03bz+VAdpcA3yYFzLsCOD/HO9oVOAfYsNqbNqOnctA1YobX84TSDeqlSwrB9qPMDUuxCikXMqQZwfIF0eHAjHzPc/nvvyX9nmRKOtf2ywX5X5NCXnT0vpFTOQhqJJRu1+jLirRLCkHSsLxDqB9wMHPzI18O/F7Sz0iLyisDt0saAAy1/ZKkBUi7lq7LdS1j+/l8/xbAI7W0oS//04IgCHqCrnoqD5a0exb5I3AWgO2HJF0IPAzMAnbPO4wGAZOzMuhPUga/zvfvKWmLLP8KNQa6i9FPEAStRCsMYlvWUzlMRkHQt2mFDrYZaUtP5SAI+jatZiVoBQVWi2Pa8qTdQEsDc4DTbZ8i6Xhgc9IOoydJ4a5fy2ahM4DPkBTOubaPznUNJKXjHJPrOsj2JfnadsBE0q6k+2zv0Fm7wjEtCIKgfjqbIdSiEJYBlrF9d94yehewFWkH0Q22Z0k6FsD2/pJ2IIWwHitpEdJ6whjb0yQdBvS3fXBekP5IXmhemZRWc0Pbr1YWrTtrV5iMgqD5aYVRcV+ju57KzwPP59f/lvQIsJztvxTEpgLbVG4BBuWdRQuTZhBv5Gs7A5/Mdc0BXsrl3wdOraThjHDZQdAetJpZp5noDWVa1xpCDmGxFnBb6dLOwB/y64tJISyeBxYheR6/kpPnABwhaQzJzDTB9r+AVXL9fyftQppo+5rO2hIjjyAIgsZSs0KQNJjkYfzDSmTTXH4Qacvo73LRuqRIp8sCiwM3S7qONEsYDvzd9o8k/Qg4geSxPIDkszAmy9wsaaTt1zpqT4w8gqBvE4PCxlOTQsgLxZcAv7P9x0L5TiQns40KcYl2AK6x/R7wQh71jwYuAt4CLs1yF5HCZEPycJ6a7/k/SY+RFMQdpXZE6IogqJHoMIN6qcUxTcBvgEds/6xQvhmwP7CB7bcKtzwDbCjpPJLJ6HPAybYt6QrSLOAGYCPSgjPAn4BxwNmSliSZkJ4qtyVCVwTtSHTcQbNQyy6jLwI3Aw+QtooC/AT4ObAgUIlFNNX2rtm0dBawGinY3Vm2j891rQD8FhgKvEjaqvpMVjonApuRzE1H2b6gs3aFQgiCIEjUM6jo1rbTZqXdFUKMGoMg6AnCU7kFiUXzIGgcMcCqjX693YAgCIKgOahlUblq6Ip8bQ9gAmnb6VW298vlawD/CwzJ96wDLEBai6gwHDjP9g8lnQR8KZcvAgyzPbTbTxcEQUBzzLhbYZZSi8loFvDjYugKSdcCHyU5oK2Rs50NA8geyucB37Z9n6QlgPdsvw2MqlQq6S5S6Gxs710o34Pk/BYEQdDUtEInXw9dDl1BCjdxjO138rVKuIlNgftt35fLXy7XmWMXDeODM4YK40g5F4KgT9BunUrQunQndMXxwHqSjgLeBvaxfQfJh8CSJgNLARfYPq5U1TjgDy5tccrbUj9O8lMIgpYkOvigVely6IpsGlqc5Hi2DnChpBG5zi/msreA6yXdZfv6QnVjSSEryowFLrY9u4M2hKdy0PQ0g706aD5aYaDQndAV04E/5lH+7ZLmAEvm8r/afinf+2dSboTr8/mawADbd1V5q7HA7lXKgfBUDoKgdal3oNCU0U47Cl1BCjexITBF0irAQFI468nAfjkXwrvABsBJhfvGAedXeZ9VSTOOW7v0JEEQdJlWGL0GPU8tM4QvkMw7D0i6N5f9BDgTOFPSg6SOf6c8W3hV0s9IgekM/Nn2VYX6tgO+WuV9xpHWG2LkHwTzmZ4yc4WiaS1aNnRFpNAMgiCon7YMXREjmiAIgsZSyxrCQsBNpMimA0i7gA6V9BFSlrQVgWnAdpUUmPm+j5HCW0+0fUIuG0cyNxmYAXwr51T+GHAOKQpqf+AA239u0DPWRewQCbpLDCqCVqWW8NcCBtmemXcb/Q3YC/gG8IrtYyQdACxue//CfZeQwlbcZvuEvE11BrBaVgLHAW/ZnijpdOAe26dJWo207rBiZ+2KXUatTXSaQdA7dMtklBd5Z+bTBfJhUtiKMbn8HGAKKWEOkrYiJbh5s1CV8jFI0sukOEdPVN4mnwMsRlIcQRsTM7GeJ5RuUC81LSpL6g/cBawEnGp7f0mvFQPQSXrV9uKSBgHXAZsA+wAzCyajbUi7k94E/gF8yfZsScsAfyFtOx0EbNyBn8L7xAwhCIIg0agEOTUtKmfP4VGShgKXShrZifhhwEnZxPR+YTY37UYKffEU8AvgQOBIcvpM2ydK+i/gt5JG2p5TrDg8lYOgZ4jZRAB17jKy/ZqkKaRUl/+StIzt5/MIvxLc7rPANnmNYCgwR9LbpPhH2H4SQNKFwAH5nu/mOrF9a17IXrJQZ+X9w1M5CHqAMOE1H83qqbwUKXz1a5IWBjYGjgUuB3YCjsl/LwOwvV7h3okkk9EkScsCq0layvaLJJPSI1n0GWAj4GxJnwIWIuVcDoK2J0bnQbNQywxhGeCcvI7QD7jQ9pWSbiUFtPsuqUPftrNKbM+QdBhwk6T3gKeB8fnyj4FfS9qbtMA8PjyWg75CjM77Bq2g+FvWUzlMRkEQtDM9pUDa0lM5CIK+TSuMuFuNLnsqF67vQ0qWs1R2ONuEtK4wkBT0bl/bN2TZ7YGDSN7IxRzMK5C2oy4FvELyYJ7esKcMgqDtaHdTW1MuKgPvABsWPZUlXW17qqTlSYvDzxTkXwI2z2sGI0nhsJfLuZWPB9a2/aKkcyRtlBPnnACca/scSRsCR1M9gU4QBEHdxGyiNrrjqQwpz8F+5B1GWf6ewu0PAQtJWhAYATyedxhBcl7bmpQ4ZzVg71x+IynXQhAEQUNohtlEKyilWjOmlT2Vb5O0BfCc7fuKDmgltibFKHpH0hPAJ3Ne5unAViSzEsB9WfYU4OvAopKWsP1y1x4rCIJWpBU6zXamq57Ka5DWAjbt6B5Jq5P8FTbNdbwqaTdShNQ5wC2kWQOkEBeTJI0nrVc8B8zqwvMEfZzoUIKg69S97VTSoaQOfQ/grVw8nBSQbl3b/5Q0HLgB+I7tv3dQzy7ASpWF5UL5YOBR28M7uGcXAPVfbO1+/QbV1fYgCIK+TmfbTmsJf132VP4LcKztKwsy04DReZfRUOCvwOG2LynVNcz2C5IWJ60VbGf7cUlLkkJpz5F0FDDb9k87a1f4IQRB44iZVd+hu34IVT2VO5GfQFprOETSIblsU9svAKdIWjOXHW778fx6DHC0JJNMRrvPq1HxBQ6CIGgs4akcBEEwH2iWQWy3TEbNyoCBy7Vmw4MgCHqRtgxd0Qz7ioP5Q7OMrIKg3ely6ApJo4BfkUJVzwJ+YPv27JF8MbAOKenNhEJdRwE7kvIvDy6ULwicC6wNvAxsb3taQ54waHlC+QftQCsMbLocugI4HDjM9tWSvgocR1ocfhs4BBiZjyJXAJNI6TOLfBd41fZKksaS/Be27+IzBUEQNB31DmyaMpZRJ6ErDAzJ5YuR/BCw/SZJaaxUpa6pAFU8m7cEJubXF5Oc1BQ5EYIg6KvUo0AapTy6E7rih8BkSSeQtqN+vhvtWA54FsD2LEmvA0uQAuUFQRC0PK1gMupXi5Dt2bZHkTyS181RTHcD9ra9PCkw3W+60Y5qq94fmh1I2kXSnZLuPOPc87vxdkEQBEGZroaueJO0TjDUtpVsQK/bHlKQG0/yXp5QpY6ZpUXlycBE27dKGgD8k5RfocPGxbbTIAiC+unWttMqoSs2Ji36zgA2AKYAG/LhheJ6uBzYCbgV2Aa4YV7rB7HzJAian1YwkwRz6XLoCkmvkUJRDCDtLNqlckOObTQEGChpK1LoioclHQfsACwiaTpwhu2JJHPTb3OI7FeAsQ16viBoeqLTDJqFlvVUjtAVQdB+hHLsedrSUzkIgvajN7ZaBnOpWSFkk9GdpCxpX5O0Lcl34FOkPAh3ZrmBwP8Co0l5E/ayPUXSIsBFwCeA2cAVtg/I96wPnAysAYy1fXFjHi8RX5wgCIJ5U88MYS/gEeY6oz0IfIPU+Rf5PoDtT0saBlwtaZ187QTbN2alcb2kr9i+GngGGE/KnFYT0ckHQRA0llod04YD/w0cBfwIwPYj+VpZfDXg+izzQl58Hm37dlJSHGy/K+lukl8DlbhFkubU2vDYZRQEQa3EALI2ap0hnAzsByxag+x9wJaSLgCWJwWsWx64vSKQs6ptDpxSR1uDoCWIzidoVWrxQ/ga8ILtuySNqaHOM0nrCncCTwO3kKKhVuobAJwP/Nz2U11ocxA0NTF7bW36skKvZYbwBWCLHNF0IWCIpPNsf6uasO1ZpFAWAEi6hQ86rZ0O/MP2yfU2VtIuZH+HX554JN/bcVy9VQTB+/TlH34QVKOWaKcHAgcC5BnCPh0pgyyzCMm/4U1JmwCzbD+crx1Jioz6va401vbpJIUSfghBt4mRfNDMNGX4646Q9HXgF8BSwFWS7rX9ZWAYKQrqHOA54NtZfjhwEPAocHdejJ5k+4y8C+lSYHFgc0mH2V69G88VBEHQ47TbLLNlPZUjuF0QBEH9tKWnckz3gyDoCdpt1F8P3fFUnkhyQnsxi/zE9p8lfRPYt3DrGsBnbN8r6RpSsLwBwM3A7rZnS/oRaV1hVq5vZ9tPd/PZWpq+/KUMgqB3qNlklDvt0cCQgkKYafuETu75NHCZ7RH5fIjtN3L+hIuBi2xfIOlLwG2235K0GzDGdqc5lWNROZifhIIO2oVum4yqeSrXyDiSzwEAtt8ovO9AclY02zcW7pkKdLiLqUL8QIMgCBpLdz2VJ0jakWRK+rHtV0vXtwe2LBbk7GjrAleTZgllvpuvdUqsIQRB3yYGhY2nO57KpwFHkEb5RwAnAjsX7vss8JbtB4v12f6ypIWA35EyrV1buOdbJLPUBl18niAI+gitNihsBQXWEE9lSb8GrizdN5aCuaiI7bclXU6aPVyb69iY5Kewge13qt0XnspBELQq9Sqw3lAgdfkhFDyVvyZpGdvP5/K9gc/aHpvP+5FCWq9fiVckaTCwqO3nczyj3wE3254kaS2S+Wgz2zXlZm73ReVWGE0EQdB69JQfwnGSRpFMRtOA/ylcWx+YXgpeNwi4XNKCQH/gBuBX+drxwGDgouzB/IztLbrRtpan1abDQRA0lqafITQT7T5DCIK+SMyMe5629FQOgqD9aIaZcV9WSqEQgl6hL//ogqBZ6U7oijVJawCDSWsI38xeyCuSci8/lm+danvXUl2XAyNsj8znuwK7A7OBmcAulZDZQXvSDCPBoLWJQUXjqWeGsBepox+Sz88g7Tj6q6SdSfGLDsnXnrQ9qlolkr5B6vSL/N72r/L1LYCfAZt11pj4MgRBEDSW7oSuWBW4Kb++FpjMXIXQUT2D8/27ABdWygshLSDtRprngnGMMIMgCBKNGiD3q1HuZFLoijmFsgeBytbQbYHlC9c+LukeSX+VVGxpxaP5rfIbSNpd0pPAccCeNbYrCIIgaBDdCV2xM/BzST8FLgfezeXPAx+z/bKktYE/SVodGAGsZHvvvM7wAWyfCpwqaQfgYGCnKm0JT+UgCJqGdjNdz9MPQdLRpDSYs8ihK4A/lkJXrAKcZ3vdKvdPAfYB1iGZlN4lKaJhwC22x5Tk+wGv2l6ss3aFH0LP025f9iAIOvdD6E7oimG2X8gd+NnAFNtnSloKeCUnvhlBSoTzaduvFOpZEbiysMto5UrICkmbA4faHt1ZW0IhBEHQE7T7QKinHNPGSdo9v/4jcFZ+vT5wuKRZpG2kuxaVQQdMyMHt3gNepYq5KAiCYH7QihtWGqXEInRFEATBfKBZZh4NMxk1EwMGLteaDQ+CIOhFGpFCcxrwb5IJaJbt0ZKOIOUzmAO8AIy3PUPSusDplVuBibYvzfVMAZYB/pOvb2r7hcL7bANcBKxj+87O2tSK07qgb9AsI8EgqJeaZghZIYy2/VKhbEjFoUzSnsBqtneVtAjwru1ZkpYB7gOWzedTSIvSH+rsJS0KXEXKtTxhXgohTEatTXSaQdA79MiickfexbaLTmcLUYPXceYIklPaPrUIR4cSBEHQWGpVCAb+IsnA/9o+HUDSUcCOwOvAlyrCOZ/ymcAKwLdtzyrUdZak2cAlwJG2nTOmLW/7Skk1KYQwGQVBECTm6y4jScvm9YFhpLhFe9i+qXD9QGAh24eW7vsUcA4plebbkpaz/Vw2D10CnJePG0hrENM6MysVCZNREAS1EhaFuTR0l5GkicBM2ycUylYArqo4mpXkbwT2LXfwksYDo4GDgCeZGwF1aeAVYIsq9xRDV6wdoSuCIGhXekqJdWsNQdIgoJ/tf+fXm5Icz973LiYFuXs0y38ceDYvIq9Aioo6TdIAYKjtlyQtAHwNuM7268CShfebQgczhGyqOh1ihhAEQWvRCrOUWtYQPgpcKqki/3vb10i6RNKqpG2nTwOVJDhfBA6Q9F6+9oOsBAYBk7My6A9cB/y6sY8TBEHQnPTkumd4KscMIQiCNqYpTUZBEARBdVrBDFQPLTtDiNAVQRAE9dNToSv+QFowBhgKvGZ7lKQlgItJ+Q/Otj2hUM9AYBIwhrS+cJDtS/KOo+OB57LoJNtndNam8EMIgqAnaLdRfz3UYzL6UjF0he3tK68lnUhyTgN4m5QIZ2Q+ihxEyr62Ss6j8JHCtT8UlUcQBEFn9OWOu6fo9hqC0vaj7YANAWy/CfxN0kpVxHcGPpnl5gAvVZEJgvlCdChB8EG6Fboisx7wr4JPQlUkDc0vj8iZ154kBbH7Vy7fWtL6wOPA3raf7ay++DEHQRA0ln41yn3B9meArwC75467wjjg/BrqGAAMB/6e67oVqHg7XwGsaHsNkn/COdUqkLSLpDsl3Tlnzps1Nj0IgiCohW6Frsjex88Ba9ueXpIbTwqZPSGfixSeYlHbcyQtD1xje/XSff1JOZkX66wd4YcQBEGQqMdi0iOhK/LljYFHy8qgGjmq6RWkHUY3ABsBD+f3WMb281l0C+CRedUXBEHjCBNsADXMECSNAC7Np5XQFUfla2cDU23/qnTPNGAIKdnNa6TMaA/n2Ea/JW1TfRH4ju1nJB1NUgSzSIHtdrP9aGftihlCEARBolEzhHBMC4Ig6EO0ZeiKcEzrGmEaCIKgI2r1VB4KnEFyNDPJn+Ax4A/AisA0YDvbr2Zv5P8l5TqYA+xle0pOilPsxYcD59n+YX6P7YCJuf77bO/QvUcLqhGKNGgXYnDTeGqdIZxC2hG0Te7wFwF+Alxv+xhJBwAHAPsD3wew/emcYe1qSevY/jcwqlKhpLuAP+bXKwMHkra3vprvC4IeJTqUIPggtewyGgKsD4wHsP0u8K6kLUk7hiD5DUwhKYTVgOuz7AuSXiPNFm4v1LkyMIy5M4bvA6fafrVyX7eeKghqIGZLQbvQqMFNLTOEEaQdQWdJWhO4C9gL+Ghlq6jt5wuj+vuALSVdACwPrJ3/3l6ocxwpdlFlYXgVAEl/JyXPmWj7mm49WRAEPULMrNqXWhTCAOAzwB62b5N0Csk81BFnAp8C7iRlUruFtJ20yFjg26X3WJk04xgO3CxppO3XijeVcioTOZWD7hAdWxB8kFr8EJYm+RqsmM/XIymElYAxeXawDDDF9qpV7r8F+J7tihPamsBFtlcpyPwqv8fZ+fx64ADbd3TUrth2GgRBUD/d2nZq+5+SnpW0qu3HmOth/DCwE3BM/nsZgKRFSIrmTUmbkPInPFyoslrsoz/l8rMlLUkyIT3VWbvC/htUI0b9QdB1at1ltAfwu7zD6CngO6TAeBdK+i7wDLBtlh0GTJY0hxTn6NulurYDvloqmwxsKulhUhKefW2/XO/DBEEMFIL5SbsNQMJTOQiCoA8RnspBEPQK7TaCbne67Kls+9Z8bR9SPuSlbL8k6ZvAvoXb1wA+Y/teSeNIDm0GZgDfyvecBHwpyy8CDLM9tLsPFwRB79KTA7dQNo2nJpORpHOAm22fUfFUtv1azmlwBikt5trFnMv5vk8Dl9kekXMnzABWy0rgOOAt2xNL9+wBrGV7587aFNFOg6D9iE6+5+luPoSqnsr58knAfuQdRlUo7ihSPgZJepkUHvuJDu45dF7tCoKg/QhT8Fx6Qzl2x1N5I+A52/elZGhV2R7YEsD2e5J2Ax4A3gT+AexeFM75Ej5OSqDTKTGSCIIgaCxd9VSeSJo1bNrRTZI+SzIJPZjPFwB2A9YibV39BSmg3ZGF28YCF9uePa9GxUgiCIJWohUGsbUohOnAdNu35fOLSQrh40BldjAcuFvSurb/meXG8kEHtFEAtp8EkHQhHw6BMZbSrKFIhK4IgqBVqXcQ2xsKpNZF5ZtJ4ScekzQRGGR738L1acDoyqKypH4kZ7X1bT+Vy5YlmZvWsP2ipCNIi9M/ztdXJTmofdw1NCr8EIIgCOqnEX4I1TyVO2N90qzi/fATtmdIOgy4SdJ7pMB34wv3jAMuqEUZQM+ZjFphWhcEQdATtKyncmw7DYL2IwZkPU9beioHQV8kOsygJwmFEAQtROyua11aQZn3q0VI0lBJF0t6VNIjkv5L0ihJUyXdK+lOSetm2RUl/SeX35tzHVTq2V7S/ZIeyp7KlfKPSbpR0j35ejkaahAEQdDD1DpDOAW4xvY2ldAVwIXAYbavzh34cczNsfyk7VHFCiQtQYp5tHbeZXSOpI1sXw8cDFxo+zRJqwF/Blbs5rMFQRAEddDl0BWSTAo/AbAYKU5RZ4wAHrf9Yj6/DtgauJ4U7K6euoIgaENawazSztSSQnMUcDopQ1oxdMXHSH4DIpmePm/7aUkrAg8BjwNvAAfbvlnS4qSwFV8kObv9ARhoe/OcgvMvwOLAIGBj23d11q7YZRQEfZtQHl2js11GtSiE0cBU4AuF0BVvkEbyf7V9iaTtgF1sbyxpQWCw7ZclrU1Kj7m67TckbU4yD80BbgFG2P66pB/ltpwo6b+A3wAjbc8ptaXoqbx2eCoHQXMTnXbz0V2FsDQw1faK+Xw9UsiJLwJDbVspfsXrtodUuX8KsI/tO0vluwAr2d5P0kPAZrafzdeeAj5n+4WO2hUzhCDonOiMg2p0yw/B9j8lPStpVduPkaKcPkxaE9gAmAJsSIpeiqSlgFdsz5Y0AliZ5N2MpGG2X8jmox+Q8itDCnOxEXC2pE8BC5EirAZB0EXafYtqKLzG053QFZcBp+TEN2+TTTmkBejDJc0CZgO72n4lXzslh9AGONz24/n1j4FfS9qbtMA8vtYQFkEQ9E3qUXihPGojQlcEQRDMB5pFKUXoiuB9muVLGQRB81GLH8KqpC2iFUYAPwWWAzYnpdN8EviO7dcK932MtNYw0fYJuWxt4GxgYZLz2V55UXoF4ExgKeAV4Fu2p3f34YIP0+525SCYn7TbAKsuk5Gk/sBzwGeBVYEbbM+SdCyA7f0LspeQtpfeVlAIt5N8GKaSFMLPs6fzRcCVts+RtCFJuXy7s7ZEPoQgCIL66cxkVFMsowIbkcJSPG37L7Zn5fKppKxpAEjairT4/FChbBlgiO1b84LxucBW+fJqJI9lgBvJeZiDIAiC+Ue9awjltJgVdiablSQNAvYHNgH2KcgsR/JQrjA9lwHcRwpjcQrwdWBRSUvYfrmjhoTpIwiCnqDdzED1ULNCyFtOtwAOLJUfBMwCfpeLDgNOsj0z51t+X7RKtRWzzz7AJEnjgZtIZqlZZeHIqRwEQVfoy518PdQzQ/gKcLftf1UKJO0EfA3YqOA38FlgmxzeeigwR9LbwCUUzEr59QxI6TWBb+Q6BwNb23693ADbp5PiKsW20yDo40Qn33jqUQjjKJiLJG1GMg1tYPutSrnt9QoyE4GZtifl839L+hxwG7Aj8ItcviTJu3kOaQZyZlcfKAiCvkG7m417Q+HVpBAkLUJaE/ifQvEkYEHg2mwammp713lUtRtzt51enQ9IeRSOziG1bwJ2r635QRA0MzGKby1a1lM5tp0GQRDUTyO3nQZBEARtSpc9lW2fLGkPYAJpR9BVOZT1JsAxwECSF/O+tm/IZqeLgE+Qgt5dYfuA/B4LkvwS1gZeBra3Pa2zdrW7/TAIguan3UxitYS/fgwYBR/wVL5U0pdIDmRr2H5H0rB8y0vA5rZnSBpJyqpW8Tc4wfaNeQvr9ZK+Yvtq4LvAq7ZXkjQWOBbYvnGPGbQy7fajC4JmpV7HtPc9lSUdDxxj+x2ASjIb2/cU5B8CFpK0YN6JdGOWeVfS3czdhrolMDG/vpjkk6AIgR1A+88GQ+EFzUJ3PJVXAdaTdBQpH8I+tu8oyW8N3FNRGhUkDSUFxjslFy0HPAuQYyO9DixBmm0EQVvTUwovFE1QL93xVB4ALA58DlgHuFDSiMqoXtLqJNPPpqV6BpCUys9tP1UprvKWMTsIgm7Q7jOrYC6NUv7d8VSeDvwxK4DbJc0BlgRelDQcuBTY0faTpXpOB/5h++RC2XRgeWB6VhiLkcJgf4AIXREEQYWYATWeLnsqA38i5VKeImkV0q6il7I56CrgQNt/L1Yg6UhSZ/+9Ut2XAzsBtwLbkMJqf2iGEKErgnYkOragWajJMS1vGX0WGFGJMZRNSGeSdiC9S1pDuEHSwSSz0j8KVWxKUhjPAo8ClTWFSbbPkLQQ8FtgLdLMYGzBnFSVUAhB0DhCKfUdOnNMC0/lIAiCPkRb5lSOBbMgCIJEbywqB0EQtD192XwWCiEIaqAvdxJB36FPKIT4MQdBEMybPqEQYr0hqEYMFILgg/QJhRAE1YiBQmsTCr0HsN1WB7BLyDZPO5pBtlna0QyyzdKOVpNtlnb05PPZbkuFcGfINk87mkG2WdrRDLLN0o5Wk22WdvTk89mOjGlBEARBIhRCEARBALSnQjg9ZJuqHc0g2yztaAbZZmlHq8k2Szt68vlaN5ZREARB0FjacYYQBEEQdIFQCEEQBAEQCiEIgiDItLSnsqRPAlsCy5FyMM8ALrf9SIPqXg64zfbMQvlmtq8pya4L2PYdklYDNgMetf3nGt7nXNs71iD3RWBd4EHbfyld+yzwiO03JC0MHAB8BngY+H/OSY0K8nsCl9p+tob3HQiMBWbYvk7SDsDngUeA022/V5L/BPB1UkrUWaRESeeX2xAEvY2kYbZf6KG6l7D9ck/U3ZO07AxB0v7ABYCA24E78uvzJR1QZ13fKZ3vCVwG7AE8KGnLwuX/V5I9FPg5cJqko4FJwGDgAEkHlWQvLx1XAN+onJdkby+8/n6ud1Hg0CrPdybwVn59CilN6bG57Kwqj3wEcJukmyX9QNJS1T8ZyPf/N7CXpN8C2wK3AesAZ5TavCfwK2ChfH1hkmK4VdKYTt6jZZE0rIfqXaIn6m0UkhaTdIykRyW9nI9HctnQOuq5unQ+RNLRkn6bBx/Fa78snS8t6TRJp0paQtJESQ9IulDSMiXZj5SOJUi54BeX9JGS7Gal5/yNpPsl/V7SR6s8wzGSlsyvR0t6ivT7elrSBiXZuyUdnAdO8/psRku6UdJ5kpaXdK2k1yXdIWmtkuxgSYdLeijLvChpqqTx83qfD1CvJ1uzHMDjwAJVygcC/6izrmdK5w8Ag/PrFYE7gb3y+T1VZPsDiwBvAENy+cLA/SXZu4HzgDHABvnv8/n1BiXZewqv7wCWyq8HAQ+UZB8pvkfp2r1Vnvce0mBgU+A3wIvANaS81ouWZO/PfwcA/wL653NVeb4HCtcXAabk1x+r8rktBhxDSqn6cj4eyWVD6/z/XV06HwIcTUrLukPp2i9L50sDpwGnAksAE/NzXAgsU5L9SOlYApgGLA58pCS7WelZfwPcD/we+GhJ9hhgyfx6NPAU8ATwdPl7UfgeHQx8oobPZjRwY/7eLQ9cC7yev1NrlWQHA4cDD2WZF4GpwPgq9U4G9geWLn2W+wPXlmQ/08GxNvB8SfaS/HlsRcq1fgmwYAff7WtIg7YD8me7f/6u7QFcVpKdA/xf6Xgv/32q/PkWXp8BHAmsAOwN/KnKZ/FA4fWNwDr59SqUvIXz+50APEMayO4NLNvB/+524CukfPbPAtvk8o2AW0uylwHjgeHAj4BDgJWBc0hWgtp+S/X88JrpIHUkK1QpXwF4rEr5/R0cDwDvlGQfrvJDuQb4GaUOlg923PeUrpVl++UvwLXAqFz2VLmtufw+UkezRJUvVfl9LgK+k1+fBYwufCHvqFJ3+Ye1ALAFcD7wYunagyQluzjwb3LHR5oFPFKSfYC5P97FgbuK9ZRka+5Q8rW27VSoo0PJ5b3eqVDlN9bRNWA2cEN+tvLxn3n8Zg4C/k76HZT/d8XfXnlQV65nn/y//nTxc+yg/Xd3Us+9VeQfBQbk11M7+t9WqXs94JfAP/NnsUtJtrPnu6d0fl/p/I78tx/JfF1bv1qrYLMdJDv9E8DVJAeM0/M//AkKo7OC/L+AUflHWTxWJNnHi7I3kDvsQtkA4Fxgdqn8NmCRyodfKF+s/AUuXBtO6sQnlf/RBZlppJHi/+W/S+fywVW+pIsBZwNP5va8l+/5K7Bmlbrvqfae+drCpfO9c11PA3sC1wO/JnX+h5Zk9yJ1qqfnH0lFSS0F3FSSrblDyWVt26lQR4dSpe5e6VSAvwD7UZjtAB8lKdTrSrIPAit38Dk9Wzp/hMLvKJftRJq1PN1Re4Eja/jcKr+7n5HMrx0NxqaTFOKP83dfhWv3V5HfI38eG5JmmCcD6wOHAb/t6H9XKOtP6s/OKpXfSprFb0v6/W2Vyzfgw4PEW4Av5tebA5M7+z11dNQk1KxH/qJ+Dtga2Ca/7t+B7G8qH1iVa7+v8sVZugPZL5TOF+xAbkkKHUcHMv9NHdO5fM8iwMc7uLYosCZp1PzRTupYpc73XJY8AgWG5s963Q5kV8/XPzmPOmvuUPK1tu1U6ulQsnyvdyqkGeCxJGX2KvBK/tyP5cPms22AVTv4nLYqnR8HbFxFbjNKpmCSeWtwFdmVgIs7+e5tTjKF/bOD64eWjoq5dmng3A7uGQP8gWSOfQD4M7ALJbM2cEEdv7s1STPpq4FPktYHX8vf489Xkb09X/9b5fMmDcb2rPk9axWMI45GHqUO5ZVSh7J4Ffm27lQ66VAGVJHtqU5ljVKnskour9qp5Po2Ln9+VJ+hf5JkpuqO7FcaVS9pjW9kI9rbwOerJvupOmVr+n90+H2pVTCOOObXQTY19YR8I2VLnUqvtKG3PguS+fAx4E8k8+aWhWtls1w9snv0kGyPtKGHn29P0oCpobKd/p/r+bLFEcf8OOhgXaUR8u0sOz/bQf078dpStlnaUW+bOzpa2jEtaF0k3d/RJdJaQpfl21m2idrR39lh0/a07GdysaQVsnxfkW2WdtTb5qqEQgh6i48CXyYtSBYRaXGzO/LtLNss7finpFG27wWwPVPS10hOkp/uQ7LN0o5621ydWqcSccTRyIM6dn3VK9/Oss3SDurbide2ss3Sjnrb3NER+RCCIAgCoIVjGQVBEASNJRRCEARBAIRCCIIgCDKhEIIgCAIgFEIQBEGQ+f/d3o4Mm6drCwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# visualize feature matrix\n",
    "sns.heatmap(X_train_indices == 0, vmin=0, vmax=1, cbar=False)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Text-only model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer lstm will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "WARNING:tensorflow:Layer lstm will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "WARNING:tensorflow:Layer lstm will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "Text-only model\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " text_input (InputLayer)        [(None, 30)]         0           []                               \n",
      "                                                                                                  \n",
      " embedding (Embedding)          (None, 30, 300)      12670200    ['text_input[0][0]']             \n",
      "                                                                                                  \n",
      " permute (Permute)              (None, 300, 30)      0           ['embedding[0][0]']              \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, 300, 30)      930         ['permute[0][0]']                \n",
      "                                                                                                  \n",
      " attention (Permute)            (None, 30, 300)      0           ['dense[0][0]']                  \n",
      "                                                                                                  \n",
      " multiply (Multiply)            (None, 30, 300)      0           ['embedding[0][0]',              \n",
      "                                                                  'attention[0][0]']              \n",
      "                                                                                                  \n",
      " bidirectional (Bidirectional)  (None, 30, 60)       79440       ['multiply[0][0]']               \n",
      "                                                                                                  \n",
      " bidirectional_1 (Bidirectional  (None, 60)          21840       ['bidirectional[0][0]']          \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " dense_1 (Dense)                (None, 64)           3904        ['bidirectional_1[0][0]']        \n",
      "                                                                                                  \n",
      " dense_2 (Dense)                (None, 1)            65          ['dense_1[0][0]']                \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 12,776,379\n",
      "Trainable params: 106,179\n",
      "Non-trainable params: 12,670,200\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# model for text features only\n",
    "model = bid_lstm((MAX_LEN,))\n",
    "\n",
    "print('Text-only model')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](arch/bid_lstm.h5.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# early stopping and model checkpoints\n",
    "\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5)\n",
    "bst_model_path = MODEL_PATH + 'bid_lstm_weights.h5'\n",
    "\n",
    "model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 64020 samples, validate on 16006 samples\n",
      "Epoch 1/50\n",
      "64020/64020 [==============================] - ETA: 0s - loss: 0.4263 - accuracy: 0.9098"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\programdata\\anaconda3\\lib\\site-packages\\keras\\engine\\training_v1.py:2057: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
      "  updates = self.state_updates\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64020/64020 [==============================] - 9s 135us/sample - loss: 0.4263 - accuracy: 0.9098 - val_loss: 0.2702 - val_accuracy: 0.9237\n",
      "Epoch 2/50\n",
      "64020/64020 [==============================] - 5s 80us/sample - loss: 0.2558 - accuracy: 0.9300 - val_loss: 0.2697 - val_accuracy: 0.9237\n",
      "Epoch 3/50\n",
      "64020/64020 [==============================] - 5s 79us/sample - loss: 0.2534 - accuracy: 0.9300 - val_loss: 0.2696 - val_accuracy: 0.9237\n",
      "Epoch 4/50\n",
      "64020/64020 [==============================] - 6s 89us/sample - loss: 0.2534 - accuracy: 0.9300 - val_loss: 0.2694 - val_accuracy: 0.9237\n",
      "Epoch 5/50\n",
      "64020/64020 [==============================] - 6s 94us/sample - loss: 0.2535 - accuracy: 0.9300 - val_loss: 0.2698 - val_accuracy: 0.9237\n",
      "Epoch 6/50\n",
      "64020/64020 [==============================] - 7s 113us/sample - loss: 0.2533 - accuracy: 0.9300 - val_loss: 0.2694 - val_accuracy: 0.9237\n",
      "Epoch 7/50\n",
      "64020/64020 [==============================] - 6s 92us/sample - loss: 0.2532 - accuracy: 0.9300 - val_loss: 0.2700 - val_accuracy: 0.9237\n",
      "Epoch 8/50\n",
      "64020/64020 [==============================] - 6s 99us/sample - loss: 0.2531 - accuracy: 0.9300 - val_loss: 0.2702 - val_accuracy: 0.9237\n",
      "Epoch 9/50\n",
      "64020/64020 [==============================] - 7s 105us/sample - loss: 0.2529 - accuracy: 0.9300 - val_loss: 0.2690 - val_accuracy: 0.9237\n",
      "Epoch 10/50\n",
      "64020/64020 [==============================] - 6s 99us/sample - loss: 0.2528 - accuracy: 0.9300 - val_loss: 0.2687 - val_accuracy: 0.9237\n",
      "Epoch 11/50\n",
      "64020/64020 [==============================] - 6s 93us/sample - loss: 0.2525 - accuracy: 0.9300 - val_loss: 0.2687 - val_accuracy: 0.9237\n",
      "Epoch 12/50\n",
      "64020/64020 [==============================] - 6s 90us/sample - loss: 0.2520 - accuracy: 0.9300 - val_loss: 0.2686 - val_accuracy: 0.9237\n",
      "Epoch 13/50\n",
      "64020/64020 [==============================] - 6s 90us/sample - loss: 0.2512 - accuracy: 0.9300 - val_loss: 0.2674 - val_accuracy: 0.9237\n",
      "Epoch 14/50\n",
      "64020/64020 [==============================] - 6s 99us/sample - loss: 0.2507 - accuracy: 0.9300 - val_loss: 0.2665 - val_accuracy: 0.9237\n",
      "Epoch 15/50\n",
      "64020/64020 [==============================] - 7s 103us/sample - loss: 0.2506 - accuracy: 0.9300 - val_loss: 0.2663 - val_accuracy: 0.9237\n",
      "Epoch 16/50\n",
      "64020/64020 [==============================] - 7s 107us/sample - loss: 0.2498 - accuracy: 0.9300 - val_loss: 0.2659 - val_accuracy: 0.9237\n",
      "Epoch 17/50\n",
      "64020/64020 [==============================] - 7s 117us/sample - loss: 0.2490 - accuracy: 0.9300 - val_loss: 0.2664 - val_accuracy: 0.9237\n",
      "Epoch 18/50\n",
      "64020/64020 [==============================] - 6s 99us/sample - loss: 0.2485 - accuracy: 0.9300 - val_loss: 0.2653 - val_accuracy: 0.9237\n",
      "Epoch 19/50\n",
      "64020/64020 [==============================] - 7s 113us/sample - loss: 0.2479 - accuracy: 0.9300 - val_loss: 0.2646 - val_accuracy: 0.9237\n",
      "Epoch 20/50\n",
      "64020/64020 [==============================] - 6s 99us/sample - loss: 0.2475 - accuracy: 0.9300 - val_loss: 0.2643 - val_accuracy: 0.9237\n",
      "Epoch 21/50\n",
      "64020/64020 [==============================] - 7s 102us/sample - loss: 0.2471 - accuracy: 0.9301 - val_loss: 0.2674 - val_accuracy: 0.9237\n",
      "Epoch 22/50\n",
      "64020/64020 [==============================] - 6s 99us/sample - loss: 0.2473 - accuracy: 0.9301 - val_loss: 0.2639 - val_accuracy: 0.9235\n",
      "Epoch 23/50\n",
      "64020/64020 [==============================] - 7s 107us/sample - loss: 0.2471 - accuracy: 0.9303 - val_loss: 0.2637 - val_accuracy: 0.9235\n",
      "Epoch 24/50\n",
      "64020/64020 [==============================] - 7s 107us/sample - loss: 0.2452 - accuracy: 0.9302 - val_loss: 0.2621 - val_accuracy: 0.9235\n",
      "Epoch 25/50\n",
      "64020/64020 [==============================] - 6s 97us/sample - loss: 0.2453 - accuracy: 0.9302 - val_loss: 0.2626 - val_accuracy: 0.9235\n",
      "Epoch 26/50\n",
      "64020/64020 [==============================] - 6s 98us/sample - loss: 0.2446 - accuracy: 0.9304 - val_loss: 0.2622 - val_accuracy: 0.9236\n",
      "Epoch 27/50\n",
      "64020/64020 [==============================] - 7s 102us/sample - loss: 0.2445 - accuracy: 0.9301 - val_loss: 0.2618 - val_accuracy: 0.9237\n",
      "Epoch 28/50\n",
      "64020/64020 [==============================] - 6s 96us/sample - loss: 0.2450 - accuracy: 0.9302 - val_loss: 0.2608 - val_accuracy: 0.9237\n",
      "Epoch 29/50\n",
      "64020/64020 [==============================] - 6s 91us/sample - loss: 0.2437 - accuracy: 0.9303 - val_loss: 0.2593 - val_accuracy: 0.9238\n",
      "Epoch 30/50\n",
      "64020/64020 [==============================] - 7s 105us/sample - loss: 0.2426 - accuracy: 0.9303 - val_loss: 0.2583 - val_accuracy: 0.9237\n",
      "Epoch 31/50\n",
      "64020/64020 [==============================] - 6s 87us/sample - loss: 0.2422 - accuracy: 0.9302 - val_loss: 0.2580 - val_accuracy: 0.9240\n",
      "Epoch 32/50\n",
      "64020/64020 [==============================] - 8s 120us/sample - loss: 0.2423 - accuracy: 0.9304 - val_loss: 0.2604 - val_accuracy: 0.9237\n",
      "Epoch 33/50\n",
      "64020/64020 [==============================] - 7s 108us/sample - loss: 0.2435 - accuracy: 0.9302 - val_loss: 0.2584 - val_accuracy: 0.9237\n",
      "Epoch 34/50\n",
      "64020/64020 [==============================] - 7s 103us/sample - loss: 0.2412 - accuracy: 0.9303 - val_loss: 0.2587 - val_accuracy: 0.9239\n",
      "Epoch 35/50\n",
      "64020/64020 [==============================] - 6s 98us/sample - loss: 0.2431 - accuracy: 0.9301 - val_loss: 0.2569 - val_accuracy: 0.9237\n",
      "Epoch 36/50\n",
      "64020/64020 [==============================] - 6s 97us/sample - loss: 0.2412 - accuracy: 0.9302 - val_loss: 0.2568 - val_accuracy: 0.9239\n",
      "Epoch 37/50\n",
      "64020/64020 [==============================] - 6s 88us/sample - loss: 0.2407 - accuracy: 0.9301 - val_loss: 0.2584 - val_accuracy: 0.9237\n",
      "Epoch 38/50\n",
      "64020/64020 [==============================] - 7s 102us/sample - loss: 0.2412 - accuracy: 0.9302 - val_loss: 0.2585 - val_accuracy: 0.9237\n",
      "Epoch 39/50\n",
      "64020/64020 [==============================] - 7s 103us/sample - loss: 0.2402 - accuracy: 0.9302 - val_loss: 0.2556 - val_accuracy: 0.9237\n",
      "Epoch 40/50\n",
      "64020/64020 [==============================] - 6s 98us/sample - loss: 0.2397 - accuracy: 0.9301 - val_loss: 0.2554 - val_accuracy: 0.9237\n",
      "Epoch 41/50\n",
      "64020/64020 [==============================] - 7s 104us/sample - loss: 0.2402 - accuracy: 0.9302 - val_loss: 0.2648 - val_accuracy: 0.9237\n",
      "Epoch 42/50\n",
      "64020/64020 [==============================] - 6s 99us/sample - loss: 0.2417 - accuracy: 0.9301 - val_loss: 0.2578 - val_accuracy: 0.9238\n",
      "Epoch 43/50\n",
      "64020/64020 [==============================] - 6s 97us/sample - loss: 0.2404 - accuracy: 0.9303 - val_loss: 0.2614 - val_accuracy: 0.9238\n",
      "Epoch 44/50\n",
      "64020/64020 [==============================] - 7s 103us/sample - loss: 0.2396 - accuracy: 0.9302 - val_loss: 0.2546 - val_accuracy: 0.9237\n",
      "Epoch 45/50\n",
      "64020/64020 [==============================] - 7s 109us/sample - loss: 0.2389 - accuracy: 0.9302 - val_loss: 0.2553 - val_accuracy: 0.9238\n",
      "Epoch 46/50\n",
      "64020/64020 [==============================] - 6s 100us/sample - loss: 0.2391 - accuracy: 0.9302 - val_loss: 0.2606 - val_accuracy: 0.9238\n",
      "Epoch 47/50\n",
      "64020/64020 [==============================] - 7s 109us/sample - loss: 0.2395 - accuracy: 0.9302 - val_loss: 0.2582 - val_accuracy: 0.9235\n",
      "Epoch 48/50\n",
      "64020/64020 [==============================] - 6s 100us/sample - loss: 0.2400 - accuracy: 0.9301 - val_loss: 0.2570 - val_accuracy: 0.9237\n",
      "Epoch 49/50\n",
      "64020/64020 [==============================] - 6s 91us/sample - loss: 0.2390 - accuracy: 0.9303 - val_loss: 0.2593 - val_accuracy: 0.9237\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "\n",
    "# text-only model\n",
    "hist = model.fit(X_train_indices, y_train, validation_split=0.2,\n",
    "                 batch_size=2048, epochs=50, callbacks=[early_stopping, model_checkpoint])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preparing sequences from test data\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(X_test)\n",
    "\n",
    "X_test_indices = tokenizer.texts_to_sequences(X_test)\n",
    "\n",
    "# padding sequences to uniform length\n",
    "X_test_indices = pad_sequences(X_test_indices, maxlen=MAX_LEN)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best val score: 0.2545878952232939\n",
      "Accuracy (text only): 92.98%\n"
     ]
    }
   ],
   "source": [
    "# loading model\n",
    "model.load_weights(bst_model_path)\n",
    "bst_val_score = min(hist.history['val_loss'])\n",
    "\n",
    "print('best val score: {}'.format(bst_val_score))\n",
    "\n",
    "# evaluate\n",
    "scores = model.evaluate(X_test_indices, y_test)\n",
    "\n",
    "print('Accuracy (text only): %.2f%%' % (scores[1]*100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\programdata\\anaconda3\\lib\\site-packages\\keras\\engine\\training_v1.py:2079: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
      "  updates=self.state_updates,\n"
     ]
    }
   ],
   "source": [
    "# test\n",
    "predictions = model.predict(X_test_indices)\n",
    "predictions = list(map(lambda x: 1 if x > 0.5 else 0, predictions))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[    2  1372]\n",
      " [   13 18349]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x19f33cc0fa0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAAEGCAYAAADyuIefAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAf+ElEQVR4nO3de7xWZZ338c8XUI6iIuJwUknRBikpGLQsx8KSeqbRZrSwEntyHtTR6WBPjjbPaNkw1TTmpCWG4njMQ6lp5jHNzAYPoARIoagoWxBEPCAisPf+PX+s64a1t/twr82+ufe+9/f9eq3XXvdvna577xc/rmtda12XIgIzM8v0qnYBzMy6EidFM7McJ0UzsxwnRTOzHCdFM7OcPtUuQN7O6hv9GFjtYlgB2mmnahfBCthY/wabGzdqe85x1EcGxivrGsrad/7CTXdHxNTtud6O1qWSYj8GcoimVLsYVkCfvUZUuwhWwP+svn67z7F2XQOP3D2qrH13Gv7M0O2+4A7WpZKimXUHQUM0VrsQFeOkaGaFBNBI7b704aRoZoU14pqimRkAQbDFzWczs0wADTXcfPZzimZWWCNR1tIeSZdLWiNpcS52g6QFaVkuaUGK7ytpY27bJbljJkpaJGmZpAslKcX7pvMtk/SIpH3bK5NrimZWSAANnTe61hXAj4Grtp4/4rOldUnnA6/n9n8mIia0cJ5ZwAzgYeAOYCpwJ3AS8GpE7C9pGvB94LMtHL+Va4pmVlhjmUt7IuJBYF1L21Jt7zPAdW2dQ9JwYHBEzI1sLMSrgGPS5qOBK9P6L4AppVpka5wUzayQIGgocwGGSpqXW2YUuNSHgdUR8XQuNkbSE5J+J+nDKTYSqMvtU5dipW0rACKinqzWuUdbF3Xz2cwKiYAt5bee10bEpA5e6nia1hJXAXtHxCuSJgK/lHQQ0FLNr1TCtra1yEnRzAoSDS3mmk68gtQH+DtgYikWEZuATWl9vqRngAPIaob59w5HASvTeh0wGqhL59yVVprrJW4+m1khATRGect2OBL4c0RsbRZL2lNS77T+LmAs8GxErALWSzo03S+cDtyaDrsNODGtHwvcH+3MweKaopkV1lk1RUnXAUeQ3XusA86NiDnANN7ZwXI4cJ6keqABOCUiSrW+U8l6svuT9TrfmeJzgKslLSOrIU5rr0xOimZWSPbwduckxYg4vpX4F1uI3QTc1Mr+84DxLcTfBo4rUiYnRTMrJIAtUbt33pwUzayQQDTUcHeEk6KZFdYYle19riYnRTMrpDPvKXZFTopmVpBo8D1FM7NMNvK2k6KZGQARYnP0rnYxKsZJ0cwKa/Q9RTOzTNbR4uazmVnijhYzs63c0WJm1kyDH942M8sEYkvUbuqo3W9mZhXhjhYzs5xAbj6bmeW5o8XMLInAj+SYmZVkHS1+zc/MbCt3tJiZJYE8yKyZWV4t1xRr95uZWUVk8z73Kmtpj6TLJa2RtDgX+5akFyUtSMsnc9vOlrRM0lJJR+XiEyUtStsuTPM/I6mvpBtS/BFJ+7ZXJidFMytINJS5lOEKYGoL8QsiYkJa7gCQNI5s3uaD0jEXSyr1+MwCZgBj01I650nAqxGxP3AB8P32CuSkaGaFZFOc9i5rafdcEQ+STVJfjqOB6yNiU0Q8BywDJksaDgyOiLkREcBVwDG5Y65M678AppRqka1xUjSzQiJUpPk8VNK83DKjzMucLmlhal7vnmIjgRW5fepSbGRabx5vckxE1AOvA3u0dWF3tJhZYQUe3l4bEZMKnn4W8B2ySul3gPOBL0GL7fFoI04721rkmqKZFZKNp6iylg6dP2J1RDRERCNwKTA5baoDRud2HQWsTPFRLcSbHCOpD7Ar7TTXnRTNrKBs5O1ylg6dPbtHWPJpoNQzfRswLfUojyHrUHk0IlYB6yUdmu4XTgduzR1zYlo/Frg/3XdslZvPZlZI9khO5zy8Lek64Aiye491wLnAEZImpEstB04GiIgnJd0ILAHqgdMioiGd6lSynuz+wJ1pAZgDXC1pGVkNcVp7ZXJSNLNCOvPd54g4voXwnDb2nwnMbCE+DxjfQvxt4LgiZXJSNLPCPHSYmVmSDR3md5/NzLbygBBmZkk2So6bz2ZmQOk1PydFK2jPEZv5xo9eYPdh9UQj3HHNHvxyzp7VLlaP9ZV/XcjkD73Ma6/uzGnTPgzAF055ikMPX0MEvLZuZy749ntZt7YfR0x9kb8/4bmtx+67/3q+csJhvPj8QM7+3hP8xai3aGwUj/5+GFf8+MBqfaUqck2xwyRNBX4E9AYui4jvVfJ6XUlDvZh93giWLRpA/4EN/Piup3j8wV144el+1S5aj/Sb20dx+437cMa3F26N3XT1GK655AAAPvXZ5Rz/D8v4yffG88BdI3ngruzV2X32W88558/n2acG07dvAzdfM4aF8/egT59GZl78KBM/+DLz/6fn/WfX0bdVuoOKpfs0pM9PgE8A44Dj09A/PcK6NTuxbNEAADZu6M2KZf0YOnxLlUvVcz35xBDWv7FTk9jGDds+9+vfQEvvOfz1USv53d0jANi0qTcL52djCdTX9+KZpYMZOuztyhW6iyr1PpezdEeVrANPBpZFxLMRsRm4nmwYnx5nr1Gb2W/8Rv78+IBqF8WamX7qU1xx+285YupKrvnp2HdsP/xjq/jdPcPfER84aAuHfHgNf3yszQFXalZnDTLbFVWy1K0N89OEpBmlYYW2sKmCxamOfgMa+NfLlnPJOSN4683anQGtu7pq1gF88W8+wgN3jeBTn3mhybYDD3qNTW/35vlndmkS79W7kTNnLuC2G/bhpRd73n90pTlaylm6o0omxbKG7ImI2RExKSIm7UTfChZnx+vdJ/jXy5Zz/82784c7d6t2cawND9w1gg9+9KUmscM/vmpr0znvn765mJUvDOTW68bsqOJ1KQHUR6+ylu6okqVubZifHiI44/wVrHi6HzfP7nk34ruDEaM3bF0/9PDV1C0fuPWzFHxoyioevLdp0/mEU55i4KB6Zv/wL3dYObuiWm4+V7L3+TFgbBri50Wy0Sk+V8HrdSkHTd7Akce9yrNL+nHxvUsB+O/vDuex+wdXuWQ905n/toD3TFzH4N02c+Xt93Pt7LFMOuxlRu6zgWgUa17qx0++u208gfHvW8faNf2aNI/3GLaRaSc9w4rnBnLhNX8A4Fc37sM9t45+x/VqWjduGpdD7Qwttn0nz2bh+i+yR3IuTyNctGqwhsQhmlKx8ljn6zPync1L67r+Z/X1vL559XZltN3fPSw+evmxZe1782Gz5ndg5O2qquhzimkWrjsqeQ0z2/FquaboN1rMrJDOHGS2K3JSNLNCAlHf2D07UcrhpGhmhdXya35OimZWTLj5bGa2le8pmpk1U8tJsXbvlppZRQSiobFXWUt7JF0uaY2kxbnYDyT9WdJCSbdI2i3F95W0UdKCtFySO2aipEWSlkm6MM3/TJoj+oYUf0TSvu2VyUnRzAprRGUtZbgCmNosdi8wPiLeCzwFnJ3b9kxETEjLKbn4LGAGMDYtpXOeBLwaEfsDFwDfb69ATopmVkikjpbOGCUnIh4km6Q+H7snIurTx4fJxk1olaThwOCImBvZK3pXAcekzUcDV6b1XwBTSrXI1jgpmllhESprAYaWhgZMy4yCl/oScGfu8xhJT0j6naQPp9hIsgFoSvLDFG4dwjAl2teBNgfBdEeLmRVUaECItR1991nSvwD1wLUptArYOyJekTQR+KWkg2h7mMKyhjDMc1I0s8Kiwr3Pkk4E/gaYkprERMQmyEaijoj5kp4BDiCrGeab2PlhCktDGNZJ6gPsSrPmenNuPptZIRHQ0Kiylo5IE979M/C3EfFWLr5nmvsJSe8i61B5NiJWAeslHZruF04Hbk2H3QacmNaPBe6PdoYGc03RzArrrNf8JF0HHEF277EOOJest7kvcG/qE3k49TQfDpwnqR5oAE6JiFKt71Synuz+ZPcgS/ch5wBXS1pGVkOc1l6ZnBTNrJCg85rPEXF8C+E5rex7E3BTK9vmAeNbiL8NHFekTE6KZlZQbY+87aRoZoVVcMD+qnNSNLPCKt37XE1OimZWSNb7XLsPrjgpmllhbj6bmeW4+WxmlgRyUjQzy6vh1rOTopkVFBAdfIWvO3BSNLPC3Hw2M8vpkb3Pki6ijVsHEfHlipTIzLq0znz3uStqq6Y4b4eVwsy6jwB6YlKMiCvznyUNjIgNlS+SmXV1tdx8bvddHUkfkLQE+FP6fLCkiyteMjProkQ0lrd0R+W8wPhfwFHAKwAR8UeywR7NrKeKMpduqKze54hY0WxWwIbKFMfMurzouR0tJSskfRAISTsDXyY1pc2sh+qmtcBylNN8PgU4jWz+1BeBCemzmfVYKnPpftqtKUbEWuDzO6AsZtZdNFa7AJVTTu/zuyT9StLLktZIujVNL2hmPVHpOcVylnZIujzllcW52BBJ90p6Ov3cPbftbEnLJC2VdFQuPlHSorTtwjTVKZL6SrohxR+RtG97ZSqn+fwz4EZgODAC+DlwXRnHmVmNiihvKcMVwNRmsbOA+yJiLHBf+oykcWRTlB6Ujrm4NA80MAuYQTYX9NjcOU8CXo2I/YELgO+3V6BykqIi4uqIqE/LNdT0bVYza1cnPZITEQ+SzcecdzRQennkSuCYXPz6iNgUEc8By4DJkoYDgyNibpro/qpmx5TO9QtgSqkW2Zq23n0eklZ/K+ks4Hqyr/lZ4NdtndTMalz5j+QMlZR/ZXh2RMxu55i9ImIVQESskjQsxUcCD+f2q0uxLWm9ebx0zIp0rnpJrwN7AGtbu3hbHS3zyZJg6dufnNsWwHfaONbMapjKbyuujYhJnXXZFmLRRrytY1rV1rvPY9o60Mx6qBBU9hW+1ZKGp1ricGBNitcBo3P7jQJWpvioFuL5Y+ok9QF25Z3N9SbKmqdQ0nhJn5E0vbSUc5yZ1ajKvuZ3G3BiWj8RuDUXn5Z6lMeQdag8mpra6yUdmu4XTm92TOlcxwL3p/uOrWr3OUVJ5wJHAOOAO4BPAA+R3cw0s56ok7paJV1Hll+GSqoDzgW+B9wo6STgBeA4gIh4UtKNwBKgHjgtIkqvHJ9K1pPdH7gzLQBzgKslLSOrIU5rr0zlvOZ3LHAw8ERE/G9JewGXlXGcmdWqTkqKEXF8K5umtLL/TGBmC/F5wPgW4m+Tkmq5ykmKGyOiUVK9pMFk7Xs/vG3WU/XUQWZz5knaDbiUrEf6TeDRShbKzLq2Ar3P3U457z7/Y1q9RNJdZA9JLqxsscysS+uJSVHS+9vaFhGPV6ZIZtbV9dSa4vltbAvgo51cFuuGfv3YHdUughUw+ajXO+dEPfGeYkR8ZEcWxMy6iW481UA5ypqOwMysCSdFM7NtVMODzDopmllxNVxTLGfkbUn6gqRz0ue9JU2ufNHMrCtSlL90R+UMCHEx8AGg9DrOeuAnFSuRmXV9nTQdQVdUTvP5kIh4v6QnACLi1TTVqZn1VN20FliOcpLiljQPQgBI2pOansvLzNrTXZvG5SgnKV4I3AIMkzSTbNSc/1fRUplZ1xU9vPc5Iq6VNJ9sKB8Bx0TEnypeMjPrunpyTVHS3sBbwK/ysYh4oZIFM7MurCcnRbKZ+0qTw/QDxgBLyeZeNbMeqEffU4yI9+Q/p9FzTm5ldzOzbq3wGy0R8bikv6pEYcysm+jJNUVJZ+Q+9gLeD7xcsRKZWdfW03ufgV1y6/Vk9xhvqkxxzKxb6Kk1xfTQ9qCI+MYOKo+ZdXGiczpaJB0I3JALvQs4B9gN+D9sa5F+MyLuSMecDZwENABfjoi7U3wi26Y4vQP4SnvzO7em1XefJfVJc6q2Oi2BmfVQLU1839LS1ikilkbEhIiYAEwke/TvlrT5gtK2XEIcRzZv80HAVODiVHEDmAXMAMamZWpHv1pbNcVHyRLiAkm3AT8HNuS+0M0dvaiZdWOVGQFnCvBMRDwvtTqQxNHA9RGxCXguTXA/WdJysgn15gJIugo4BrizIwUp557iEOAVsjlZSs8rBuCkaNZTld/RMlTSvNzn2RExu4X9pgHX5T6fLmk6MA/4ekS8CowEHs7tU5diW9J683iHtJUUh6We58VsS4YlNXyb1czaU6CmuDYiJrV5rmzUrb8Fzk6hWcB3yPLMd8gm0fsSTXNQSfPclI93SFtJsTcwqLMvaGY1oHMzwCeAxyNiNUDpJ4CkS4Hb08c6YHTuuFHAyhQf1UK8Q9pKiqsi4ryOntjMalTnz+Z3PLmms6ThEbEqffw0WWsV4DbgZ5J+CIwg61B5NCIaJK2XdCjwCDAduKijhWkrKXbPYXPNrOI6q6NF0gDgYzR9dfg/JE0gS73LS9si4klJNwJLyJ6ZPi09IQNwKtseybmTDnayQNtJcUpHT2pmNa6TkmJEvAXs0Sx2Qhv7zwRmthCfB4zvjDK1mhQjYl1nXMDMak9Pf83PzGybzr+n2KU4KZpZIaK2OxycFM2sONcUzcy26dEjb5uZvYOToplZ4kFmzcyacU3RzGwb31M0M8tzUjQz28Y1RTOzkqDIILPdjpOimRXSWRNXdVVOimZWnJOimdk26tjsod2Ck6KZFeNRcszMmvI9RTOzHL/mZ2aW55qimVkStd187lXtAphZNxRlLu2QtFzSIkkLJM1LsSGS7pX0dPq5e27/syUtk7RU0lG5+MR0nmWSLpTU4cHBnRTNrJDSw9vlLGX6SERMiIhJ6fNZwH0RMRa4L31G0jhgGnAQMBW4WFLvdMwsYAbZXNBj0/YOcVI0s8LUGGUtHXQ0cGVavxI4Jhe/PiI2RcRzwDJgsqThwOCImBsRAVyVO6YwJ0UzK6bcpnN5OTGAeyTNlzQjxfaKiFUA6eewFB8JrMgdW5diI9N683iHuKOlk53xwxc45Mj1vLa2Dyd/9EAApn9jFR846g0i4LW1ffjPr+7NutU7Vbmkte/8r43mkd8MZreh9cz+7VIAnlncnwvPGsXmt3vRu09w+nfrePf73uLPTwzgR98YDWT/Sk/4+ksc9onXm5zv3BPHsOqFnbeea3XdTvzwjL15/ZU+7LJbA2de9Dx7jtiyQ79jtRR4JGdo6V5hMjsiZuc+HxYRKyUNA+6V9Oe2LttCLNqId0jFaoqSLpe0RtLiSl2jK7rnhiH8y+fHNIn9YtYwTj3yQP7xYwfyyG8G84Wvra5S6XqWj392HTOvfbZJ7LJ/G84XzniJWb9ZyvRvrGLOv40AYN8DN/Lju5Yy6zdLmXntM/zozFE01G877qE7dqXfwKaZ4NLzRnLkseu45L6lfP5rL/Hf3x1e8e/UZZRfU1wbEZNyy+wmp4lYmX6uAW4BJgOrU5OY9HNN2r0OGJ07fBSwMsVHtRDvkEo2n69gO252dleLHxnE+lebVsDferP31vV+/Rup4ddGu5T3HLqBXXZvaBKTYMP67O+x4Y3eDNkrq9n1GxD0Tn+2LZt6ke+73LihFzf/dE8+99WXmpzr+af6MuFDbwJw8GFvMvfuXSv0TbqezuhokTRQ0i6ldeDjwGLgNuDEtNuJwK1p/TZgmqS+ksaQdag8mprY6yUdmnqdp+eOKaxizeeIeFDSvpU6f3fzxX9exZHHvcqGN3pz5rH7Vbs4PdYp573IN4/fj0vPG0EEXHDb01u3/fnxAZx/xmjW1O3MmRe9sDVJXvkff8Hfn/Iyffs3/Vf+rnFv89Adu/Lpf1jLH+7clbfe7M0b63ozeEjTRFxzAjrpf/a9gFvS0zN9gJ9FxF2SHgNulHQS8AJwHEBEPCnpRmAJUA+cFhGlX/apZBWx/sCdaemQqne0SJohaZ6keVvYVO3iVMwV3x/OFyaN4/6bd+Nvv7S22sXpsW6/cignf/tFrp2/hJO/tZIfnrH31m3vfv9bXPrAUi668ymuv2gYm98Wzyzuz8rn+r7j/iLAjHNeZNHcQfzjxw5g0dxBDB2+md59ekYzQI3lLW2JiGcj4uC0HBQRM1P8lYiYEhFj0891uWNmRsR+EXFgRNyZi8+LiPFp2+mpF7pDqp4UI2J26X7DTvStdnEq7re37M6HPvnOf2C2Y9z78yFbf/+Hf+o1nlow4B377D12E/0GNLJ8aT+WzB/A04sGMH3yOL5+zP68+GxfvvH3+wOwx1/Uc86c5Vx871N88axVAAwcXMMvBScVeE6xS3Hv8w4wYswmVj6XJfxDj3qdFctqP/l3VXvstYWFcwdx8AffZMFDgxgxJmudvPTCzuw5YjO9+2S9ynXP9GOvUZs54OCNfOrEV7J9VuzMOdPH8IOblgHw+iu92WX3Bnr1gusvGsbHP7uu1evWlIjOaj53SU6Kneysi5/nvR94k12H1HPNvCVcff5eTP7oekbtt4nGRljz4s5c+M+j2j+RbbfvnroPC+cO4vV1ffj8xHGc8PWX+OoPVjDrnJE0NIid+zby1R9kj70tfnQgN/x4DH36QK9ewT/9ex277tH2vcGFcwdx+XdHIAXvOWQDp/17XZv715LuWgssh7aj6d32iaXrgCOAocBq4NyImNPWMYM1JA7RlIqUxyrj7pULql0EK2DyUSuY98e3O/xeMMAuu42K9x3+lbL2/f2vzpyfe32vW6hk7/PxlTq3mVVXLdcU3Xw2s2ICaKjdrOikaGaFuaZoZpbn3mczs21cUzQzK/EUp2Zm2wiQO1rMzLaR7ymamSVuPpuZ5fndZzOzJtz7bGaW55qimVkS7n02M2uqdnOik6KZFedHcszM8pwUzcySAGp4KpqqT1xlZt2LCBTlLW2eRxot6beS/iTpSUlfSfFvSXpR0oK0fDJ3zNmSlklaKumoXHyipEVp24Vp/ucOcU3RzIpr7JSqYj3w9Yh4XNIuwHxJ96ZtF0TEf+Z3ljQOmAYcBIwAfiPpgDT38yxgBvAwcAcwlQ7O/eyaopkVU2o+l7O0dZqIVRHxeFpfD/wJGNnGIUcD10fEpoh4DlgGTJY0HBgcEXPTfM9XAcd09Os5KZpZYQWaz0MlzcstM1o8n7Qv8D7gkRQ6XdJCSZdL2j3FRgIrcofVpdjItN483iFOimZWXGnu5/YWWBsRk3LL7OankjQIuAn4akS8QdYU3g+YAKwCzi/t2lJJ2oh3iO8pmllBnTcghKSdyBLitRFxM0BErM5tvxS4PX2sA0bnDh8FrEzxUS3EO8Q1RTMrpjSbXzlLG1IP8RzgTxHxw1x8eG63TwOL0/ptwDRJfSWNAcYCj0bEKmC9pEPTOacDt3b067mmaGaFddIbLYcBJwCLJC1IsW8Cx0uaQJZ+lwMnA0TEk5JuBJaQ9VyflnqeAU4FrgD6k/U6d6jnGZwUzawjOiEpRsRDtHw/8I42jpkJzGwhPg8Yv92FwknRzIoKoNGv+ZmZJR5528ysKSdFM7MkgIbaHRHCSdHMCgoIJ0Uzs23cfDYzS9z7bGbWjGuKZmY5TopmZkkENDS0v1835aRoZsW5pmhmluOkaGZWEu59NjPbKiD88LaZWY5f8zMzSyI6a4rTLslJ0cyKc0eLmdk24ZqimVmJB5k1M9vGA0KYmW0TQPg1PzOzJDzIrJlZE+Hms5lZTg3XFBVdqBdJ0svA89UuRwUMBdZWuxBWSK3+zfaJiD235wSS7iL7/ZRjbURM3Z7r7WhdKinWKknzImJStcth5fPfrOfqVe0CmJl1JU6KZmY5Too7xuxqF8AK89+sh/I9RTOzHNcUzcxynBTNzHKcFCtI0lRJSyUtk3RWtctj7ZN0uaQ1khZXuyxWHU6KFSKpN/AT4BPAOOB4SeOqWyorwxVAt3rY2DqXk2LlTAaWRcSzEbEZuB44usplsnZExIPAumqXw6rHSbFyRgIrcp/rUszMujAnxcpRCzE//2TWxTkpVk4dMDr3eRSwskplMbMyOSlWzmPAWEljJO0MTANuq3KZzKwdTooVEhH1wOnA3cCfgBsj4snqlsraI+k6YC5woKQ6SSdVu0y2Y/k1PzOzHNcUzcxynBTNzHKcFM3McpwUzcxynBTNzHKcFLsRSQ2SFkhaLOnnkgZsx7mukHRsWr+srcEqJB0h6YMduMZySe+Y9a21eLN93ix4rW9J+r9Fy2jWnJNi97IxIiZExHhgM3BKfmMamaewiPiHiFjSxi5HAIWToll35KTYff0e2D/V4n4r6WfAIkm9Jf1A0mOSFko6GUCZH0taIunXwLDSiSQ9IGlSWp8q6XFJf5R0n6R9yZLv11It9cOS9pR0U7rGY5IOS8fuIekeSU9I+iktv//dhKRfSpov6UlJM5ptOz+V5T5Je6bYfpLuSsf8XtK7O+W3aZb0qXYBrDhJfcjGabwrhSYD4yPiuZRYXo+Iv5LUF/iDpHuA9wEHAu8B9gKWAJc3O++ewKXA4elcQyJinaRLgDcj4j/Tfj8DLoiIhyTtTfbWzl8C5wIPRcR5kv4X0CTJteJL6Rr9gcck3RQRrwADgccj4uuSzknnPp1sQqlTIuJpSYcAFwMf7cCv0axFTordS39JC9L674E5ZM3aRyPiuRT/OPDe0v1CYFdgLHA4cF1ENAArJd3fwvkPBR4snSsiWhtX8EhgnLS1IjhY0i7pGn+Xjv21pFfL+E5flvTptD46lfUVoBG4IcWvAW6WNCh935/nrt23jGuYlc1JsXvZGBET8oGUHDbkQ8A/RcTdzfb7JO0PXaYy9oHstssHImJjC2Up+71RSUeQJdgPRMRbkh4A+rWye6Trvtb8d2DWmXxPsfbcDZwqaScASQdIGgg8CExL9xyHAx9p4di5wF9LGpOOHZLi64FdcvvdQ9aUJe03Ia0+CHw+xT4B7N5OWXcFXk0J8d1kNdWSXkCptvs5smb5G8Bzko5L15Ckg9u5hlkhToq15zKy+4WPp8mXfkrWIrgFeBpYBMwCftf8wIh4mew+4M2S/si25uuvgE+XOlqALwOTUkfOErb1gn8bOFzS42TN+BfaKetdQB9JC4HvAA/ntm0ADpI0n+ye4Xkp/nngpFS+J/EUD9bJPEqOmVmOa4pmZjlOimZmOU6KZmY5TopmZjlOimZmOU6KZmY5TopmZjn/H3kdAhFT1L5GAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# confusion matrix\n",
    "conf_matrix = confusion_matrix(y_test, predictions)\n",
    "print(conf_matrix)\n",
    "\n",
    "ConfusionMatrixDisplay(conf_matrix).plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Text + additional features model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "WARNING:tensorflow:Layer lstm_3 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "WARNING:tensorflow:Layer lstm_3 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "WARNING:tensorflow:Layer lstm_3 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
      "Text + additional features model\n",
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " text_input (InputLayer)        [(None, 30)]         0           []                               \n",
      "                                                                                                  \n",
      " embedding (Embedding)          (None, 30, 300)      12670200    ['text_input[0][0]']             \n",
      "                                                                                                  \n",
      " permute_1 (Permute)            (None, 300, 30)      0           ['embedding[1][0]']              \n",
      "                                                                                                  \n",
      " dense_3 (Dense)                (None, 300, 30)      930         ['permute_1[0][0]']              \n",
      "                                                                                                  \n",
      " attention (Permute)            (None, 30, 300)      0           ['dense_3[0][0]']                \n",
      "                                                                                                  \n",
      " multiply_1 (Multiply)          (None, 30, 300)      0           ['embedding[1][0]',              \n",
      "                                                                  'attention[0][0]']              \n",
      "                                                                                                  \n",
      " bidirectional_2 (Bidirectional  (None, 30, 60)      79440       ['multiply_1[0][0]']             \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " feat_input (InputLayer)        [(None, 4)]          0           []                               \n",
      "                                                                                                  \n",
      " bidirectional_3 (Bidirectional  (None, 60)          21840       ['bidirectional_2[0][0]']        \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " activation (Activation)        (None, 4)            0           ['feat_input[0][0]']             \n",
      "                                                                                                  \n",
      " concatenate (Concatenate)      (None, 64)           0           ['bidirectional_3[0][0]',        \n",
      "                                                                  'activation[0][0]']             \n",
      "                                                                                                  \n",
      " dense_4 (Dense)                (None, 64)           4160        ['concatenate[0][0]']            \n",
      "                                                                                                  \n",
      " dense_5 (Dense)                (None, 1)            65          ['dense_4[0][0]']                \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 12,776,635\n",
      "Trainable params: 106,435\n",
      "Non-trainable params: 12,670,200\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# model for text + additional features\n",
    "model = bid_lstm((MAX_LEN,), (feat_train.shape[1],))\n",
    "\n",
    "print('Text + additional features model')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](arch/bid_lstm_feats.h5.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# early stopping and model checkpoints\n",
    "\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5)\n",
    "bst_model_path = MODEL_PATH + 'bid_lstm_feats_weights.h5'\n",
    "\n",
    "model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 64020 samples, validate on 16006 samples\n",
      "Epoch 1/50\n",
      "64020/64020 [==============================] - ETA: 0s - loss: 0.4594 - accuracy: 0.7856"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\programdata\\anaconda3\\lib\\site-packages\\keras\\engine\\training_v1.py:2057: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
      "  updates = self.state_updates\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64020/64020 [==============================] - 8s 129us/sample - loss: 0.4594 - accuracy: 0.7856 - val_loss: 0.2836 - val_accuracy: 0.9237\n",
      "Epoch 2/50\n",
      "64020/64020 [==============================] - 5s 82us/sample - loss: 0.2562 - accuracy: 0.9300 - val_loss: 0.2684 - val_accuracy: 0.9237\n",
      "Epoch 3/50\n",
      "64020/64020 [==============================] - 5s 85us/sample - loss: 0.2503 - accuracy: 0.9300 - val_loss: 0.2634 - val_accuracy: 0.9237\n",
      "Epoch 4/50\n",
      "64020/64020 [==============================] - 6s 87us/sample - loss: 0.2463 - accuracy: 0.9300 - val_loss: 0.2597 - val_accuracy: 0.9237\n",
      "Epoch 5/50\n",
      "64020/64020 [==============================] - 6s 89us/sample - loss: 0.2428 - accuracy: 0.9300 - val_loss: 0.2558 - val_accuracy: 0.9237\n",
      "Epoch 6/50\n",
      "64020/64020 [==============================] - 6s 96us/sample - loss: 0.2397 - accuracy: 0.9300 - val_loss: 0.2530 - val_accuracy: 0.9237\n",
      "Epoch 7/50\n",
      "64020/64020 [==============================] - 6s 88us/sample - loss: 0.2373 - accuracy: 0.9300 - val_loss: 0.2498 - val_accuracy: 0.9237\n",
      "Epoch 8/50\n",
      "64020/64020 [==============================] - 5s 85us/sample - loss: 0.2354 - accuracy: 0.9300 - val_loss: 0.2482 - val_accuracy: 0.9237\n",
      "Epoch 9/50\n",
      "64020/64020 [==============================] - 7s 105us/sample - loss: 0.2340 - accuracy: 0.9300 - val_loss: 0.2468 - val_accuracy: 0.9237\n",
      "Epoch 10/50\n",
      "64020/64020 [==============================] - 6s 98us/sample - loss: 0.2336 - accuracy: 0.9300 - val_loss: 0.2469 - val_accuracy: 0.9237\n",
      "Epoch 11/50\n",
      "64020/64020 [==============================] - 7s 103us/sample - loss: 0.2326 - accuracy: 0.9300 - val_loss: 0.2456 - val_accuracy: 0.9237\n",
      "Epoch 12/50\n",
      "64020/64020 [==============================] - 7s 102us/sample - loss: 0.2321 - accuracy: 0.9300 - val_loss: 0.2457 - val_accuracy: 0.9237\n",
      "Epoch 13/50\n",
      "64020/64020 [==============================] - 7s 106us/sample - loss: 0.2316 - accuracy: 0.9300 - val_loss: 0.2442 - val_accuracy: 0.9237\n",
      "Epoch 14/50\n",
      "64020/64020 [==============================] - 6s 97us/sample - loss: 0.2314 - accuracy: 0.9300 - val_loss: 0.2443 - val_accuracy: 0.9237\n",
      "Epoch 15/50\n",
      "64020/64020 [==============================] - 6s 100us/sample - loss: 0.2311 - accuracy: 0.9300 - val_loss: 0.2434 - val_accuracy: 0.9237\n",
      "Epoch 16/50\n",
      "64020/64020 [==============================] - 6s 99us/sample - loss: 0.2307 - accuracy: 0.9300 - val_loss: 0.2435 - val_accuracy: 0.9237\n",
      "Epoch 17/50\n",
      "64020/64020 [==============================] - 7s 109us/sample - loss: 0.2304 - accuracy: 0.9300 - val_loss: 0.2430 - val_accuracy: 0.9237\n",
      "Epoch 18/50\n",
      "64020/64020 [==============================] - 7s 109us/sample - loss: 0.2298 - accuracy: 0.9300 - val_loss: 0.2423 - val_accuracy: 0.9237\n",
      "Epoch 19/50\n",
      "64020/64020 [==============================] - 6s 99us/sample - loss: 0.2298 - accuracy: 0.9300 - val_loss: 0.2423 - val_accuracy: 0.9237\n",
      "Epoch 20/50\n",
      "64020/64020 [==============================] - 6s 86us/sample - loss: 0.2294 - accuracy: 0.9300 - val_loss: 0.2419 - val_accuracy: 0.9237\n",
      "Epoch 21/50\n",
      "64020/64020 [==============================] - 7s 116us/sample - loss: 0.2287 - accuracy: 0.9300 - val_loss: 0.2412 - val_accuracy: 0.9237\n",
      "Epoch 22/50\n",
      "64020/64020 [==============================] - 7s 107us/sample - loss: 0.2285 - accuracy: 0.9300 - val_loss: 0.2409 - val_accuracy: 0.9237\n",
      "Epoch 23/50\n",
      "64020/64020 [==============================] - 7s 114us/sample - loss: 0.2280 - accuracy: 0.9300 - val_loss: 0.2408 - val_accuracy: 0.9237\n",
      "Epoch 24/50\n",
      "64020/64020 [==============================] - 7s 113us/sample - loss: 0.2278 - accuracy: 0.9300 - val_loss: 0.2406 - val_accuracy: 0.9237\n",
      "Epoch 25/50\n",
      "64020/64020 [==============================] - 7s 109us/sample - loss: 0.2277 - accuracy: 0.9300 - val_loss: 0.2417 - val_accuracy: 0.9237\n",
      "Epoch 26/50\n",
      "64020/64020 [==============================] - 7s 105us/sample - loss: 0.2274 - accuracy: 0.9300 - val_loss: 0.2398 - val_accuracy: 0.9237\n",
      "Epoch 27/50\n",
      "64020/64020 [==============================] - 7s 105us/sample - loss: 0.2270 - accuracy: 0.9300 - val_loss: 0.2394 - val_accuracy: 0.9237\n",
      "Epoch 28/50\n",
      "64020/64020 [==============================] - 7s 108us/sample - loss: 0.2268 - accuracy: 0.9300 - val_loss: 0.2392 - val_accuracy: 0.9237\n",
      "Epoch 29/50\n",
      "64020/64020 [==============================] - 6s 100us/sample - loss: 0.2268 - accuracy: 0.9300 - val_loss: 0.2396 - val_accuracy: 0.9237\n",
      "Epoch 30/50\n",
      "64020/64020 [==============================] - 7s 104us/sample - loss: 0.2265 - accuracy: 0.9300 - val_loss: 0.2385 - val_accuracy: 0.9237\n",
      "Epoch 31/50\n",
      "64020/64020 [==============================] - 7s 109us/sample - loss: 0.2255 - accuracy: 0.9300 - val_loss: 0.2389 - val_accuracy: 0.9237\n",
      "Epoch 32/50\n",
      "64020/64020 [==============================] - 6s 88us/sample - loss: 0.2244 - accuracy: 0.9301 - val_loss: 0.2380 - val_accuracy: 0.9237\n",
      "Epoch 33/50\n",
      "64020/64020 [==============================] - 7s 103us/sample - loss: 0.2237 - accuracy: 0.9302 - val_loss: 0.2372 - val_accuracy: 0.9237\n",
      "Epoch 34/50\n",
      "64020/64020 [==============================] - 6s 100us/sample - loss: 0.2230 - accuracy: 0.9304 - val_loss: 0.2361 - val_accuracy: 0.9238\n",
      "Epoch 35/50\n",
      "64020/64020 [==============================] - 6s 101us/sample - loss: 0.2231 - accuracy: 0.9303 - val_loss: 0.2357 - val_accuracy: 0.9238\n",
      "Epoch 36/50\n",
      "64020/64020 [==============================] - 6s 101us/sample - loss: 0.2220 - accuracy: 0.9303 - val_loss: 0.2381 - val_accuracy: 0.9237\n",
      "Epoch 37/50\n",
      "64020/64020 [==============================] - 6s 93us/sample - loss: 0.2224 - accuracy: 0.9303 - val_loss: 0.2355 - val_accuracy: 0.9239\n",
      "Epoch 38/50\n",
      "64020/64020 [==============================] - 7s 108us/sample - loss: 0.2211 - accuracy: 0.9304 - val_loss: 0.2342 - val_accuracy: 0.9238\n",
      "Epoch 39/50\n",
      "64020/64020 [==============================] - 7s 106us/sample - loss: 0.2219 - accuracy: 0.9304 - val_loss: 0.2340 - val_accuracy: 0.9234\n",
      "Epoch 40/50\n",
      "64020/64020 [==============================] - 6s 89us/sample - loss: 0.2212 - accuracy: 0.9304 - val_loss: 0.2339 - val_accuracy: 0.9237\n",
      "Epoch 41/50\n",
      "64020/64020 [==============================] - 6s 101us/sample - loss: 0.2208 - accuracy: 0.9304 - val_loss: 0.2342 - val_accuracy: 0.9238\n",
      "Epoch 42/50\n",
      "64020/64020 [==============================] - 6s 94us/sample - loss: 0.2216 - accuracy: 0.9305 - val_loss: 0.2340 - val_accuracy: 0.9237\n",
      "Epoch 43/50\n",
      "64020/64020 [==============================] - 7s 107us/sample - loss: 0.2198 - accuracy: 0.9305 - val_loss: 0.2326 - val_accuracy: 0.9233\n",
      "Epoch 44/50\n",
      "64020/64020 [==============================] - 7s 110us/sample - loss: 0.2206 - accuracy: 0.9305 - val_loss: 0.2321 - val_accuracy: 0.9235\n",
      "Epoch 45/50\n",
      "64020/64020 [==============================] - 7s 103us/sample - loss: 0.2192 - accuracy: 0.9305 - val_loss: 0.2345 - val_accuracy: 0.9237\n",
      "Epoch 46/50\n",
      "64020/64020 [==============================] - 6s 89us/sample - loss: 0.2196 - accuracy: 0.9304 - val_loss: 0.2333 - val_accuracy: 0.9238\n",
      "Epoch 47/50\n",
      "64020/64020 [==============================] - 7s 106us/sample - loss: 0.2186 - accuracy: 0.9305 - val_loss: 0.2320 - val_accuracy: 0.9238\n",
      "Epoch 48/50\n",
      "64020/64020 [==============================] - 6s 93us/sample - loss: 0.2187 - accuracy: 0.9306 - val_loss: 0.2308 - val_accuracy: 0.9239\n",
      "Epoch 49/50\n",
      "64020/64020 [==============================] - 6s 100us/sample - loss: 0.2175 - accuracy: 0.9305 - val_loss: 0.2309 - val_accuracy: 0.9236\n",
      "Epoch 50/50\n",
      "64020/64020 [==============================] - 6s 92us/sample - loss: 0.2176 - accuracy: 0.9306 - val_loss: 0.2325 - val_accuracy: 0.9239\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "\n",
    "# text + additional features model\n",
    "hist_text_feat = model.fit([X_train_indices, feat_train], y_train, validation_split=0.2,\n",
    "                           batch_size=2048, epochs=50, callbacks=[early_stopping, model_checkpoint])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preparing sequences from test data\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(X_test)\n",
    "\n",
    "X_test_indices = tokenizer.texts_to_sequences(X_test)\n",
    "\n",
    "# padding sequences to uniform length\n",
    "X_test_indices = pad_sequences(X_test_indices, maxlen=MAX_LEN)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best val score: 0.2545878952232939\n",
      "Accuracy (text + additional features): 93.04%\n"
     ]
    }
   ],
   "source": [
    "# loading model (text + additional features)\n",
    "model.load_weights(bst_model_path)\n",
    "bst_val_score = min(hist.history['val_loss'])\n",
    "\n",
    "print('best val score: {}'.format(bst_val_score))\n",
    "\n",
    "# evaluate\n",
    "scores = model.evaluate([X_test_indices, feat_test], y_test)\n",
    "\n",
    "print('Accuracy (text + additional features): %.2f%%' % (scores[1]*100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\programdata\\anaconda3\\lib\\site-packages\\keras\\engine\\training_v1.py:2079: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
      "  updates=self.state_updates,\n"
     ]
    }
   ],
   "source": [
    "# test\n",
    "predictions = model.predict([X_test_indices, feat_test])\n",
    "predictions = list(map(lambda x: 1 if x > 0.5 else 0, predictions))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[    2  1372]\n",
      " [    2 18360]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x19f309588b0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAAEGCAYAAADyuIefAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAf5ElEQVR4nO3de7xWZZ338c+Xg5wEAUGHk0qFltJEQUSZjmUlHWa0ebRgGrXJeVAfS2fsqSebScvi6eCYk5kYiqNmoZaWZIo6mpmFB0ACT8hWCbYgiCgSIrD3/s0f67phsd2He232zb0P3/frtV6s9Vun64aXP69rXWtdlyICMzPL9Kh2AczMOhInRTOzHCdFM7McJ0UzsxwnRTOznF7VLkDePuoTfRlQ7WJYAerdu9pFsAK21r3K9oat2pNrHPeBAfHSxvqyjl20dNudETF1T+63t3WopNiXAbxHx1a7GFZArwNHVrsIVsAf192wx9fYsLGeh+4cXdaxvUc8M2yPb7iXdaikaGadQVAfDdUuRMU4KZpZIQE00HU/+nBSNLPCGnBN0cwMgCDY4eazmVkmgPou3Hz2e4pmVlgDUdbSGklXS1ov6bFc7EZJS9KyUtKSFD9E0tbcvity50yUtExSjaRLJSnF+6Tr1Uh6SNIhrZXJNUUzKySA+vYbXesa4DLgup3Xj/h0aV3SxcCm3PHPRMSEJq4zC5gBPAjcDkwF7gBOA16OiLdImgZ8F/h0E+fv5JqimRXWUObSmoi4H9jY1L5U2/sUMLela0gaAQyKiAWRjYV4HXBC2n08cG1a/wVwbKkW2RwnRTMrJAjqy1z20FHAuohYkYuNlfSopN9JOirFRgG1uWNqU6y0bzVARNSR1Tr3b+mmbj6bWSERsKP8fDdM0sLc9uyImF3mudPZvZa4FjgoIl6SNBH4laQjgKZqfqUStrSvSU6KZlaQqG8y1zRpQ0RMKnwHqRfw98DEUiwitgHb0voiSc8Ah5LVDPPfHY4G1qT1WmAMUJuuuR/NNNdL3Hw2s0ICaIjylj3wIeCpiNjZLJY0XFLPtP4mYBzwbESsBTZLmpKeF54C3JpOmwecmtZPBO6NVuZgcU3RzAorUFNskaS5wDFkzexa4IKImANM440dLEcDF0qqA+qBMyKiVOs7k6wnux9Zr/MdKT4H+ImkGrIa4rTWyuSkaGaFZC9vt09SjIjpzcQ/20TsZuDmZo5fCIxvIv46cFKRMjkpmlkhAeyIrvvkzUnRzAoJRH0X7o5wUjSzwhqifZrPHZGTopkV0p7PFDsiJ0UzK0jU+5mimVkmG3nbSdHMDIAIsT16VrsYFeOkaGaFNfiZoplZJutocfPZzCxxR4uZ2U7uaDEza6TeL2+bmWUCsSO6burour/MzCrCHS1mZjmB3Hw2M8tzR4uZWRKBX8kxMyvJOlr8mZ+Z2U7uaDEzSwJ5kFkzs7yuXFPsur/MzCoim/e5R1lLayRdLWm9pMdysa9Lel7SkrR8LLfvPEk1kpZLOi4XnyhpWdp3aZr/GUl9JN2Y4g9JOqS1MjkpmllBor7MpQzXAFObiF8SERPScjuApMPJ5m0+Ip1zuaRSj88sYAYwLi2la54GvBwRbwEuAb7bWoGcFM2skGyK055lLa1eK+J+sknqy3E8cENEbIuI54AaYLKkEcCgiFgQEQFcB5yQO+fatP4L4NhSLbI5TopmVkiEijSfh0lamFtmlHmbz0tamprXQ1JsFLA6d0xtio1K643ju50TEXXAJmD/lm7sjhYzK6zAy9sbImJSwcvPAr5JVin9JnAx8Dlosj0eLcRpZV+TXFM0s0Ky8RRV1tKm60esi4j6iGgArgQmp121wJjcoaOBNSk+uon4budI6gXsRyvNdSdFMysoG3m7nKVNV8+eEZZ8Eij1TM8DpqUe5bFkHSoPR8RaYLOkKel54SnArblzTk3rJwL3pueOzXLz2cwKyV7JaZ+XtyXNBY4he/ZYC1wAHCNpQrrVSuB0gIh4XNJNwBNAHXBWRNSnS51J1pPdD7gjLQBzgJ9IqiGrIU5rrUxOimZWSHt++xwR05sIz2nh+JnAzCbiC4HxTcRfB04qUiYnRTMrzEOHmZkl2dBh/vbZzGwnDwhhZpZko+S4+WxmBpQ+83NStIKGj9zOl36wiiEH1BENcPv1+/OrOcOrXaxu65yvLWXy+1/klZf34axpRwHwj2c8zZSj1xMBr2zch0u+8dds3NCXY6Y+z/86+bmd5x7yls2cc/KRPP/nAZz3nUf5q9Gv0dAgHv79AVxz2WHV+klV5Jpim0maCvwA6AlcFRHfqeT9OpL6OjH7wpHULOtPvwH1XDb/aRbfP5BVK/pWu2jd0n/fNprbbjqYc7+xdGfs5p+M5forDgXgbz+9kun/XMOPvjOe++aP4r752aezB795M+dfvIhnnx5Enz713HL9WJYu2p9evRqYefnDTHzfiyz6Y/f7n11bv1bpDCqW7tOQPj8CPgocDkxPQ/90CxvX96ZmWX8Atm7pyeqavgwbsaPKpeq+Hn90KJtf7b1bbOuWXdt9+9XT1HcOf3PcGn5350gAtm3rydJF2VgCdXU9eGb5IIYd8HrlCt1BlXqfy1k6o0rWgScDNRHxbERsB24gG8an2zlw9HbePH4rTy3uX+2iWCOnnPk019z2W46ZuobrfzzuDfuP/vBafnfXiDfEB+y7g/cctZ4/PdLigCtdVnsNMtsRVbLUzQ3zsxtJM0rDCu1gWwWLUx19+9fztatWcsX5I3ntL113BrTO6rpZh/LZT3yA++aP5G8/tWq3fYcd8QrbXu/Jn58ZuFu8R88GvjxzCfNuPJgXnu9+/6MrzdFSztIZVTIpljVkT0TMjohJETGpN30qWJy9r2ev4GtXreTeW4bwhzsGV7s41oL75o/kfR98YbfY0R9Zu7PpnPeFrz7GmlUDuHXu2L1VvA4lgLroUdbSGVWy1M0N89NNBOdevJrVK/pyy+zu9yC+Mxg5ZsvO9SlHr6N25YCd21Lw/mPXcv/duzedTz7jaQbsW8fs779tr5WzI+rKzedK9j4/AoxLQ/w8TzY6xT9U8H4dyhGTt/Chk17m2Sf6cvndywH4r2+P4JF7B1W5ZN3Tl7+1hLdP3Migwdu59rZ7+enscUw68kVGHbyFaBDrX+jLj769azyB8e/cyIb1fXdrHu9/wFamnfYMq58bwKXX/wGAX990MHfdOuYN9+vSOnHTuBxqZWixPbt4NgvXf5K9knN1GuGiWYM0NN6jYytWHmt/vUa9sXlpHdcf193Apu3r9iijDXnrAfHBq08s69hbjpy1qA0jb1dVRd9TTLNw3V7Je5jZ3teVa4r+osXMCmnPQWY7IidFMyskEHUNnbMTpRxOimZWWFf+zM9J0cyKCTefzcx28jNFM7NGunJS7LpPS82sIgJR39CjrKU1kq6WtF7SY7nYRZKekrRU0i8lDU7xQyRtlbQkLVfkzpkoaZmkGkmXpvmfSXNE35jiD0k6pLUyOSmaWWENqKylDNcAUxvF7gbGR8RfA08D5+X2PRMRE9JyRi4+C5gBjEtL6ZqnAS9HxFuAS4DvtlYgJ0UzKyRSR0t7jJITEfeTTVKfj90VEXVp80GycROaJWkEMCgiFkT2id51wAlp9/HAtWn9F8CxpVpkc5wUzaywCJW1AMNKQwOmZUbBW30OuCO3PVbSo5J+J+moFBtFNgBNSX6Ywp1DGKZEuwlocRBMd7SYWUGFBoTY0NZvnyX9G1AH/DSF1gIHRcRLkiYCv5J0BC0PU1jWEIZ5TopmVlhUuPdZ0qnAJ4BjU5OYiNgG2UjUEbFI0jPAoWQ1w3wTOz9MYWkIw1pJvYD9aNRcb8zNZzMrJALqG1TW0hZpwrv/B/xdRLyWiw9Pcz8h6U1kHSrPRsRaYLOkKel54SnArem0ecCpaf1E4N5oZWgw1xTNrLD2+sxP0lzgGLJnj7XABWS9zX2Au1OfyIOpp/lo4EJJdUA9cEZElGp9Z5L1ZPcjewZZeg45B/iJpBqyGuK01srkpGhmhQTt13yOiOlNhOc0c+zNwM3N7FsIjG8i/jpwUpEyOSmaWUFde+RtJ0UzK6yCA/ZXnZOimRVW6d7nanJSNLNCst7nrvviipOimRXm5rOZWY6bz2ZmSSAnRTOzvC7cenZSNLOCAqKNn/B1Bk6KZlaYm89mZjndsvdZ0g9p4dFBRJxdkRKZWYfWnt8+d0Qt1RQX7rVSmFnnEUB3TIoRcW1+W9KAiNhS+SKZWUfXlZvPrX6rI+m9kp4Ankzb75B0ecVLZmYdlIiG8pbOqJwPGP8TOA54CSAi/kQ22KOZdVdR5tIJldX7HBGrG80KWF+Z4phZhxfdt6OlZLWk9wEhaR/gbFJT2sy6qU5aCyxHOc3nM4CzyOZPfR6YkLbNrNtSmUvn02pNMSI2AJ/ZC2Uxs86iodoFqJxyep/fJOnXkl6UtF7SrWl6QTPrjkrvKZaztELS1SmvPJaLDZV0t6QV6c8huX3nSaqRtFzScbn4REnL0r5L01SnSOoj6cYUf0jSIa2VqZzm88+Am4ARwEjg58DcMs4zsy4qorylDNcAUxvFvgLcExHjgHvSNpIOJ5ui9Ih0zuWleaCBWcAMsrmgx+WueRrwckS8BbgE+G5rBSonKSoifhIRdWm5ni79mNXMWtVOr+RExP1k8zHnHQ+UPh65FjghF78hIrZFxHNADTBZ0ghgUEQsSBPdX9fonNK1fgEcW6pFNqelb5+HptXfSvoKcAPZz/w08JuWLmpmXVz5r+QMk5T/ZHh2RMxu5ZwDI2ItQESslXRAio8CHswdV5tiO9J643jpnNXpWnWSNgH7Axuau3lLHS2LyJJg6defntsXwDdbONfMujCV31bcEBGT2uu2TcSihXhL5zSrpW+fx7Z0opl1UyGo7Cd86ySNSLXEEcD6FK8FxuSOGw2sSfHRTcTz59RK6gXsxxub67spa55CSeMlfUrSKaWlnPPMrIuq7Gd+84BT0/qpwK25+LTUozyWrEPl4dTU3ixpSnpeeEqjc0rXOhG4Nz13bFar7ylKugA4BjgcuB34KPAA2cNMM+uO2qmrVdJcsvwyTFItcAHwHeAmSacBq4CTACLicUk3AU8AdcBZEVH65PhMsp7sfsAdaQGYA/xEUg1ZDXFaa2Uq5zO/E4F3AI9GxD9JOhC4qozzzKyraqekGBHTm9l1bDPHzwRmNhFfCIxvIv46KamWq5ykuDUiGiTVSRpE1r73y9tm3VV3HWQ2Z6GkwcCVZD3SfwEermShzKxjK9D73OmU8+3z/0mrV0iaT/aS5NLKFsvMOrTumBQlvaulfRGxuDJFMrOOrrvWFC9uYV8AH2znslgn9JtHbq92EayAycdtap8LdcdnihHxgb1ZEDPrJDrxVAPlKGs6AjOz3Tgpmpntoi48yKyTopkV14VriuWMvC1J/yjp/LR9kKTJlS+amXVEivKXzqicASEuB94LlD7H2Qz8qGIlMrOOr52mI+iIymk+vyci3iXpUYCIeDlNdWpm3VUnrQWWo5ykuCPNgxAAkobTpefyMrPWdNamcTnKSYqXAr8EDpA0k2zUnH+vaKnMrOOKbt77HBE/lbSIbCgfASdExJMVL5mZdVzduaYo6SDgNeDX+VhErKpkwcysA+vOSZFs5r7S5DB9gbHAcrK5V82sG+rWzxQj4u357TR6zunNHG5m1qkV/qIlIhZLenclCmNmnUR3rilKOje32QN4F/BixUpkZh1bd+99Bgbm1uvInjHeXJnimFmn0F1riuml7X0j4kt7qTxm1sGJ9ulokXQYcGMu9CbgfGAw8L/Z1SL9akTcns45DzgNqAfOjog7U3wiu6Y4vR04p7X5nZvT7LfPknqlOVWbnZbAzLqppia+b2pp6RIRyyNiQkRMACaSvfr3y7T7ktK+XEI8nGze5iOAqcDlqeIGMAuYAYxLy9S2/rSWaooPkyXEJZLmAT8HtuR+0C1tvamZdWKVGQHnWOCZiPiz1OxAEscDN0TENuC5NMH9ZEkrySbUWwAg6TrgBOCOthSknGeKQ4GXyOZkKb2vGICToll3VX5HyzBJC3PbsyNidhPHTQPm5rY/L+kUYCHwxYh4GRgFPJg7pjbFdqT1xvE2aSkpHpB6nh9jVzIs6cKPWc2sNQVqihsiYlKL18pG3fo74LwUmgV8kyzPfJNsEr3PsXsOKmmcm/LxNmkpKfYE9m3vG5pZF9C+GeCjwOKIWAdQ+hNA0pXAbWmzFhiTO280sCbFRzcRb5OWkuLaiLiwrRc2sy6q/Wfzm06u6SxpRESsTZufJGutAswDfibp+8BIsg6VhyOiXtJmSVOAh4BTgB+2tTAtJcXOOWyumVVce3W0SOoPfJjdPx3+nqQJZKl3ZWlfRDwu6SbgCbJ3ps9Kb8gAnMmuV3LuoI2dLNByUjy2rRc1sy6unZJiRLwG7N8odnILx88EZjYRXwiMb48yNZsUI2Jje9zAzLqe7v6Zn5nZLu3/TLFDcVI0s0JE1+5wcFI0s+JcUzQz26Vbj7xtZvYGTopmZokHmTUza8Q1RTOzXfxM0cwsz0nRzGwX1xTNzEqCIoPMdjpOimZWSHtNXNVROSmaWXFOimZmu6hts4d2Ck6KZlaMR8kxM9udnymameX4Mz8zszzXFM3Mkujazece1S6AmXVCUebSCkkrJS2TtETSwhQbKuluSSvSn0Nyx58nqUbScknH5eIT03VqJF0qqc2DgzspmlkhpZe3y1nK9IGImBARk9L2V4B7ImIccE/aRtLhwDTgCGAqcLmknumcWcAMsrmgx6X9beKkaGaFqSHKWtroeODatH4tcEIufkNEbIuI54AaYLKkEcCgiFgQEQFclzunMCdFMyum3KZzeTkxgLskLZI0I8UOjIi1AOnPA1J8FLA6d25tio1K643jbeKOlgoZPnI7X/rBKoYcUEc0wO3X78+v5gyvdrG6lYv/dQwP/fcgBg+rY/ZvlwPwzGP9uPQro9n+eg969go+/+1a3vrO13jq0f784EtjgOy/0pO/+AJHfnQTADu2ix/92yiWLtgXCT77lbUc9fFNbN8mLjr7IFYs68+gIXV89Yo/81djtlfr5+5VBV7JGVZ6VpjMjojZue0jI2KNpAOAuyU91dJtm4hFC/E2qVhSlHQ18AlgfUSMr9R9Oqr6OjH7wpHULOtPvwH1XDb/aRbfP5BVK/pWu2jdxkc+vZG/+6cNXHTOQTtjV31rBP947gu8+4Obefiegcz51kguurmGQw7bymXzl9OzF7y0rhdnfugwpnx4Ez17wdwfHMjgYXVc/cBTNDTA5pezx1h3zh3KvoPrueaPT3LfrwYz51sj+Lcf/7laP3fvKj/lbMg9K3zjZSLWpD/XS/olMBlYJ2lERKxNTeP16fBaYEzu9NHAmhQf3US8TSrZfL6GPXjY2dltXN+bmmX9Adi6pSera/oybMSOKpeqe3n7lC0MHFK/W0yCLZuzpLbl1Z4MPTD7N+nbP+iZqgg7tvUg33d55w1DmfaF7L/LHj1gv/2zay64cz8+fNJGAI76xCsseWAgXfiT4N20R0eLpAGSBpbWgY8AjwHzgFPTYacCt6b1ecA0SX0kjSXrUHk4NbE3S5qSep1PyZ1TWMVqihFxv6RDKnX9zuTA0dt58/itPLW4f7WL0u2dceHzfHX6m7nywpFEwCXzVuzc99Ti/lx87hjW1+7Dl3+4ip694C+bsgR67ff+iqV/3JcRh2znrJm1DBlex4YXejN8ZJZUe/aCAYPqeXVjz51Js8sKaKfsfyDwy/T2TC/gZxExX9IjwE2STgNWAScBRMTjkm4CngDqgLMiovSXfSZZRawfcEda2qTqzxTTw9UZAH3pekmjb/96vnbVSq44fySv/aVn6ydYRd127TBO/8bzHPXxTfxu3mC+f+5BfPemZwB467te48r7lrNqRR8uOucg3v2BV6mvgw1r9+Hwd2/h9K+v4eYfD+fKC0fy5R+uajIvtP3tuM6lPT7zi4hngXc0EX8JOLaZc2YCM5uILwTa5TFd1XufI2J2REyKiEm96VPt4rSrnr2Cr121kntvGcIf7hhc7eIYcPfPh/L+j2UdKEf/7Ss8veSN/yM+aNw2+vZvYOXyvgwaWk+ffvU7O12O+sQrrFjWD4DhI3bw4preANTXZc3xxs31rqgC7yl2KFVPil1XcO7Fq1m9oi+3zHavc0ex/4E7WLpgXwCWPLAvI8duA+CFVftQX5cds662N7XP9OXA0duRYMqHX2XpH0vnDOTgQ7NzpnzkVe7++VAAfn/bYN7x/s3do6YYUf7SCVW9+dxVHTF5Cx866WWefaIvl9+dvQ7yX98ewSP3DqpyybqPb595MEsX7Mumjb34zMTDOfmLL/AvF61m1vmjqK8X+/Rp4F8uyl57e+zhAdx42Vh69YIePYIv/P/anc8GT/v3NXzvCwdzxQU92W//Or74/VUATJ3+Et87+2A++763MXBwHV+d1U16num8tcByKCqUzSXNBY4BhgHrgAsiYk5L5wzS0HiPmnyUYB3UnWuWVLsIVsDk41az8E+v71F9duDg0fHOo88p69jf//rLi1p6JacjqmTv8/RKXdvMqqsr1xTdfDazYgKo77pZ0UnRzApzTdHMLK+T9iyXw0nRzApzTdHMrMRTnJqZ7SJA7mgxM9tFfqZoZpa4+Wxmltd5v2suh5OimRXm3mczszzXFM3MknDvs5nZ7rpuTnRSNLPi/EqOmVmek6KZWRJAO0xc1VF5jhYzK0QEivKWFq8jjZH0W0lPSnpc0jkp/nVJz0takpaP5c45T1KNpOWSjsvFJ0palvZdmuZ/bhPXFM2suIZ2qSrWAV+MiMWSBgKLJN2d9l0SEf+RP1jS4cA04AhgJPDfkg5Ncz/PIpsq+UHgdmAqbZz72TVFMyum1HwuZ2npMhFrI2JxWt8MPAmMauGU44EbImJbRDwH1ACTJY0ABkXEgsgmnboOOKGtP89J0cwKK9B8HiZpYW6Z0eT1pEOAdwIPpdDnJS2VdLWkISk2ClidO602xUal9cbxNnFSNLPiyp/3eUNETMotsxtfStK+wM3Av0TEq2RN4TcDE4C1wMWlQ5sqSQvxNvEzRTMrqP0GhJDUmywh/jQibgGIiHW5/VcCt6XNWmBM7vTRwJoUH91EvE1cUzSzYkqz+ZWztCD1EM8BnoyI7+fiI3KHfRJ4LK3PA6ZJ6iNpLDAOeDgi1gKbJU1J1zwFuLWtP881RTMrrJ2+aDkSOBlYJmlJin0VmC5pAln6XQmcDhARj0u6CXiCrOf6rNTzDHAmcA3Qj6zXuU09z+CkaGZt0Q5JMSIeoOnngbe3cM5MYGYT8YXA+D0uFE6KZlZUAA3+zM/MLPHI22Zmu3NSNDNLAqjvuiNCOCmaWUEB4aRoZraLm89mZol7n83MGnFN0cwsx0nRzCyJgPr61o/rpJwUzaw41xTNzHKcFM3MSsK9z2ZmOwWEX942M8vxZ35mZklEe01x2iE5KZpZce5oMTPbJVxTNDMr8SCzZma7eEAIM7NdAgh/5mdmloQHmTUz2024+WxmltOFa4qKDtSLJOlF4M/VLkcFDAM2VLsQVkhX/Tc7OCKG78kFJM0n+/spx4aImLon99vbOlRS7KokLYyISdUuh5XP/2bdV49qF8DMrCNxUjQzy3FS3DtmV7sAVpj/zbopP1M0M8txTdHMLMdJ0cwsx0mxgiRNlbRcUo2kr1S7PNY6SVdLWi/psWqXxarDSbFCJPUEfgR8FDgcmC7p8OqWyspwDdCpXja29uWkWDmTgZqIeDYitgM3AMdXuUzWioi4H9hY7XJY9TgpVs4oYHVuuzbFzKwDc1KsHDUR8/tPZh2ck2Ll1AJjctujgTVVKouZlclJsXIeAcZJGitpH2AaMK/KZTKzVjgpVkhE1AGfB+4EngRuiojHq1sqa42kucAC4DBJtZJOq3aZbO/yZ35mZjmuKZqZ5TgpmpnlOCmameU4KZqZ5TgpmpnlOCl2IpLqJS2R9Jikn0vqvwfXukbSiWn9qpYGq5B0jKT3teEeKyW9Yda35uKNjvlLwXt9XdL/LVpGs8acFDuXrRExISLGA9uBM/I708g8hUXEP0fEEy0ccgxQOCmadUZOip3X74G3pFrcbyX9DFgmqaekiyQ9ImmppNMBlLlM0hOSfgMcULqQpPskTUrrUyUtlvQnSfdIOoQs+f5rqqUeJWm4pJvTPR6RdGQ6d39Jd0l6VNKPafr7791I+pWkRZIelzSj0b6LU1nukTQ8xd4saX465/eS3touf5tmSa9qF8CKk9SLbJzG+Sk0GRgfEc+lxLIpIt4tqQ/wB0l3Ae8EDgPeDhwIPAFc3ei6w4ErgaPTtYZGxEZJVwB/iYj/SMf9DLgkIh6QdBDZVztvAy4AHoiICyV9HNgtyTXjc+ke/YBHJN0cES8BA4DFEfFFSeena3+ebEKpMyJihaT3AJcDH2zDX6NZk5wUO5d+kpak9d8Dc8iatQ9HxHMp/hHgr0vPC4H9gHHA0cDciKgH1ki6t4nrTwHuL10rIpobV/BDwOHSzorgIEkD0z3+Pp37G0kvl/Gbzpb0ybQ+JpX1JaABuDHFrwdukbRv+r0/z927Txn3MCubk2LnsjUiJuQDKTlsyYeAL0TEnY2O+xitD12mMo6B7LHLeyNiaxNlKfu7UUnHkCXY90bEa5LuA/o2c3ik+77S+O/ArD35mWLXcydwpqTeAJIOlTQAuB+Ylp45jgA+0MS5C4C/kTQ2nTs0xTcDA3PH3UXWlCUdNyGt3g98JsU+Cgxppaz7AS+nhPhWsppqSQ+gVNv9B7Jm+avAc5JOSveQpHe0cg+zQpwUu56ryJ4XLk6TL/2YrEXwS2AFsAyYBfyu8YkR8SLZc8BbJP2JXc3XXwOfLHW0AGcDk1JHzhPs6gX/BnC0pMVkzfhVrZR1PtBL0lLgm8CDuX1bgCMkLSJ7Znhhin8GOC2V73E8xYO1M4+SY2aW45qimVmOk6KZWY6ToplZjpOimVmOk6KZWY6ToplZjpOimVnO/wDcFtMf11uaGgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# confusion matrix\n",
    "conf_matrix = confusion_matrix(y_test, predictions)\n",
    "print(conf_matrix)\n",
    "\n",
    "ConfusionMatrixDisplay(conf_matrix).plot()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transfer-learning with BERT (no pre-processing of text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "from tensorflow.keras import layers, models, optimizers\n",
    "from sklearn import model_selection\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\programdata\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3444: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version.\n",
      "\n",
      "\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "b'Skipping line 1925: expected 15 fields, saw 22\\nSkipping line 1977: expected 15 fields, saw 22\\nSkipping line 35265: expected 15 fields, saw 22\\nSkipping line 53357: expected 15 fields, saw 22\\n'\n",
      "b'Skipping line 68361: expected 15 fields, saw 22\\nSkipping line 70922: expected 15 fields, saw 22\\nSkipping line 73503: expected 15 fields, saw 22\\nSkipping line 85612: expected 15 fields, saw 22\\n'\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.head of       marketplace  customer_id       review_id  product_id  product_parent  \\\n",
       "0              US     42521656  R26MV8D0KG6QI6  B000SAQCWC       159713740   \n",
       "1              US     12049833  R1OF8GP57AQ1A0  B00509LVIQ       138680402   \n",
       "2              US       107642  R3VDC1QB6MC4ZZ  B00KHXESLC       252021703   \n",
       "3              US      6042304  R12FA3DCF8F9ER  B000F8JIIC       752728342   \n",
       "4              US     18123821   RTWHVNV6X4CNJ  B004ZWR9RQ       552138758   \n",
       "...           ...          ...             ...         ...             ...   \n",
       "98635          US     27635275  R1PHJCOXJ43DD5  B00V3W9SMU       217641189   \n",
       "98636          US      1369561   R2ESP8RM8ERG4  B00FY2UESM       243349133   \n",
       "98637          US     49479456  R1VRW0RFCHM8F4  B001L9MQK6       626575660   \n",
       "98638          US      9965703  R1ZKNAAKDNX73G  B00OX6FIMW       403680832   \n",
       "98639          US     11372488  R1L2GI2QOIBQJU  B00IN4HABM       974652159   \n",
       "\n",
       "                                           product_title product_category  \\\n",
       "0      The Cravings Place Chocolate Chunk Cookie Mix,...          Grocery   \n",
       "1                Mauna Loa Macadamias, 11 Ounce Packages          Grocery   \n",
       "2      Organic Matcha Green Tea Powder - 100% Pure Ma...          Grocery   \n",
       "3      15oz Raspberry Lyons Designer Dessert Syrup Sauce          Grocery   \n",
       "4      Stride Spark Kinetic Fruit Sugar Free Gum, 14-...          Grocery   \n",
       "...                                                  ...              ...   \n",
       "98635           Two Pound Assorted Roasted Nuts Gift Tin          Grocery   \n",
       "98636  Award Winning Gourmet Beer Brittle - An IPA Be...          Grocery   \n",
       "98637  Rokz Design Group Infused Margarita Salt, Lime...          Grocery   \n",
       "98638  Jus By Julie 5 Day Blended Juice Cleanse + Mea...          Grocery   \n",
       "98639  Konpeito [Primrose \"Sakura Sou\"] (50g) [Kyoto ...          Grocery   \n",
       "\n",
       "       star_rating  helpful_votes  total_votes vine verified_purchase  \\\n",
       "0                5              0            0    N                 Y   \n",
       "1                5              0            0    N                 Y   \n",
       "2                5              0            0    N                 N   \n",
       "3                5              0            0    N                 Y   \n",
       "4                5              0            0    N                 Y   \n",
       "...            ...            ...          ...  ...               ...   \n",
       "98635            5              0            0    N                 Y   \n",
       "98636            5              0            0    N                 N   \n",
       "98637            5              0            0    N                 Y   \n",
       "98638            1              0            1    N                 Y   \n",
       "98639            5              0            0    N                 Y   \n",
       "\n",
       "                          review_headline  \\\n",
       "0      Using these for years - love them.   \n",
       "1                               Wonderful   \n",
       "2                              Five Stars   \n",
       "3                              Five Stars   \n",
       "4                              Five Stars   \n",
       "...                                   ...   \n",
       "98635                          Five Stars   \n",
       "98636                  Best brittle ever!   \n",
       "98637                          Five Stars   \n",
       "98638                            One Star   \n",
       "98639                          Five Stars   \n",
       "\n",
       "                                             review_body review_date  \n",
       "0      As a family allergic to wheat, dairy, eggs, nu...  2015-08-31  \n",
       "1      My favorite nut.  Creamy, crunchy, salty, and ...  2015-08-31  \n",
       "2      This green tea tastes so good! My girlfriend l...  2015-08-31  \n",
       "3      I love Melissa's brand but this is a great sec...  2015-08-31  \n",
       "4                                                   good  2015-08-31  \n",
       "...                                                  ...         ...  \n",
       "98635          This was a gift. Exactly what I expected.  2015-07-28  \n",
       "98636  This is absolutely the most savory brittle I h...  2015-07-28  \n",
       "98637                              Exactly what I wanted  2015-07-28  \n",
       "98638                                      Not impressed  2015-07-28  \n",
       "98639                                             Yummy.  2015-07-28  \n",
       "\n",
       "[98640 rows x 15 columns]>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read tsv into dataframe\n",
    "dtf = pd.read_csv(DATA_PATH + 'amazon_reviews_grocery_100k.tsv', sep='\\t', error_bad_lines=False)\n",
    "\n",
    "dtf.head\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.head of                                              review_body  star_rating\n",
       "0      As a family allergic to wheat, dairy, eggs, nu...            5\n",
       "1      My favorite nut.  Creamy, crunchy, salty, and ...            5\n",
       "2      This green tea tastes so good! My girlfriend l...            5\n",
       "3      I love Melissa's brand but this is a great sec...            5\n",
       "4                                                   good            5\n",
       "...                                                  ...          ...\n",
       "98635          This was a gift. Exactly what I expected.            5\n",
       "98636  This is absolutely the most savory brittle I h...            5\n",
       "98637                              Exactly what I wanted            5\n",
       "98638                                      Not impressed            1\n",
       "98639                                             Yummy.            5\n",
       "\n",
       "[98640 rows x 2 columns]>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filter columns\n",
    "dtf = dtf[['review_body', 'star_rating']]\n",
    "\n",
    "dtf.head\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.head of                                                     text  y\n",
       "0      As a family allergic to wheat, dairy, eggs, nu...  5\n",
       "1      My favorite nut.  Creamy, crunchy, salty, and ...  5\n",
       "2      This green tea tastes so good! My girlfriend l...  5\n",
       "3      I love Melissa's brand but this is a great sec...  5\n",
       "4                                                   good  5\n",
       "...                                                  ... ..\n",
       "98635          This was a gift. Exactly what I expected.  5\n",
       "98636  This is absolutely the most savory brittle I h...  5\n",
       "98637                              Exactly what I wanted  5\n",
       "98638                                      Not impressed  1\n",
       "98639                                             Yummy.  5\n",
       "\n",
       "[98640 rows x 2 columns]>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rename columns\n",
    "dtf = dtf.rename(columns={'review_body': 'text', 'star_rating': 'y'})\n",
    "\n",
    "dtf.head\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>86839</th>\n",
       "      <td>Very tasty!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8423</th>\n",
       "      <td>Kind of skimpy on the noodles, carrots, celery...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27484</th>\n",
       "      <td>My wonderful sister and mother sent me this gi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24686</th>\n",
       "      <td>I am happy I tried the variety pack and found ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79274</th>\n",
       "      <td>Very delicious and much more tender and flavor...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  y\n",
       "86839                                        Very tasty!  1\n",
       "8423   Kind of skimpy on the noodles, carrots, celery...  1\n",
       "27484  My wonderful sister and mother sent me this gi...  1\n",
       "24686  I am happy I tried the variety pack and found ...  1\n",
       "79274  Very delicious and much more tender and flavor...  1"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# map rating to class (<=1 = 0 | <1 = 1)\n",
    "\n",
    "dtf['y'] = dtf['y'].map(lambda x: 0 if int(x) <= 1 else 1)\n",
    "\n",
    "# print 5 random rows\n",
    "dtf.sample(5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split dataset\n",
    "dtf_train, dtf_test = model_selection.train_test_split(dtf, test_size=0.2)\n",
    "\n",
    "# get target\n",
    "y1_train = dtf_train[\"y\"].values\n",
    "y1_test = dtf_test[\"y\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# distil-bert tokenizer\n",
    "tokenizer = transformers.AutoTokenizer.from_pretrained(\n",
    "    'distilbert-base-uncased', do_lower_case=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_feature_matrix(corpus, maxlen=50):\n",
    "    # add special tokens\n",
    "    maxqnans = int((maxlen-20)/2)\n",
    "    corpus_tokenized = [\"[CLS] \" +\n",
    "                        \" \".join(tokenizer.tokenize(re.sub(r'[^\\w\\s]+|\\n', '',\n",
    "                                                    str(txt).lower().strip()))[:maxqnans]) +\n",
    "                        \" [SEP] \" for txt in corpus]\n",
    "\n",
    "    # generate masks\n",
    "    masks = [[1]*len(txt.split(\" \")) + [0]*(maxlen - len(\n",
    "        txt.split(\" \"))) for txt in corpus_tokenized]\n",
    "\n",
    "    # padding\n",
    "    txt2seq = [txt + \" [PAD]\"*(maxlen-len(txt.split(\" \"))) if len(txt.split(\" \"))\n",
    "               != maxlen else txt for txt in corpus_tokenized]\n",
    "\n",
    "    # generate idx\n",
    "    idx = [tokenizer.encode(seq)[:50] for seq in txt2seq]\n",
    "\n",
    "    # generate segments\n",
    "    segments = []\n",
    "    for seq in txt2seq:\n",
    "        temp, i = [], 0\n",
    "        for token in seq.split(\" \"):\n",
    "            temp.append(i)\n",
    "            if token == \"[SEP]\":\n",
    "                i += 1\n",
    "        segments.append(temp)\n",
    "\n",
    "    # feature matrix\n",
    "    return [np.asarray(idx, dtype='object'),\n",
    "            np.asarray(masks, dtype='int32'),\n",
    "            np.asarray(segments, dtype='int32')]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_sample(corpus, X):\n",
    "    i = 0\n",
    "\n",
    "    print(\"txt: \", corpus.iloc[0])\n",
    "    print(\"tokenized:\", [tokenizer.convert_ids_to_tokens(idx)\n",
    "        for idx in X[0][i].tolist()])\n",
    "    print(\"idx: \", X[0][i])\n",
    "    print(\"mask: \", X[1][i])\n",
    "    print(\"segment: \", X[2][i])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (714 > 512). Running this sequence through the model will result in indexing errors\n"
     ]
    }
   ],
   "source": [
    "X1_train = prepare_feature_matrix(dtf_train['text'])\n",
    "X1_test = prepare_feature_matrix(dtf_test['text'])\n",
    "\n",
    "print_sample(dtf_train['text'], X1_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at distilbert-base-uncased were not used when initializing TFDistilBertModel: ['vocab_projector', 'activation_13', 'vocab_layer_norm', 'vocab_transform']\n",
      "- This IS expected if you are initializing TFDistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFDistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFDistilBertModel were initialized from the model checkpoint at distilbert-base-uncased.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFDistilBertModel for predictions without further training.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_idx (InputLayer)         [(None, 50)]         0           []                               \n",
      "                                                                                                  \n",
      " input_masks (InputLayer)       [(None, 50)]         0           []                               \n",
      "                                                                                                  \n",
      " tf_distil_bert_model (TFDistil  TFBaseModelOutput(l  66362880   ['input_idx[0][0]',              \n",
      " BertModel)                     ast_hidden_state=(N               'input_masks[0][0]']            \n",
      "                                one, 50, 768),                                                    \n",
      "                                 hidden_states=None                                               \n",
      "                                , attentions=None)                                                \n",
      "                                                                                                  \n",
      " global_average_pooling1d (Glob  (None, 768)         0           ['tf_distil_bert_model[0][0]']   \n",
      " alAveragePooling1D)                                                                              \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, 64)           49216       ['global_average_pooling1d[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " dense_1 (Dense)                (None, 1)            65          ['dense[0][0]']                  \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 66,412,161\n",
      "Trainable params: 49,281\n",
      "Non-trainable params: 66,362,880\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# building the model\n",
    "\n",
    "# inputs\n",
    "idx = layers.Input((50), dtype=\"int32\", name=\"input_idx\")\n",
    "masks = layers.Input((50), dtype=\"int32\", name=\"input_masks\")\n",
    "\n",
    "# pre-trained bert with config\n",
    "config = transformers.DistilBertConfig(dropout=0.2, attention_dropout=0.2)\n",
    "config.output_hidden_states = False\n",
    "nlp = transformers.TFDistilBertModel.from_pretrained(\n",
    "    'distilbert-base-uncased', config=config)\n",
    "bert_out = nlp(idx, attention_mask=masks)[0]\n",
    "\n",
    "# fine-tuning\n",
    "x = layers.GlobalAveragePooling1D()(bert_out)\n",
    "x = layers.Dense(64, activation=\"relu\")(x)\n",
    "y_out = layers.Dense(1, activation='sigmoid')(x)\n",
    "\n",
    "# compile\n",
    "model1 = models.Model([idx, masks], y_out)\n",
    "for layer in model1.layers[:3]:\n",
    "    layer.trainable = False\n",
    "model1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model1.summary()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](arch/distil_bert.h5.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encode y\n",
    "dic_y_mapping = {n: label for n, label in enumerate(np.unique(y1_train))}\n",
    "inverse_dic = {v: k for k, v in dic_y_mapping.items()}\n",
    "y1_train = np.array([inverse_dic[y] for y in y1_train])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 63129 samples, validate on 15783 samples\n",
      "Epoch 1/10\n",
      "63104/63129 [============================>.] - ETA: 0s - loss: 0.2399 - accuracy: 0.9286"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\programdata\\anaconda3\\lib\\site-packages\\keras\\engine\\training_v1.py:2057: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
      "  updates = self.state_updates\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63129/63129 [==============================] - 73s 1ms/sample - loss: 0.2399 - accuracy: 0.9286 - val_loss: 0.2286 - val_accuracy: 0.9271\n",
      "Epoch 2/10\n",
      "63129/63129 [==============================] - 73s 1ms/sample - loss: 0.2189 - accuracy: 0.9294 - val_loss: 0.2345 - val_accuracy: 0.9245\n",
      "Epoch 3/10\n",
      "63129/63129 [==============================] - 75s 1ms/sample - loss: 0.2136 - accuracy: 0.9295 - val_loss: 0.2156 - val_accuracy: 0.9266\n",
      "Epoch 4/10\n",
      "63129/63129 [==============================] - 74s 1ms/sample - loss: 0.2086 - accuracy: 0.9295 - val_loss: 0.2355 - val_accuracy: 0.9221\n",
      "Epoch 5/10\n",
      "63129/63129 [==============================] - 74s 1ms/sample - loss: 0.2074 - accuracy: 0.9302 - val_loss: 0.2058 - val_accuracy: 0.9273\n",
      "Epoch 6/10\n",
      "63129/63129 [==============================] - 74s 1ms/sample - loss: 0.2052 - accuracy: 0.9298 - val_loss: 0.2282 - val_accuracy: 0.9274\n",
      "Epoch 7/10\n",
      "63129/63129 [==============================] - 78s 1ms/sample - loss: 0.2045 - accuracy: 0.9303 - val_loss: 0.2042 - val_accuracy: 0.9270\n",
      "Epoch 8/10\n",
      "63129/63129 [==============================] - 78s 1ms/sample - loss: 0.2028 - accuracy: 0.9300 - val_loss: 0.2185 - val_accuracy: 0.9251\n",
      "Epoch 9/10\n",
      "63129/63129 [==============================] - 75s 1ms/sample - loss: 0.2012 - accuracy: 0.9313 - val_loss: 0.2037 - val_accuracy: 0.9278\n",
      "Epoch 10/10\n",
      "63129/63129 [==============================] - 78s 1ms/sample - loss: 0.2009 - accuracy: 0.9310 - val_loss: 0.2031 - val_accuracy: 0.9278\n"
     ]
    }
   ],
   "source": [
    "# train\n",
    "training = model1.fit(x=X1_train, y=y1_train, batch_size=64,\n",
    "                      epochs=10, shuffle=True, verbose=1, validation_split=0.2)\n",
    "\n",
    "model1 = model1.save(MODEL_PATH + 'distil_bert.h5')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer.\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_idx (InputLayer)         [(None, 50)]         0           []                               \n",
      "                                                                                                  \n",
      " input_masks (InputLayer)       [(None, 50)]         0           []                               \n",
      "                                                                                                  \n",
      " tf_distil_bert_model_1 (TFDist  TFBaseModelOutput(l  66362880   ['input_idx[0][0]',              \n",
      " ilBertModel)                   ast_hidden_state=(N               'input_masks[0][0]']            \n",
      "                                one, 50, 768),                                                    \n",
      "                                 hidden_states=None                                               \n",
      "                                , attentions=None)                                                \n",
      "                                                                                                  \n",
      " global_average_pooling1d (Glob  (None, 768)         0           ['tf_distil_bert_model_1[0][0]'] \n",
      " alAveragePooling1D)                                                                              \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, 64)           49216       ['global_average_pooling1d[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " dense_1 (Dense)                (None, 1)            65          ['dense[0][0]']                  \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 66,412,161\n",
      "Trainable params: 66,412,161\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 63129 samples, validate on 15783 samples\n",
      "Epoch 1/5\n",
      "63129/63129 [==============================] - ETA: 0s - loss: 0.1990 - accuracy: 0.9307"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\programdata\\anaconda3\\lib\\site-packages\\keras\\engine\\training_v1.py:2057: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
      "  updates = self.state_updates\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63129/63129 [==============================] - 179s 3ms/sample - loss: 0.1990 - accuracy: 0.9307 - val_loss: 0.1757 - val_accuracy: 0.9354\n",
      "Epoch 2/5\n",
      "63129/63129 [==============================] - 179s 3ms/sample - loss: 0.1605 - accuracy: 0.9391 - val_loss: 0.1681 - val_accuracy: 0.9389\n",
      "Epoch 3/5\n",
      "63129/63129 [==============================] - 180s 3ms/sample - loss: 0.1464 - accuracy: 0.9436 - val_loss: 0.1637 - val_accuracy: 0.9404\n",
      "Epoch 4/5\n",
      "63129/63129 [==============================] - 180s 3ms/sample - loss: 0.1340 - accuracy: 0.9473 - val_loss: 0.1674 - val_accuracy: 0.9383\n",
      "Epoch 5/5\n",
      "63129/63129 [==============================] - 180s 3ms/sample - loss: 0.1248 - accuracy: 0.9511 - val_loss: 0.1759 - val_accuracy: 0.9410\n"
     ]
    }
   ],
   "source": [
    "# load model\n",
    "model1 = models.load_model(MODEL_PATH + 'distil_bert.h5', custom_objects={\n",
    "                           'TFDistilBertModel': transformers.TFDistilBertModel})\n",
    "\n",
    "# unfreeze and fine-tune\n",
    "for layer in model1.layers[:3]:\n",
    "    layer.trainable = True\n",
    "model1.compile(loss='binary_crossentropy',\n",
    "               optimizer=optimizers.Adam(learning_rate=1e-5), metrics=['accuracy'])\n",
    "model1.summary()\n",
    "\n",
    "fine_tuning = model1.fit(x=X1_train, y=y1_train, batch_size=64,\n",
    "                         epochs=5, shuffle=True, verbose=1, validation_split=0.2)\n",
    "\n",
    "# save model\n",
    "model1.save(MODEL_PATH + 'distil_bert.h5')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\programdata\\anaconda3\\lib\\site-packages\\keras\\engine\\training_v1.py:2057: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
      "  updates = self.state_updates\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 94.69%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\programdata\\anaconda3\\lib\\site-packages\\keras\\engine\\training_v1.py:2079: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
      "  updates=self.state_updates,\n"
     ]
    }
   ],
   "source": [
    "# load model\n",
    "model1 = models.load_model(MODEL_PATH + 'distil_bert.h5', custom_objects={\n",
    "                           'TFDistilBertModel': transformers.TFDistilBertModel})\n",
    "\n",
    "# evaluate\n",
    "scores = model1.evaluate(X1_test, y1_test)\n",
    "\n",
    "print('Accuracy: %.2f%%' % (scores[1]*100))\n",
    "\n",
    "# test\n",
    "predicted = model1.predict(X1_test)\n",
    "predicted = list(map(lambda x: 1 if x > 0.5 else 0, predicted))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  480   904]\n",
      " [  144 18200]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x20d6a318d30>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAAEGCAYAAADyuIefAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhDklEQVR4nO3deZydVX3H8c83e8hKFiArBAloEiFKjAELIqAJagu2oKEotKABCiJKbYlaoWDagkWUFqJgkEUIi6wiO1iBNiwJBghBJEAgQzaSQAiELDPz6x/PucmTYZZ7J3MzM3e+79frvHLv79nOTV78OOc5z3OOIgIzM8t0au0KmJm1JU6KZmY5TopmZjlOimZmOU6KZmY5XVq7Annd1CN6qFdrV8NKoE5q7SpYCd6vfZdNtRu26x9t8md6xeo1NUXtO+/ZjfdFxJTtud6O1qaSYg/1YlLXdvX31+F16tmjtatgJZjz7h3bfY5Va2p44r7hRe3bdcjLg7b7gjtYm0qKZtYeBDVR29qVKBsnRTMrSQC1VO5LH06KZlayWtxSNDMDIAg2V3D32Y/kmFlJAqghiipNkXSlpJWSFuRiN0qan8piSfNTfA9J7+e2/Tx3zP6SnpO0SNIlkpTi3dP5Fkl6QtIeTdXJSdHMSlZLFFWKcBWwzSMnEfGViBgfEeOBW4Bbc5tfLmyLiFNy8ZnANGB0KoVzngS8FRF7ARcDFzRVISdFMytJADURRZUmzxXxCLCmvm2ptfdlYHZj55A0BOgbEXMim/brGuCotPlI4Or0+TfAYYVWZEOcFM2sZLVFFmCQpLm5Mq2EyxwErIiIl3KxUZL+KOkPkg5KsWFAVW6fqhQrbFsCEBHVwFpgYGMX9UCLmZUkirxfmKyKiAnNvNSxbNtKXAaMjIjVkvYHbpc0Fqiv5VeoYGPb6uWkaGYliYDNZX5MUVIX4K+B/bdeNzYCG9PneZJeBvYmaxnmX7EZDixNn6uAEUBVOmc/GuiuF7j7bGYlEjVFlu1wOPCniNjSLZY0WFLn9HlPsgGVVyJiGbBO0qR0v/B4oPA+453ACenz0cDD0cRyA24pmllJAqhtoZaipNnAIWT3HquAcyJiFjCVDw6wHAycJ6kaqAFOiYhCq+9UspHsnsA9qQDMAq6VtIishTi1qTo5KZpZybazFbhFRBzbQPzv6ondQvaITn37zwXG1RPfABxTSp2cFM2sJNnD25U7ZZyTopmVJIDNUbnDEU6KZlaSQNRU8Bitk6KZlaw23H02MwN8T9HMrA5R43uKZmaZbOZtJ0UzMwAixKbo3NrVKBsnRTMrWa3vKZqZZbKBFnefzcwSD7SYmW3hgRYzszpq/PC2mVkmEJujclNH5f4yMysLD7SYmeUEcvfZzCzPAy1mZkkEfiTHzKwgG2jxa35mZlt4oMXMLAlU0ZPMVm66N7OyqaFTUaUpkq6UtFLSglzsXElvSJqfyudz26ZLWiTpRUmTc/H9JT2Xtl2S1n9GUndJN6b4E5L2aKpOTopmVpJs3edORZUiXAVMqSd+cUSMT+VuAEljyNZtHpuOuUxS4ebmTGAaMDqVwjlPAt6KiL2Ai4ELmqqQk6KZlUjUFFmaEhGPkC1SX4wjgRsiYmNEvAosAiZKGgL0jYg5ERHANcBRuWOuTp9/AxxWaEU2xEnRzEqSLXHauagCDJI0N1emFXmZ0yU9m7rXO6fYMGBJbp+qFBuWPteNb3NMRFQDa4GBjV3YAy1mVpIIFds1BlgVERNKvMRM4Hyy/Hs+cBFwItTb9IxG4jSxrV5OimZWsnI+vB0RKwqfJV0B3JW+VgEjcrsOB5am+PB64vljqiR1AfrRRHfd3WczK0k2n6KKKs2R7hEWfAkojEzfCUxNI8qjyAZUnoyIZcA6SZPS/cLjgTtyx5yQPh8NPJzuOzbILUUzK1HLzbwtaTZwCNm9xyrgHOAQSePJ8u9i4GSAiHhe0k3AQqAaOC0iatKpTiUbye4J3JMKwCzgWkmLyFqIU5uqk5OimZUkeySnZR7ejohj6wnPamT/GcCMeuJzgXH1xDcAx5RSJydFMyuJ3302M6vDU4eZmSXZ1GGV++6zk6KZlaySJ4RwUjSzkmSz5Lj7bGYGFF7zc1K0JnTqFFxy10JWL+/KOSfuzZ5j1vPNGYvp1r2Wmhrx3z/YnT8/0xuAr/zDUiZ/ZRW1NWLmuSOZ90i/Vq59x3Pk195g8jHLkeDem3fjjmuG0bvfZqb/5E/sMmwDK9/owb9/+8O8+07XLccMHrKBn981j+su3Z1brxzeyNkrXWW3FMv6yyRNSfOeLZJ0djmv1dqOOnEFSxb12PL9pOlLuO5nQznt8+O49ifD+Pr07H31kaPf59N/uYaTPzuO75+wN6f96DU6dWr0AXtrYbuPfo/Jxyzn218ez2lHfZyJh6xh6O7v8+VvVDH/8f58Y8onmP94f475RtU2x02b/gpzHx3QSrVuW8r5RktrK1tSTPOcXQocAYwBjk3zoVWcQbtt4hOHvs29NwzeGgzYqXf2sH2vPjWsXpm1OA747Fv84bcD2LypEyuWdGfZ4u7sM/691qh2hzViz/W8+EwfNm7oTG2NWPBUPw48fBWTDlvNg7fvCsCDt+/KAYev3nLMAYetYtmSHry+aKfWqnabURh9Lqa0R+VsKU4EFkXEKxGxCbiBbG6zinPyOa8z699GELVbYz8/byRf/14V186Zz9e/v4RfXZB1twbutpk3l3Xbst+q5d0YuNumHV3lDu21l3ox7hPv0Kf/Zrr3qGHCp9cwaMhG+g/cxFtvZv82b73ZjX4DNgPQvWcNR3+jiusv3b01q92mtOAks21OOe8p1jf32Sfr7pTmV5sG0IP293/hiYe+zduru7BoQS/2nfTOlvgXv7qSX5w/gv+9ZwAHfWEN375wMdOP24f6prds/PV0a2lLXtmJm68YzoxZz7FhfWde/VMvaqobbtV89ZuvcftVw9iwvnLf4ihFpa/RUs6kWNQ8ZhFxOXA5QN9OA9tdehg7YR2TDn+biYc8Q9futezUp5Z/+unLfPKwtcw8dyQAj/5uZ8684FUAVi3ryuAhW1uGg3bbxJoV3eo9t5XP/bfsxv237AbACd9ezKrl3Xh7dTd2Hpy1FncevIm1a7JbHvvsu46/mLyKE7/7Kr36VBO1YtPGTtx13dDW/AmtJoDqdtoKLEY5k2JDc59VlF9dOIJfXZj9zH0nvcPfTFvOhWd+iMsfeo59J63j2cf7Mv5T61i6OBuEefyBnfnnS17m1l/uxoBdNzN01EZenN+rNX9Ch9RvwCbWrunG4CEbOPCzqzhr6n7sOnwDhx+1gpuvGMHhR63g8YeyCZr/6av7bTnuuNNf4/31nTtsQixor13jYpQzKT4FjE7znr1BNmXP35bxem3Kz/55D04593U6dw42bezEz87eA4DXXurJI78bwC8eXEBttbj0X0ZSW1u5XZG26vuXvEDf/pupru7EZed9iHff6crNV4xg+sUv8Lm/Wc6by7rzb2d+pLWr2TZFZXef1cR8i9t38mxpwp8CnYEr07Q/DerbaWBM6lrfwl7WVnXq2aPpnazNmPPuHaytXrVdGW3nD+8Sh155dFH73vqpmfOasRxBqyrrw9tpacK7y3kNM9vxKrml6DdazKwkLTnJbFvkpGhmJQlEda0HWszMtmivr/AVw0nRzEoT7j6bmW1R6fcUK/fGgJmVTW16VrGp0hRJV0paKWlBLvZjSX+S9Kyk2yT1T/E9JL0vaX4qP88ds7+k59KMXJek9Z9Ja0TfmOJPSNqjqTo5KZpZSQJRU9upqFKEq4C6Dyc/AIyLiH2BPwPTc9tejojxqZySi88km0NhdCqFc54EvBURewEXAxc0VSEnRTMrWUvNpxgRj5AtUp+P3R8R1enr42SvCDdI0hCgb0TMiextlGuAo9LmI4Gr0+ffAIcVWpENcVI0s5JElNR9HiRpbq5MK/FyJwL35L6PkvRHSX+QdFCKDSOba6GgKsUK25Zk9Y5qYC0wsLELeqDFzEoWxQ+0rGrua36Svg9UA9el0DJgZESslrQ/cLuksTQ+I1dRs3XlOSmaWYnKPyGEpBOALwKHpS4xEbER2Jg+z5P0MrA3Wcsw38XOz8hVmK2rSlIXoB91uut1uftsZiWLUFGlOSRNAf4Z+KuIWJ+LD07LnCBpT7IBlVciYhmwTtKkdL/weOCOdNidwAnp89HAw9HELDhuKZpZSSKgpoWmu5M0GziE7N5jFXAO2Whzd+CBNCbyeBppPhg4T1I1UAOcEhGFVt+pZCPZPcnuQRbuQ84CrpW0iKyFOLWpOjkpmlnJWuo1v4g4tp7wrAb2vQW4pYFtc4Fx9cQ3AMeUUicnRTMrSVDSQEu746RoZiWq7Jm3nRTNrGSVvAKlk6KZlczdZzOzJBt9rtyn+ZwUzaxk7j6bmeW4+2xmlgTNf1ulPXBSNLOSVXDv2UnRzEoUEC30ml9b5KRoZiVz99nMLKdDjj5L+i8auXUQEWeUpUZm1qZ15Hef5+6wWphZ+xFAR0yKEXF1/rukXhHxXvmrZGZtXSV3n5t8V0fSAZIWAi+k7/tJuqzsNTOzNkpEbXGlPSrmBcafApOB1QAR8QzZDLhm1lFFkaUdKmr0OSKW1FkqtaY81TGzNi867kBLwRJJBwIhqRtwBqkrbWYdVDttBRajmO7zKcBpZItKvwGMT9/NrMNSkaX9abKlGBGrgON2QF3MrL2obe0KlE8xo897SvqtpDclrZR0R1pz1cw6osJzisWUJki6MuWVBbnYAEkPSHop/blzbtt0SYskvShpci6+v6Tn0rZL0vrPSOou6cYUf0LSHk3VqZju8/XATcAQYChwMzC7iOPMrEJFFFeKcBUwpU7sbOChiBgNPJS+I2kM2brNY9Mxl0nqnI6ZCUwDRqdSOOdJwFsRsRdwMXBBUxUqJikqIq6NiOpUfk1F32Y1sya10CM5EfEI2SL1eUcChZdHrgaOysVviIiNEfEqsAiYKGkI0Dci5kREANfUOaZwrt8AhxVakQ1p7N3nAenj7yWdDdxA9jO/AvyusZOaWYUr/pGcQZLyrwxfHhGXN3HMrhGxDCAilknaJcWHAY/n9qtKsc3pc9144Zgl6VzVktYCA4FVDV28sYGWeWRJsPDrT85tC+D8Ro41swqm4vuKqyJiQktdtp5YNBJv7JgGNfbu86jGDjSzDioE5X2Fb4WkIamVOARYmeJVwIjcfsOBpSk+vJ54/pgqSV2Afnywu76NotYplDRO0pclHV8oxRxnZhWqvK/53QmckD6fANyRi09NI8qjyAZUnkxd7XWSJqX7hcfXOaZwrqOBh9N9xwY1+ZyipHOAQ4AxwN3AEcBjZDczzawjaqGhVkmzyfLLIElVwDnAfwA3SToJeB04BiAinpd0E7AQqAZOi4jCK8enko1k9wTuSQVgFnCtpEVkLcSpTdWpmNf8jgb2A/4YEX8vaVfgl0UcZ2aVqoWSYkQc28CmwxrYfwYwo574XGBcPfENpKRarGKS4vsRUSupWlJfsv69H94266g66iSzOXMl9QeuIBuRfhd4spyVMrO2rYTR53anmHef/yF9/Lmke8kekny2vNUyszatIyZFSR9vbFtEPF2eKplZW9dRW4oXNbItgENbuC4QQWze1OKntfK5+zXfSWlPJk5+t2VO1BHvKUbEZ3ZkRcysnWjHSw0Uo6jlCMzMtuGkaGa2lSp4klknRTMrXQW3FIuZeVuSvirph+n7SEkTy181M2uLFMWX9qiYCSEuAw4ACq/jrAMuLVuNzKzta6HlCNqiYrrPn4yIj0v6I0BEvJWWOjWzjqqdtgKLUUxS3JzWQQgASYOp6LW8zKwp7bVrXIxikuIlwG3ALpJmkM2a84Oy1srM2q7o4KPPEXGdpHlkU/kIOCoiXih7zcys7erILUVJI4H1wG/zsYh4vZwVM7M2rCMnRbKV+wqLw/QARgEvkq29amYdUIe+pxgRH81/T7PnnNzA7mZm7VrJb7RExNOSPlGOyphZO9GRW4qSvpP72gn4OPBm2WpkZm1bRx99BvrkPleT3WO8pTzVMbN2oaO2FNND270j4rs7qD5m1saJlhlokbQPcGMutCfwQ6A/8A229ki/FxF3p2OmAycBNcAZEXFfiu/P1iVO7wa+1dT6zg1p8N1nSV3SmqoNLktgZh1UfQvf11caO0XEixExPiLGA/uTPfp3W9p8cWFbLiGOIVu3eSwwBbgsNdwAZgLTgNGpTGnuT2uspfgkWUKcL+lO4GbgvdwPurW5FzWzdqw8M+AcBrwcEa9JDU4kcSRwQ0RsBF5NC9xPlLSYbEG9OQCSrgGOAu5pTkWKuac4AFhNtiZL4XnFAJwUzTqq4gdaBkmam/t+eURcXs9+U4HZue+nSzoemAucFRFvAcOAx3P7VKXY5vS5brxZGkuKu6SR5wVsTYYFFXyb1cyaUkJLcVVETGj0XNmsW38FTE+hmcD5ZHnmfLJF9E5k2xxUUDc35ePN0lhS7Az0bukLmlkFaNkMcATwdESsACj8CSDpCuCu9LUKGJE7bjiwNMWH1xNvlsaS4rKIOK+5JzazCtXyq/kdS67rLGlIRCxLX79E1lsFuBO4XtJPgKFkAypPRkSNpHWSJgFPAMcD/9XcyjSWFNvntLlmVnYtNdAiaSfgs2z76vCFksaTpd7FhW0R8bykm4CFZM9Mn5aekAE4la2P5NxDMwdZoPGkeFhzT2pmFa6FkmJErAcG1ol9rZH9ZwAz6onPBca1RJ0aTIoRsaYlLmBmlaejv+ZnZrZVy99TbFOcFM2sJKKyBxycFM2sdG4pmplt1aFn3jYz+wAnRTOzxJPMmpnV4ZaimdlWvqdoZpbnpGhmtpVbimZmBUEpk8y2O06KZlaSllq4qq1yUjSz0jkpmpltpeatHtouOCmaWWk8S46Z2bZ8T9HMLMev+ZmZ5bmlaGaWhLvPZmbbquCk2Km1K2Bm7Uvh4e1iSpPnkhZLek7SfElzU2yApAckvZT+3Dm3/3RJiyS9KGlyLr5/Os8iSZdIavaKCU6KZlYy1UZRpUifiYjxETEhfT8beCgiRgMPpe9IGgNMBcYCU4DLJHVOx8wEpgGjU5nS3N/mpGhmpYkSSvMcCVydPl8NHJWL3xARGyPiVWARMFHSEKBvRMyJiACuyR1TMifF7fSdn7zOjc8+zy8efvED244+ZSX3LX2GvgOqt4kPHraJ2196jqNPWbmjqtkhXfTtEXz5o2OZ9pl9tsReXtCTb31xNKcevg+nT9mbP/1xJwDm/aE3p03em5MP3YfTJu/N/Md6bznmpWd7cvKh+/B3B36Ey34wjMLLHJs2ihkn787fHfgRzvjCaJYv6bZDf19rUm1xBRgkaW6uTKtzqgDulzQvt23XiFgGkP7cJcWHAUtyx1al2LD0uW68WcqWFCVdKWmlpAXlukZbcP+NA/j+caM+EB88dBMfO3gdK6q6fmDbKecu5amH++yI6nVon/vKGmZc98o2sV/+aAhf/c5yZj74Isd/dxmzfjQUgH4Dajjv6lf4xcMv8t2fvc6FZ4zccswlZw/nWxcu4Vf/+wJvvNqdub/P/u3umz2A3v1ruOr/XuCvv/Ems340ZMf9uNZWfEtxVURMyJXL65zpUxHxceAI4DRJBzdy1fruE0Yj8WYpZ0vxKrajX99eLHiiN+ve+uAg/snnLmXWj4ZS9xXRA6asZdnr3Xjtzz12UA07ro9Oeo8+O9dsE5PgvXXZbaj33unMgF03A7DXR99n4G5Zi373fTawaWMnNm0Uq1d0Yf26zoyZsB4JDj96Df93bz8A5tzXj88eswaAg774NvMf6/OBf+9K1VIDLRGxNP25ErgNmAisSF1i0p+FLlUVMCJ3+HBgaYoPryfeLGVLihHxCLCmXOdvyyZ9bi2rlnfllYU9t4l371nDl/9hJb++aNdWqpmdct4b/PL8oRy3/xiuOH8oJ37vg//tPPa7fnxo7Pt06x6sXt6VQUM2b9k2aOhmVi3PWv+rlndl8NBsW+cu0KtvDe+s6fyB81WcACKKK42Q1EtSn8Jn4HPAAuBO4IS02wnAHenzncBUSd0ljSIbUHkydbHXSZqURp2Pzx1TslZ/TjHdR5gG0IOdWrk22697z1qOPWMl04/d8wPbjv/uCm67YjAb1neA/3DaqLuuHsTJ//oGB31hLX+4sz8/+c5ILrjp5S3bF7/Yg1kzhvJvs7NYff9dF/pq9W5r9oMg7UsLvea3K3BbenqmC3B9RNwr6SngJkknAa8DxwBExPOSbgIWAtXAaRFR6AqcStY77Qnck0qztHpSTPcYLgfoqwHtvvMxZPeN7DZyEzMfzAZeBg/ZzKX3/ZkzPj+aD39sPX/xhbc56QdL6d23hqgVmzZ24s5fDWrlWnccD9w8gFPPfwOAg//ybX76j1t7Y28u7cp5J+3Bd3/2OkP32ATAoCGbWbVs633hVUu7MnC3rHU4eMhm3lyatRZrqrPueN3ueiVqqUlmI+IVYL964quBwxo4ZgYwo574XGDc9teqDSTFSrP4Tz35yr5jt3y/+omFfPOIvXlnTRfO+tJeW+JfPWs5G95zQtzRBu66mWfn9Ga/A99l/mO9GTpqIwDvru3Mvxy/J38/fRljJ76X27+anXrX8sK8nfjwx9fz4G8GcOSJbwIw6XPv8MDNAxgzYT2P3tWf/f5iXcdoKRbRNW7PnBS309mXvca+B7xLvwHV/HruQq69aFfumz2wtatlwL+fujvPzunN2jVdOG7/MXztrOWc+eMlzPzhMGpqRLfutZz54+wJjzt/NYilr3bj+ot34/qLd8uOv+Fl+g+q5pv/sYT/PHMkmzZ0YsJn3uETh64DYMqxq7nwjOyRnD79q/nezNda7bfuaJX87rOiTBlf0mzgEGAQsAI4JyJmNXZMXw2IT6reVrO1Ufctnd/aVbASTJy8hLnPbNiu9myf/sPjYwd/q6h9H/3tP83LvanSLpStpRgRx5br3GbWuiq5pejus5mVJoCays2KTopmVjK3FM3M8jz6bGa2lVuKZmYFXuLUzGwrAfJAi5nZVvI9RTOzxN1nM7M8v/tsZrYNjz6bmeW5pWhmloRHn83MtlW5OdFJ0cxK50dyzMzynBTNzJIAWmbhqjbJSdHMSiKiorvPZVv32cwqWG1tcaURkkZI+r2kFyQ9L+lbKX6upDckzU/l87ljpktaJOlFSZNz8f0lPZe2XZLWf24WtxTNrDQt132uBs6KiKcl9QHmSXogbbs4Iv4zv7OkMcBUYCwwFHhQ0t5p7eeZZOvHPw7cDUyhmWs/u6VoZiVTRFGlMRGxLCKeTp/XAS8Awxo55EjghojYGBGvAouAiZKGAH0jYk5kK/FdAxzV3N/mpGhmpSus/dxUgUGS5ubKtPpOJ2kP4GPAEyl0uqRnJV0paecUGwYsyR1WlWLD0ue68WZxUjSzEhWZELOkuCoiJuTK5XXPJqk3cAtwZkS8Q9YV/hAwHlgGXFTYtf7KNBhvFt9TNLPStOBqfpK6kiXE6yLiVoCIWJHbfgVwV/paBYzIHT4cWJriw+uJN4tbimZWspa4p5hGiGcBL0TET3LxIbndvgQsSJ/vBKZK6i5pFDAaeDIilgHrJE1K5zweuKO5v80tRTMrXcs8p/gp4GvAc5Lmp9j3gGMljSdrky4GTs4uGc9LuglYSDZyfVoaeQY4FbgK6Ek26tyskWdwUjSzUgVQu/1JMSIeo/77gXc3cswMYEY98bnAuO2uFE6KZlYyz7xtZrYtJ0UzsySAmsqdEcJJ0cxKFBBOimZmW7n7bGaWtNDoc1vlpGhmpXNL0cwsx0nRzCyJgJqapvdrp5wUzax0bimameU4KZqZFYRHn83MtggIP7xtZpbj1/zMzJKIJpcvbc+cFM2sdB5oMTPbKtxSNDMr8CSzZmZbeUIIM7OtAgi/5mdmloQnmTUz20a4+2xmllPBLUVFGxpFkvQm8Fpr16MMBgGrWrsSVpJK/TfbPSIGb88JJN1L9vdTjFURMWV7rrejtamkWKkkzY2ICa1dDyue/806rk6tXQEzs7bESdHMLMdJcce4vLUrYCXzv1kH5XuKZmY5bimameU4KZqZ5TgplpGkKZJelLRI0tmtXR9rmqQrJa2UtKC162Ktw0mxTCR1Bi4FjgDGAMdKGtO6tbIiXAW0q4eNrWU5KZbPRGBRRLwSEZuAG4AjW7lO1oSIeARY09r1sNbjpFg+w4Alue9VKWZmbZiTYvmonpiffzJr45wUy6cKGJH7PhxY2kp1MbMiOSmWz1PAaEmjJHUDpgJ3tnKdzKwJToplEhHVwOnAfcALwE0R8Xzr1sqaImk2MAfYR1KVpJNau062Y/k1PzOzHLcUzcxynBTNzHKcFM3McpwUzcxynBTNzHKcFNsRSTWS5ktaIOlmSTttx7muknR0+vzLxiarkHSIpAObcY3Fkj6w6ltD8Tr7vFvitc6V9I+l1tGsLifF9uX9iBgfEeOATcAp+Y1pZp6SRcTXI2JhI7scApScFM3aIyfF9utRYK/Uivu9pOuB5yR1lvRjSU9JelbSyQDK/LekhZJ+B+xSOJGk/5E0IX2eIulpSc9IekjSHmTJ99uplXqQpMGSbknXeErSp9KxAyXdL+mPkn5B/e9/b0PS7ZLmSXpe0rQ62y5KdXlI0uAU+5Cke9Mxj0r6cIv8bZolXVq7AlY6SV3I5mm8N4UmAuMi4tWUWNZGxCckdQf+V9L9wMeAfYCPArsCC4Er65x3MHAFcHA614CIWCPp58C7EfGfab/rgYsj4jFJI8ne2vkIcA7wWEScJ+kLwDZJrgEnpmv0BJ6SdEtErAZ6AU9HxFmSfpjOfTrZglKnRMRLkj4JXAYc2oy/RrN6OSm2Lz0lzU+fHwVmkXVrn4yIV1P8c8C+hfuFQD9gNHAwMDsiaoClkh6u5/yTgEcK54qIhuYVPBwYI21pCPaV1Cdd46/Tsb+T9FYRv+kMSV9Kn0ekuq4GaoEbU/zXwK2Seqffe3Pu2t2LuIZZ0ZwU25f3I2J8PpCSw3v5EPDNiLivzn6fp+mpy1TEPpDddjkgIt6vpy5Fvzcq6RCyBHtARKyX9D9AjwZ2j3Tdt+v+HZi1JN9TrDz3AadK6gogaW9JvYBHgKnpnuMQ4DP1HDsH+LSkUenYASm+DuiT2+9+sq4sab/x6eMjwHEpdgSwcxN17Qe8lRLih8laqgWdgEJr92/JuuXvAK9KOiZdQ5L2a+IaZiVxUqw8vyS7X/h0WnzpF2Q9gtuAl4DngJnAH+oeGBFvkt0HvFXSM2ztvv4W+FJhoAU4A5iQBnIWsnUU/F+BgyU9TdaNf72Jut4LdJH0LHA+8Hhu23vAWEnzyO4ZnpfixwEnpfo9j5d4sBbmWXLMzHLcUjQzy3FSNDPLcVI0M8txUjQzy3FSNDPLcVI0M8txUjQzy/l/YY0JI8tyxLoAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# confusion matrix\n",
    "conf_matrix = confusion_matrix(y1_test, predicted)\n",
    "print(conf_matrix)\n",
    "\n",
    "ConfusionMatrixDisplay(conf_matrix).plot()\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "e04390b745d7540077d5afc7ea9b350ff3db0922faee759c3145cf85f5ee4c0c"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}