Localization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Libraries and Prerequisite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "O6CZJZFQJQBh",
    "outputId": "9db4887f-e25b-4f1f-af32-5172040e17c5"
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Input, Dense, Reshape, Flatten\n",
    "from keras.layers import BatchNormalization\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.optimizers import Adam\n",
    "from keras.backend import clear_session\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from tqdm import tqdm_notebook\n",
    "from tqdm import tqdm\n",
    "from numpy.random import seed\n",
    "from tensorflow import set_random_seed\n",
    "from IPython.display import display\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import ipywidgets as widgets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LD6568Gti5Rj"
   },
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "02ZHiSPXdwkA"
   },
   "outputs": [],
   "source": [
    "#Data Parameters\n",
    "num_of_classes = 4\n",
    "data_shape = (7,1)\n",
    "\n",
    "#MLP Parameters\n",
    "times_to_run = 50 #Number of times to run MLP model\n",
    "mlp_epochs = 40\n",
    "valid_split = 0.20\n",
    "\n",
    "#GAN Parameters\n",
    "latent_dim = 100\n",
    "gan_epochs = 5000\n",
    "\n",
    "#Random Seeds\n",
    "selection_seed = 150\n",
    "seed_multiplier = 1000000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of Real Data:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2dca2cb327a4b8893a12864fd552523",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatSlider(value=0.1, max=1.0, min=0.05, step=0.05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of datapoints GAN generates:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "670a020535b8444da9fdfa504d9f5756",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(IntSlider(value=250, max=1000, step=250),))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffed8f91420a40c7a8475a55c48315c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Checkbox(value=False, description='Generate missing data only')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cb1 = widgets.Checkbox(description=\"Generate missing data only\")\n",
    "slider1 = widgets.FloatSlider(value=0.1, min=0.05, max=1, step=0.05)\n",
    "slider2 = widgets.IntSlider(value=250, min=0, max=1000, step=250)\n",
    "vb = widgets.VBox(children = [slider2])\n",
    "def checkbox(button):\n",
    "    if button['new']:\n",
    "        vb.children = []\n",
    "        slider2.value = 250 - int(slider1.value*250) \n",
    "    else:\n",
    "        vb.children = [slider2]\n",
    "        experiment3 = False\n",
    "cb1.observe(checkbox, names='value')\n",
    "\n",
    "print(\"Percentage of Real Data:\")\n",
    "display(slider1)\n",
    "print(\"Number of datapoints GAN generates:\")\n",
    "display(vb)\n",
    "display(cb1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "fraction_of_data = slider1.value\n",
    "data_to_gen = slider2.value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9MsAItkbKGyo"
   },
   "source": [
    "## Data preprocessing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "MJC4EkTTR-nm"
   },
   "outputs": [],
   "source": [
    "dataset = pd.read_csv(\"./data/dataset.csv\")\n",
    "labels = dataset.Class.values\n",
    "labels = labels - 1 #Original label values are from 1 to 4\n",
    "features = dataset.drop(columns='Class').values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "WLmpaIJr4dFd"
   },
   "outputs": [],
   "source": [
    "tr_fea, X_test, tr_label, Y_test = train_test_split(features, \n",
    "                                                    labels, \n",
    "                                                    test_size=0.5,\n",
    "                                                    random_state= selection_seed,\n",
    "                                                    stratify=labels)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "poLSMMSIZNkN"
   },
   "outputs": [],
   "source": [
    "X_train = []\n",
    "Z_train = [] #This is the same as X_train, but it's used for training the GAN\n",
    "Y_train = []\n",
    "\n",
    "for idx in range(4):\n",
    "    number_filter = np.where(tr_label == idx)\n",
    "    X_filtered, Y_filtered = tr_fea[number_filter], tr_label[number_filter]\n",
    "\n",
    "    num_of_data = (int)(fraction_of_data*X_filtered.shape[0])\n",
    "    RandIndex = np.random.choice(X_filtered.shape[0], \n",
    "                                 num_of_data, \n",
    "                                 replace=False)\n",
    "    Z_train.append(X_filtered[RandIndex])\n",
    "    X_train.extend(X_filtered[RandIndex])\n",
    "    Y_train.extend(Y_filtered[RandIndex])\n",
    "\n",
    "X_train = np.asarray(X_train, dtype=np.float32)\n",
    "Y_train = np.asarray(Y_train, dtype=np.float32)\n",
    "\n",
    "X_train, Y_train = shuffle(X_train, Y_train)\n",
    "\n",
    "Y_train_encoded = to_categorical(Y_train)\n",
    "Y_test_encoded = to_categorical(Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "QwgSzR8gGyK3"
   },
   "outputs": [],
   "source": [
    "#Data must be standized using standard scaler before using the MLP.\n",
    "scaler = StandardScaler()\n",
    "X_train_transformed = scaler.fit_transform(X_train)\n",
    "X_test_transformed = scaler.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EKXtn23ztZOt"
   },
   "source": [
    "# Classification with MLP for Real Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 222,
     "referenced_widgets": [
      "242638032f3d448497a89e566a0729cc",
      "1c12d3a5f44f4a61bbeb2aa7a499fcfb",
      "c8784995eaa54ed084acc9f12a746c59",
      "187a15988ac943e6a3d1855725d1115a",
      "07ca9dda197f46e391f911867de47da6",
      "ad1a4cf935e544fa91265778f15ff117",
      "149f19108d984a6e8ec835992c0f75a2",
      "99a69a686ef44dd3bac52523cfa3febe"
     ]
    },
    "id": "OQbM_BZGkRq-",
    "outputId": "59faa767-f86c-4807-8ac6-3e4f9991063f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:5: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  \"\"\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "242638032f3d448497a89e566a0729cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=50.0), HTML(value='')))"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /tensorflow-1.15.2/python3.7/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "WARNING:tensorflow:From /tensorflow-1.15.2/python3.7/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "all_test_loss =[]\n",
    "all_test_acc = []\n",
    "history = []\n",
    "\n",
    "for i in tqdm_notebook(range(times_to_run)):\n",
    "    seed(i*seed_multiplier)\n",
    "    set_random_seed(i*seed_multiplier)\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Dense(128, input_shape=(7,), activation='relu'))\n",
    "    model.add(Dense(64, activation='relu'))\n",
    "    model.add(Dense(32, activation='relu'))\n",
    "    model.add(Dense(16, activation='relu'))\n",
    "    model.add(Dense(8, activation='relu'))\n",
    "    model.add(Dense(4, activation='softmax'))\n",
    "\n",
    "    model.compile(optimizer=Adam(0.0002, 0.5),\n",
    "                  loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "    \n",
    "    history_temp = model.fit(X_train_transformed,\n",
    "                            Y_train_encoded,\n",
    "                            epochs=mlp_epochs,\n",
    "                            batch_size=64,\n",
    "                            validation_split=valid_split,\n",
    "                            verbose=0)\n",
    "    history.append(history_temp)\n",
    "    test_loss, test_acc = model.evaluate(X_test_transformed, \n",
    "                                         Y_test_encoded, \n",
    "                                         verbose=0)\n",
    "\n",
    "    all_test_acc.append(test_acc)\n",
    "    all_test_loss.append(test_loss)\n",
    "    del(model)\n",
    "    clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 545
    },
    "id": "pXnx7rbx2z6g",
    "outputId": "73f61154-1ed0-4646-81ef-be47784ac1d5"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1dnA8d8DAmEJIpsLQRYLUkSSQGRzAwVBQRBQC+VV0LaIiFbfVsVXRWqltUrdWpfGDUVsEFREiwsuVKsViBoUUCRCKEHRFAHZIgl53j/OnWQyTCaTZTJLnu/nM5/M3HvmzjN3kidnzjn3HFFVjDHGxL8G0Q7AGGNM7bCEbowxCcISujHGJAhL6MYYkyAsoRtjTIKwhG6MMQnCEnqEicirIjK5tstGk4jkicjQCBxXReQn3v1HROTWcMpW43Umicgb1Y3TgIiMFZGtIrJXRNKjHY9xLKEH4f2S+m4lInLA7/GkqhxLVc9V1adqu2yiU9Vpqvr7mh5HRDp7yf8Iv2MvUNVzanrsem4uMENVW6jqJzU9mIjME5GDAX97Df32ny0iX4jIfhF5R0Q6hThWZ6/Mfu85QwOOs1lEtovIBL/trUTkYxFJrul7iSZL6EF4v6QtVLUF8B/gfL9tC3zl/JOEMdFWx7+PnYB11Xmif6IOcJf/356qHvLKtwVeAG4FWgPZwMIQL/F34BOgDXAzsFhE2nn77gPOB4YDD/nF8kfgTlXdU533FCssoVeBiAwWkXwRuVFEtgNPishRIvKKiBSIyE7vforfc1aIyC+9+1NE5F8iMtcru1lEzq1m2S4i8q6I7BGRN0XkQRF5poK4w4nx9yLyvne8N7w/It/+S0Rki4jsEJGbQ5yf/l7Nx79mNVZEPvXu9xORf4vILhH5RkT+KiKNKzjWPBG5w+/x9d5zvhaRywPKjhSRT0TkB68ZYLbf7ne9n7u8Wt9A37n1e/4gEVktIru9n4PCPTdVPM+tReRJ7z3sFJElfvvGiEiO9x6+EpER3vZyzVsiMtv3OUvZt49fiMh/gLe97Yu8z2G39ztykt/zm4rIn73Pc7f3O9ZURP4hIlcHvJ9PRWRswLYmIrIXaAisEZGvvO0/9c7VLhFZJyKjAz7Lh0VkmYjsA4YEO38hjAPWqeoiVS0EZgOpItIjyGfQHegD3KaqB1T1eeAzYLxXpLmqrlXVNcBBoI2I9AO6qOpzVYwr5lhCr7pjcLWETsBU3Dl80nt8PHAA+GuI5/cHNgBtgbuAx0VEqlH2WWAVrhYyG7gkxGuGE+PPgcuA9kBj4LcAItITeNg7/nHe66UQhKquBPYBZwUc91nv/iHgOu/9DATOBqaHiBsvhhFePMOAbkBg+/0+4FKgFTASuFJELvD2neH9bOXV+v4dcOzWwD+AB7z3dg/wDxFpE/AeDjs3QVR2nucDzYCTvGPd68XQD3gauN57D2cAeRWdjyDOBH6Kq3UCvIo7T+2Bj4EFfmXnAn2BQbjf4xuAEuAp4H98hUQkFeiAOzelVPVH75srQKqqniAijYCXgTe817waWCAiJ/o99efAHCAZ+BfBTReR70XkIxEZ77f9JGCNXwz7gK+87YFOAjYF1LTX+JX9TkRSvfdXAuwE7geuqSCm+KKqdgtxw/1hDfXuD8b9V08KUT4N2On3eAXwS+/+FCDXb18zQIFjqlIWlyyKgWZ++58BngnzPQWL8Ra/x9OB17z7s4Asv33NvXMwtIJj3wE84d1PxiXbThWUvRZ40e+xAj/x7s8D7vDuP4H7Ouwr192/bJDj3gfc693v7JU9wm//FOBf3v1LgFUBz/83MKWyc1OV8wwci0sgRwUp9zdfvKF+/7zHs32fs9976xoihlZemSNx/3AO4BJxYLkkXHLr5j2eCzwU4rj+n9XpwHaggd/+vwOz/T7Lpys5V31w/1CPAM4D9gCnevse9//8vW3v+z6jgO2XAB8GbJsDzPP7TFYAK3EVimuA3wO9gdeBd4Azw/l8Y/FmNfSqK1D3tQ8AEWkmIn/zvsL+gPuK30oqbifc7rujqvu9uy2qWPY44Hu/bQBbKwo4zBi3+93f7xfTcf7HVlc72lHRa+Fq4+NEpAnuq/LHqrrFi6O71wyx3YvjD7jaemXKxQBsCXh//cV1ghWIyG5gWpjH9R17S8C2LbjaqU9F56acSs5zR9xntjPIUzviapzVVXpuRKShiNzpNdv8QFlNv613Swr2Wt7v9ELgf0SkATAR940iHMcBW1W1xG9b4Dms8PfTe/2PVXWHqhar6jLct4px3u69QMuAp7TEJf1AIcuqao6qDlbV/sB64HLc7+FjwO9w38Tmh/jWHNMsoVdd4PSUvwFOBPqrakvKvuJH8hfiG6C1iDTz29YxRPmaxPiN/7G912xTUWFVXY/7Yz6X8s0t4JpuvsDVAlsC/1edGHDfUPw9CywFOqrqkcAjfsetbDrRr3FNJP6OB7aFEVegUOd5K+4zaxXkeVuBEyo45j7ctzOfY4KU8X+PPwfG4JqljsTV4n0x/BcoDPFaTwGTcDXX/RrQPBXC10BH7x+BT+A5rOq0rkrZZ7gOSPXtEJHmuPcQrFN2HdBVyo9WSa2g7L24b18HgJOBbFXNAxoB7YKUj3mW0GsuGfc1dpfXHntbpF/Qq/FmA7NFpLGIDMT13EcixsXAKBE5TVwH5u1U/nvzLPBrXEJbFBDHD8Ber0PryjBjeA6YIiI9vX8ogfEn42q/hV579M/99hXgmjq6VnDsZUB3Efm5iBwhIj8DegKvhBlbYBxBz7OqfoNr237I6zxtJCK+hP84cJm4IXUNRKSDX4dfDjDBK58BXBhGDD/ivkU1w9U+fTGU4Jqv7hGR47za/EDv2xReAi8B/kz4tXNwzRf7gRu8OAfjfh+zwj2AiFwoIi28938Orj1/qbf7RaCXiIwXkSRcM+CnqvpF4HFU9UvcObtNRJK8Tt3ewPMBrzcM13Tq+5w3A2d5HchNCP0tNGZZQq+5+4CmuNrPh8BrdfS6k3Adiztw7dYLcX/IwVQ7RlVdB1yFS9Lf4NpZ8yt52t9xHXVvq+p//bb/Fpds9wCPEnromX8Mr3rv4W0g1/vpbzpwu4jswf2xP+f33P24NtT3vREYAwKOvQMYhatd78B1Eo4KiDtclZ3nS4Ai3LeU73B9CKjqKtxX/XuB3cA/KfvWcCuuNroT1yTwLKE9jfuGtA3XpPBhwP7f4kZ9rAa+B/5E+TzwNK62GnTEVDCqehCXwM/FvfeHgEuDJdwQfu3FvAu4G/iVqq7wjl+AG6UyB3ce+gP+Y8gfEZFH/I41Acjwyt4JXOgdw1e+ifcav/Z7ztW4b3ZvAtPVGzIZb8TrKDBxTkQWAl+oasS/IZjEJSKXAlNV9bRox2KqzmrocUpEThGRE7yvqCNw7aZLKnueMRXxmrOmA5nRjsVUjyX0+HUMbvjVXtwY6iu1Fi7BNvWTiAzH9Td8S+XNOiZGWZOLMcYkCKuhG2NMgoja5FJt27bVzp07R+vljTEmLn300Uf/VdWg4+SjltA7d+5MdnZ2tF7eGGPikogEXtlcyppcjDEmQVhCN8aYBGEJ3RhjEkRMrbhTVFREfn4+hYWFlRc2UZGUlERKSgqNGjWKdijGmAAxldDz8/NJTk6mc+fOxOnslQlNVdmxYwf5+fl06dIl2uEYYwKE1eQiIiNEZIOI5IrIzCD7O4nIW+KWrFohfstuVUVhYSFt2rSxZB6jRIQ2bdrYNyhjqmnBAujcGRo0cD8XLKjsGVVTaUL3Jud/EDeTWk9gorcsmb+5uBVJeuOmV/1jdQOyZB7b7PMxpmKhEvaCBTB1KmzZAqru59SptZvUw6mh98MthbbJmyYzCzcRlL+elE1p+k6Q/cYYk9AqS9g33wz795d/zv79bnttCSehd6D88lH5lF9aCtwirL7losYCyQGL7AIgIlNFJFtEsgsKCgJ3R92OHTtIS0sjLS2NY445hg4dOpQ+PnjwYMjnZmdnc801la8zO2jQoErLGGNiU6gaeGUJ+z//CX7MirZXR20NW/wtcKaIfIJb2GAbboX3clQ1U1UzVDWjXbuar/BU2+1Rbdq0IScnh5ycHKZNm8Z1111X+rhx48YUFxdX+NyMjAweeOCBSl/jgw8+qFmQxpioqKwGXlnCPj5w4URCb6+OcBL6Nsqv55hCwHqLqvq1qo5T1XTgZm/brlqLMoi6aI8CmDJlCtOmTaN///7ccMMNrFq1ioEDB5Kens6gQYPYsGEDACtWrGDUqFEAzJ49m8svv5zBgwfTtWvXcom+RYsWpeUHDx7MhRdeSI8ePZg0aZJvhXKWLVtGjx496Nu3L9dcc03pcf3l5eVx+umn06dPH/r06VPuH8Wf/vQnTj75ZFJTU5k50/Vh5+bmMnToUFJTU+nTpw9ffVWTNYmNSUw1qYFXlrDnzIFmzcrva9bMba81qhryhhvauAnoAjTGNa+cFFCmLdDAuz8HuL2y4/bt21cDrV+//rBtFenUSdWl8vK3Tp3CPkRIt912m9599906efJkHTlypBYXF6uq6u7du7WoqEhVVZcvX67jxo1TVdV33nlHR44cWfrcgQMHamFhoRYUFGjr1q314MGDqqravHnz0vItW7bUrVu36qFDh3TAgAH63nvv6YEDBzQlJUU3bdqkqqoTJkwoPa6/ffv26YEDB1RV9csvv1Tf+Vy2bJkOHDhQ9+3bp6qqO3bsUFXVfv366QsvvKCqqgcOHCjdXx1V+ZyMiSXPPONyhIj7+cwz5fc1a1Y+nzRrVlZGJHjOEQnv+ZW9frhwi1kHzauVjkNX1WIRmQG8DjQEnlDVdSJyu3fgpcBg4I8iosC7uDUoI6ou2qN8LrroIho2bAjA7t27mTx5Mhs3bkREKCoqCvqckSNH0qRJE5o0aUL79u359ttvSUkpP5qzX79+pdvS0tLIy8ujRYsWdO3atXSc98SJE8nMPHwBmaKiImbMmEFOTg4NGzbkyy+/BODNN9/ksssuo5lXFWjdujV79uxh27ZtjB07FnAXBxlT3/i+1ftq2b5v9QCTJoWugU+a5GraW4JMi+WrgU+a5H7efLPLQ8cf72rfvu2+Mv6Pa1tYbeiqukxVu6vqCao6x9s2y0vmqOpiVe3mlfmlqla0WHGtqYv2KJ/mzZuX3r/11lsZMmQIa9eu5eWXX65wTHaTJk1K7zds2DBo+3s4ZSpy7733cvTRR7NmzRqys7Mr7bQ1pr6oqNmkpp2W4TSZTJoEeXlQUuJ+RjJ5BxO3c7nUSXtUELt376ZDBzfIZ968ebV+/BNPPJFNmzaRl5cHwMKFCyuM49hjj6VBgwbMnz+fQ4dcH/SwYcN48skn2e/95n7//fckJyeTkpLCkiVuydEff/yxdL8x8aa6Y71r2mk5aRJkZkKnTiDifmZm1n3SDiVuE3q0Tu4NN9zATTfdRHp6epVq1OFq2rQpDz30ECNGjKBv374kJydz5JFHHlZu+vTpPPXUU6SmpvLFF1+UfosYMWIEo0ePJiMjg7S0NObOnQvA/PnzeeCBB+jduzeDBg1i+/bttR67MZFWk7HetdFpGe0aeKUqalyP9K2mnaKJbM+ePaqqWlJSoldeeaXec889UY6oPPucTCSF6jisbDBEqI7Luuq0DKWkRDU/X3XnzuofgxCdonFbQ09kjz76KGlpaZx00kns3r2bK664ItohGVNranJ5fE2aTcL5Vl9bNXBV+OYbePNNuP9+9x5OPRVat4aUFFi0qHrHrYyoN/a5rmVkZGjgEnSff/45P/3pT6MSjwmffU6mugJHmoBr1vAl1s6dg48k6dTJJdjK9ld2/NqkCt99B5s2wVdflb99/jns3FlWtk0bOOmkstuwYdC9e/VeV0Q+UtWMYPtiavpcY0z8W7Cg4qF7lQ0NDGekSbCE7WvnDmfoYFWpwtdfw5o17paTA1984RL53r3ly6akwAknwEUXlSXvXr2gfXv3rSDSLKEbY2pNZWO9w2kyieZY75IS2LABVq8uS95r1sCOHWVlOneGnj1h8GDo2tUl8BNOgC5dINqXeFhCN8ZUWUW18JpenFNZDRxq9+KcPXtg1Sr44AP497/hww/LmkqSklzt+oILIDXV3Xr3hlataue1I8ESujGmSkLVwmOxyQTgwAHYvLmsjfuLL1zy/uwzVysXcbXu8eNh0CDo39+1cR8RbxmyouEvkb7F4rDFwYMH62uvvVZu27333qvTpk2r8Dlnnnmmrl69WlVVzz33XN0ZZDySb16YUF588UVdt25d6eNbb71Vly9fXpXw60y0PycTXaGGDoYzx1Kkhgb+8IPqRx+pZmWp3nGH6pQpqqefrtqhw+HxHHmk6rBhqrfdpvraazUbRljXqMlcLvXJxIkTycrKYvjw4aXbsrKyuOuuu8J6/rJly6r92kuWLGHUqFH07OkWg7r99turfSxjIilULXz+/Mg2mRw4ALm5rp37yy9h40b3eONG+Pbb8mWPO861cQ8dWtbO7bu1aVM3nZR1rqJMH+lbLNbQd+zYoe3atdMff/xRVVU3b96sHTt21JKSEp02bZr27dtXe/bsqbNmzSp9jn8NvVOnTlpQUKCqqnfccYd269ZNTz31VJ0wYUJpDT0zM1MzMjK0d+/eOm7cON23b5++//77etRRR2nnzp01NTVVc3NzdfLkybpo0SJVVX3zzTc1LS1Ne/XqpZdddpkWFhaWvt6sWbM0PT1de/XqpZ9//vlh72nz5s162mmnaXp6uqanp+v7779fuu/OO+/UXr16ae/evfXGG29UVdWNGzfq2Wefrb1799b09HTNzc097JjR/pxM5NXk4p6a1sCLilQ3b1Zdvlz1r39Vvfpq1XPOKTum/2see6zqGWeo/uIXqn/8o+rixapr1qjWYDLRmEc81tCvvdb1MNemtDS4776K97du3Zp+/frx6quvMmbMGLKysrj44osREebMmUPr1q05dOgQZ599Np9++im9e/cOepyPPvqIrKwscnJyKC4upk+fPvTt2xeAcePG8atf/QqAW265hccff5yrr76a0aNHM2rUKC688MJyxyosLGTKlCm89dZbdO/enUsvvZSHH36Ya6+9FoC2bdvy8ccf89BDDzF37lwee+yxcs9v3749y5cvJykpiY0bNzJx4kSys7N59dVXeemll1i5ciXNmjXj+++/B2DSpEnMnDmTsWPHUlhYSElJSbXOtYlflY1UCacdvLIa+O7dsG6da9f2v+XluZr+Ib/lcVq0cO3ZgwbBZZfBiSe6W7dubp8pE7MJPVp8zS6+hP74448D8Nxzz5GZmUlxcTHffPMN69evrzChv/fee4wdO7Z0CtvRo0eX7lu7di233HILu3btYu/eveWad4LZsGEDXbp0obt3FcLkyZN58MEHSxP6uHFu5b++ffvywgsvHPZ8m2bXBFOTseJV7bgsLHSVs9Wr3W3VKtdk4u/oo92wvwEDYOJENzTwhBNc4j7uuARtHomAmE3ooWrSkTRmzBiuu+46Pv74Y/bv30/fvn3ZvHkzc+fOZfXq1Rx11FFMmTKlwmlzKzNlyhSWLFlCamoq8+bNY8WKFTWK1zcFb0XT7/pPs1tSUmJJ2tR4rLivXEUJvKAA3n4bVqxwyfvTT8H3q3nMMdCvH/zP/0B6umvj7tTp8EmxTPXYXC4BWrRowZAhQ7j88suZOHEiAD/88APNmzfnyCOP5Ntvv+XVV18NeYwzzjiDJUuWcODAAfbs2cPLL79cum/Pnj0ce+yxFBUVscBvEovk5GT27Nlz2LFOPPFE8vLyyM3NBdysiWeeeWbY78em2a2fIrmUWqC9e+HVV+E3v3HNmu3bw4QJ8Oyzbu6S66+HF16ArVvdFZcvvQS33AIjR8JPf2rJvDbFbA09miZOnMjYsWPJysoCIDU1lfT0dHr06EHHjh059dRTQz6/T58+/OxnPyM1NZX27dtzyimnlO77/e9/T//+/WnXrh39+/cvTeITJkzgV7/6FQ888ACLFy8uLZ+UlMSTTz7JRRddRHFxMaeccgrTpk0L+71Mnz6d8ePH8/TTTzNixIhy0+zm5OSQkZFB48aNOe+88/jDH/7A/PnzueKKK5g1axaNGjVi0aJFdO3aNezXM9FX0xp4ZW3kRUWu6eTNN93tww/dtiZN3ARUf/iDG1nSpw94C32ZOmKTc5kqs88p+kK1gdd0gqvA43fsCFdeCU2bugS+YoWrlYu4pD10qLudeqorYyLLJucyJoFEugYOcP757grK11+Ht96Cm25y27t1c+3fQ4fCkCGuScXEDmtDNyZGVXdtzOoupXbhhbBkCVx8sRt1cumlsHw5nHUWPP64q71/+SU8/LC7RN6SeeyJuYQerSYgEx77fGpPpNbGrMpSakVF8MQT8M47LomPHeuaVH75Szdh1fbt7jUvv9wlfhPbYqrJJSkpiR07dtCmTRvEBp7GHFVlx44dNvSxFlTWbFLZ2pg1mWL2xx/h3XfhlVdg8WI38qRFCxg3Dn7+czj77DiclMoAYXaKisgI4H6gIfCYqt4ZsP944CmglVdmpqqGnNgkWKdoUVER+fn51R7jbSIvKSmJlJQUGjVqFO1QYl5NOi4bNHA180AiFc+XEmpVnu3bYdkyl8SXL3edmklJMHy4S+KjRtnwwXgRqlO00jlXcAn6K6Ar0BhYA/QMKJMJXOnd7wnkVXbcYHO5GJMoKluQONRixqo1my/lxx9VN2xQXbZM9Xe/Uz3llLLnp6SoTpum+soriT3fSSKjhnO59ANyVXWT998hCxgDrPf/vwC09O4fCXxdhX84xiScSC/0MGkSnHmmGwOem+vavR9/3M31nZ/vRqiAq9EPGOCeN3KkW6DBWjMTVzgJvQOw1e9xPtA/oMxs4A0RuRpoDgwNdiARmQpMBTi+oq54Y+JEqCaVSCz0cMMN0KgRTJvmLq3fuLHsue3aublPTj+9/DSxPXq4qWJN/VBbXR8TgXmq+mcRGQjMF5Feqlpuqj5VzcQ1z5CRkWHDJUzcqqxTszbWxhw61CX5d95xt6uuctuTk+GMM1xiP+MMN4FVcnJk3qeJL5V2inoJeraqDvce3wSgqn/0K7MOGKGqW73Hm4ABqvpdRccN1ilqTLyorFMzMOFD6I7L4mJYu7ZsbcsPPnCryoO7+vK009x48CFDoG9fG4VSn9X0StHVQDcR6QJsAyYAPw8o8x/gbGCeiPwUSAIKqh+yMdFXkyaVUDVwVZesP/nE3T78EFauhH373HOOOQYGDnQ18IED4ZRT3DwpxlQm3GGL5wH34Ua8PKGqc0Tkdlxv61IR6Qk8CrTAdZDeoKpvhDqm1dBNLKushh3OfCjgat6ff+4S98cfu585OfDDD25/w4ZuNflBg1zyHjjQHds6Lk1FQtXQY2pyLmPqWkW18Jo0qYwf78Z6v/ACLF0K3mJQNG3qknd6etmtVy83HtyYcNnkXMYEEapjs6pNKikpMGaMS+BXXOGaT1q1cpNcnXOOm5XwxBNtOlkTWVZDN/VWqFo4VN6ksmmTq4m/8gq88QYcPOjmQ7ngAncZ/eDB0LhxhII39VaoGnrMTc5lTG0KNQFWqFp4sAmumjZ1yXr6dPjJT9w472nT4LPP3JDC996DbdvgkUdcrdySualr1uRiElZNxor7RqNcf72bB6VJEzep1f33u4mshgyBa6+FYcPcivTWiWligTW5mIRVnY7Npk3h1792k1ctWeIuoxdxQwfPOcfd+ve32reJHusUNfVSuB2bN93kFjBu3tzVyu+80yX24cPhjjvcHCht29ZNzMbUhLWhm7gWqo081Mo9eXnw4IPwzDPwnXc9c1KSW61nyRL473/hxRdh8mRL5iZ+WA3dxK3K2siDTYB1xBFw6BB06eIed+vmFkC+4AK3yLFdUm/imbWhm7hVWRv5f/8Ls2bBU0+VJfWGDd20s6NGuaaU7t3rMmJjas7a0E1CqqiNfMsWdwn9ypWuTbxtW7cA8qhRrlPzyCPrNk5j6oq1oZuYVp02cnALPMya5ZL6t9+6WvpFF1kyN4nNaugmZoVqI09Ph5NOOrzJpXFjuPded/GPMfWNJXQTsypaxu3yy91l9kccARkZLqkXFLi288BFIoypTyyhm5hVURv5wYMwdy5ccgm0b1+3MRkTy6wN3cSsY48Nvv344+E3v7FkbkwgS+gmqoJ1eubnu7by7dsPL9+sGfzhD3UdpTHxwZpcTNQE6/ScMqVsoqsZM9wc4nfdVfFCysaYMpbQTdQE6/QsLnZzqqxd62rsYCNWjAmXNbmYqAl2lSe4JO9L5saY8FkN3dS5b76Be+5xTSvBZp4IdcGQMaZiVkM3Eefr+BSB5GQ3Xvyee9zl+YELJDdr5trJjTFVZwndRNSCBfDLX5Y1r+zd6y7Lv/tueP99eOwxl+BF3M/MTOv0NKa6bLZFExElJfDaazB+PBQWHr7ff7FlY0z4arxItIiMEJENIpIrIjOD7L9XRHK825cisqumQZv4tG8fPPww9OzppqcNlsyh4qtAjTHVV2lCF5GGwIPAuUBPYKKI9PQvo6rXqWqaqqYBfwFeiESwJnbl58PMmdCxoxtm2LIlPPts6FWDjDG1K5xRLv2AXFXdBCAiWcAYYH0F5ScCt9VOeCbWqcKjj8K118KPP8LYsXDddTBokGsXLyk5fNUg6/g0JjLCaXLpAGz1e5zvbTuMiHQCugBvV7B/qohki0h2QUFBVWM1MWbnTrcG5xVXuMReUgLZ2a5t3He156RJrqPTOj6NibzaHoc+AVisqoeC7VTVTCATXKdoLb+2qUMffAATJ8K2bdCoUVlbeeC6nr6flsCNibxwaujbgI5+j1O8bcFMAP5e06BM7Dp0CO64A844w81H3q4dFBWVL7N/v7us3xhTt8JJ6KuBbiLSRUQa45L20sBCItIDOAr4d+2GaGLFtm0wdCjceqtravnkE7e8WzA2isWYuldpQlfVYmAG8DrwOfCcqq4TkdtFZLRf0QlAlkZrYLuJqJdfhtRUWLUKnnzSXTDUsqWNYjEmloTVhq6qy4BlAdtmBTyeXXthmVhRWAg33ggPPABpaZCV5aa09Zkzx0axGBMr7BYpmcoAABNDSURBVNJ/U6EvvoABA1wyT06GnBwYPtzVzn1sFIsxscNmWzSHUXXNKldfDQ0bQpMmsGeP22ejWIyJXVZDN+Xs3u2GI/7iF6523rKlu2DIn41iMSY2WUI3pVauhPR0WLzYrdv5xhvw9dfBy9ooFmNijyV0A7gJtU47zV3t+d57cNNNrrnFRrEYEz8soRseeshNqHXuua7jc+DAsn1z5rhRK/5sFIsxsckSej2XmQlXXQWjR7umllatyu+3USzGxA9L6PXYE0+4ibXS0lzNPCnJLRXnPywRXPLOy3PNMXl5lsyNiVU2bLGeeuoptzTcySfDhg1w4IDbHmxYojEmPlgNvR5asAAuuwzOPht27SpL5j42LNGY+GQ19Hpm4UK49FIYPBheeglatAhezoYlGhN/rIZejyxe7JpRTj3VTbbVrJkNSzQmkVhCryfmzXNXgA4YAP/4BzRv7rbbsERjEocl9ARXWAhnneXazIuLXVPKUr/Z7G1YojGJw9rQE9iWLTBkCGzeXLZt61abXMuYRGU19AT1+uvQp48bNx7IRrEYk5gsoSeYkhK4/XZ3Gf9xx7mpcIOxUSzGJB5L6Ank++/h/PPhtttcE8qHH7o28WBsFIsxiccSeoJYtQoyMmD5cjfZ1tNPu5EsNorFmPrDEnqcKymBu+5yY8sPHXJT3155pRuxAjaKxZj6xBJ6HNu+3bWV33ijW5iipMRNfRs4wZZNrmVM/WAJPU69/jqkpsK777ox5mvXQn6+6wT1TbAVOGuiMSaxhZXQRWSEiGwQkVwRmVlBmYtFZL2IrBORZ2s3TONz8CBcfz2MGAHt20N2Nrz9tk2wZYwJ48IiEWkIPAgMA/KB1SKyVFXX+5XpBtwEnKqqO0WkfaQCrs+++spdvr96tWsn//OfoWnTiocg2tBEY+qXcGro/YBcVd2kqgeBLGBMQJlfAQ+q6k4AVf2udsM0S5a4dvKNG+H5591IlqZN3T6bYMsYA+El9A7AVr/H+d42f92B7iLyvoh8KCIjgh1IRKaKSLaIZBcUFFQv4nrm0CG45RYYOxZ69IA1a2DcuPJlbGiiMQZqr1P0CKAbMBiYCDwqIq0CC6lqpqpmqGpGu3btaumlE9fOnTBqlEvMv/iF6wANVuu2oYnGGAgvoW8DOvo9TvG2+csHlqpqkapuBr7EJXhTTZ9+6i4UeusteOQRtyBFjx7QoIGt+2mMCS6chL4a6CYiXUSkMTABWBpQZgmudo6ItMU1wWyqxTjrlawsN578wAH45z/dqkJXXOGGI9qwRGNMRSpN6KpaDMwAXgc+B55T1XUicruIjPaKvQ7sEJH1wDvA9aq6I1JBJ6riYvjtb91IlvR0+Ogjl9hvvtkNQ/RnwxKNMYFEK5qOL8IyMjI0Ozs7Kq8di3bsgIsvdmPKZ8xwQxIbN3b7GjQIPmuiiGtiMcbUHyLykapmBNtnV4rGgM8+g1NOgfffhyefhL/8pSyZgw1LNMaExxJ6lL34omtW2bkTjjoKLr/88E5PG5ZojAmHJfQo8S1EMW4cHHOMW/tz+/bgnZ42LNEYEw5rQ4+CvXthyhR3xecll7iRLMEu0+/UKfgScsaY+itUG7otEl3H8vJgzBg3O+LcufC//wsNGwYva3OxGGOqwhJ6HXrnHTeSpagIli2D4cPd9uOPd80sgazT0xhTFdaGXgeKi+HWW+Hss6FtW7dcnC+Zg3V6GmNqhyX0CNuyBc48E+64AyZPdlPfdu9evox1ehpjaoMl9Ah6/nlIS3PjzKdPd00uLVvaXCzGmMiwhB4B+/e7uVcuvBC6dYPZs2HePJuLxRgTWZbQa9natdCvn2syueEG+Ne/4IEHbC4WY0zk2SiXWqIKf/sbXHcdHHmkW8T5nHPcPlsizhhTF6yGXgt27XLDEa+80nWArllTlszB5mIxxtQNS+g1tHKlm+p2yRK46y43vvzoo8uXsWGJxpi6YAm9mkpKXAI/7TT3+L334Prr3VS3gWxYojGmLlhCr4bvvoPzzoMbb3SX8X/yCXz1lRuOaEvEGWOixRJ6Fb31FqSmwooV8PDDsGgR/OMfbhiiDUs0xkSTJfQqeOklGDYMWrVyl+9Pm+aaUGyJOGNMLLBhi2H64gs31W1Ghrvis3nzsn02LNEYEwushh6GPXtg7FhISnKX8/snc7BhicaY2GAJvRKqbjGKjRvhueegY8fDy9iwRGNMLLCEXok//QleeAHuvhsGDw5exoYlGmNiQVgJXURGiMgGEckVkZlB9k8RkQIRyfFuv6z9UOveG2+4js0JE+Daa0OXtWGJxphoq7RTVEQaAg8Cw4B8YLWILFXV9QFFF6rqjAjEGBWbN8PEiXDSSfDYY67mbYwxsSycGno/IFdVN6nqQSALGBPZsKJr/34YN87Vtl98sawTdMGC0BcPGWNMNIWT0DsAW/0e53vbAo0XkU9FZLGIBOk6BBGZKiLZIpJdUFBQjXAjT9WNL1+zxiXsE05w2xcssIuHjDGxrbY6RV8GOqtqb2A58FSwQqqaqaoZqprRrl27Wnrp2vXggzB/Pvzud+7yfh+7eMgYE+vCSejbAP8ad4q3rZSq7lDVH72HjwF9aye8urVunZvP/PzzD0/UdvGQMSbWhZPQVwPdRKSLiDQGJgBL/QuIyLF+D0cDn9deiHVD1a372bIlPPHE4bMm2sVDxphYV2lCV9ViYAbwOi5RP6eq60TkdhEZ7RW7RkTWicga4BpgSqQCjpSnn4Z333Xjztu2PXy/XTxkjIl1oqpReeGMjAzNzs6OymsH+v576NEDfvITtwZosDnNwXWA3nyza2Y5/niXzG28uTGmLonIR6qaEWyfTc4F/N//uaT+yCMVJ3NwydsSuDEmVtX7S/9XrnSX6V9zDfTuHe1ojDGm+up1Qi8udgs7H3ecG6ZojDHxrF43uTz0kFs+btEiSE6OdjTGGFMz9baG/vXXcMstMGIEjB8f7WiMMabm6m1C/81v4OBB+MtfyibesrlajDHxrF42uSxfDllZrt38Jz9x23xztfgu7/fN1QI2ssUYEx/q3Tj0wsKy0SyffQZNmrj7nTu7JB6oUyc3v7kxxsQCG4fu5+673XJyb7xRlszB5moxxsS/etWG/v337tL+iy6CYcPK77O5Wowx8a5eJfSHH4Z9++DWWw/fZ3O1GGPiXb1J6AcOwAMPwLnnwsknH77fFno2xsS7etOG/vTT8N13cMMNFZexuVqMMfGsXtTQDx2CuXPhlFPgzDOjHY0xxkRGvaihL1kCubnuEn/fRUTGGJNoEr6GrupGtpxwAowdG+1ojDEmchK+hv7uu7B6tRvh0rBhtKMxxpjISfga+l13Qbt2MHlytCMxxpjISuiEvnYtLFvmFq9o2jTa0RhjTGQldEKfO9ddHDR9untssykaYxJZwrah5+e7hH3VVdC6tc2maIxJfAlbQ7/vPjfC5brr3OObby5L5j7797vtxhiTCMJK6CIyQkQ2iEiuiMwMUW68iKiIBJ3asa7s2gV/+xtMmOAu4QebTdEYk/gqTegi0hB4EDgX6AlMFJGeQcolA78GVtZ2kFX1yCOwdy9cf33ZNptN0RiT6MKpofcDclV1k6oeBLKAMUHK/R74E1BYi/FVWWEh3H8/DB8Oqall2202RWNMogsnoXcAtvo9zve2lRKRPkBHVf1HqAOJyFQRyRaR7IKCgioHG45nnoHt2w+fhMtmUzTGJLoaj3IRkQbAPcCUysqqaiaQCW4Jupq+djCZma5mPmTI4ftsNkVjTCILp4a+Dejo9zjF2+aTDPQCVohIHjAAWBqNjtFNm9xl/pMm2SRcxpj6J5yEvhroJiJdRKQxMAFY6tupqrtVta2qdlbVzsCHwGhVrfMVoJ97zv28+OK6fmVjjIm+ShO6qhYDM4DXgc+B51R1nYjcLiKjIx1gVWRlwYABZUMVjTGmPgmrDV1VlwHLArbNqqDs4JqHVXUbNsCaNXDvvdF4dWOMib6EuVJ04ULXbn7RRdGOxBhjoiOhEvrpp0OHDpWXNcaYRJQQCX3tWli/Hn72s2hHYowx0ZMQCX3hQjcl7vjx0Y7EGGOiJ+4TuqpL6EOGwNFHRzsaY4yJnrhP6Dk5sHGjNbcYY0zcJ/SsLDjiCBg3LtqRGGNMdMV1Qld1V4cOHQpt2kQ7GmOMia64TuirVkFeXllzi60Zaoypz+J6TdGFC6FxY7jgAlsz1Bhj4raGXlLimluGD4dWrWzNUGOMiduE/sEHsG1bWXOLrRlqjKnv4jahL1wISUkw2pvv0dYMNcbUd3GZ0A8dgkWLYORISE5222zNUGNMfReXCf2f/4Rvvy1/MZGtGWqMqe/icpTLwoXQvLmrofuzNUONMfVZ3NXQi4rg+efh/PMPb2Ixxpj6LO4S+ttvw44dNneLMcYEiruE/uWX0K4djBgR7UiMMSa2xF1Cv/pq2LrVDVk0xhhTJu4SOkCTJtGOwBhjYk9cJnRjjDGHs4RujDEJIqyELiIjRGSDiOSKyMwg+6eJyGcikiMi/xKRnrUfqjHGmFAqTegi0hB4EDgX6AlMDJKwn1XVk1U1DbgLuKfWIzXGGBNSODX0fkCuqm5S1YNAFjDGv4Cq/uD3sDmgtReiMcaYcIRz6X8HYKvf43ygf2AhEbkK+F+gMXBWsAOJyFRgKsDxNg2iMcbUqlrrFFXVB1X1BOBG4JYKymSqaoaqZrRr1662XtoYYwzhJfRtQEe/xynetopkARfUJChjjDFVF05CXw10E5EuItIYmAAs9S8gIt38Ho4ENtZeiMYYY8JRaRu6qhaLyAzgdaAh8ISqrhOR24FsVV0KzBCRoUARsBOYHMmgjTHGHC6s+dBVdRmwLGDbLL/7v67luIwxxlSRXSlqjDEJIq4S+oIF0LkzNGjgfi5YEO2IjDEmdsTNEnQLFsDUqbB/v3u8ZYt7DLbsnDHGQBzV0G++uSyZ++zf77YbY4yJo4T+n/9UbbsxxtQ3cZPQK5opwGYQMMYYJ24S+pw50KxZ+W3Nmrntxhhj4iihT5oEmZnQqROIuJ+ZmdYhaowxPnEzygVc8rYEbowxwcVNDd0YY0xoltCNMSZBWEI3xpgEYQndGGMShCV0Y4xJEKIanfWcRaQA2FLB7rbAf+swnKqK5fgstuqx2KrHYquemsTWSVWDruEZtYQeiohkq2pGtOOoSCzHZ7FVj8VWPRZb9UQqNmtyMcaYBGEJ3RhjEkSsJvTMaAdQiViOz2KrHouteiy26olIbDHZhm6MMabqYrWGbowxpoosoRtjTIKIuYQuIiNEZIOI5IrIzGjH409E8kTkMxHJEZHsKMfyhIh8JyJr/ba1FpHlIrLR+3lUDMU2W0S2eecuR0TOi1JsHUXkHRFZLyLrROTX3vaon7sQsUX93IlIkoisEpE1Xmy/87Z3EZGV3t/rQhFpHEOxzRORzX7nLa2uY/OLsaGIfCIir3iPI3PeVDVmbkBD4CugK9AYWAP0jHZcfvHlAW2jHYcXyxlAH2Ct37a7gJne/ZnAn2IottnAb2PgvB0L9PHuJwNfAj1j4dyFiC3q5w4QoIV3vxGwEhgAPAdM8LY/AlwZQ7HNAy6M9u+cF9f/As8Cr3iPI3LeYq2G3g/IVdVNqnoQyALGRDmmmKSq7wLfB2weAzzl3X8KuKBOg/JUEFtMUNVvVPVj7/4e4HOgAzFw7kLEFnXq7PUeNvJuCpwFLPa2R+u8VRRbTBCRFGAk8Jj3WIjQeYu1hN4B2Or3OJ8Y+YX2KPCGiHwkIlOjHUwQR6vqN9797cDR0QwmiBki8qnXJBOV5iB/ItIZSMfV6GLq3AXEBjFw7rxmgxzgO2A57tv0LlUt9opE7e81MDZV9Z23Od55u1dEmkQjNuA+4AagxHvchgidt1hL6LHuNFXtA5wLXCUiZ0Q7oIqo+y4XM7UU4GHgBCAN+Ab4czSDEZEWwPPAtar6g/++aJ+7ILHFxLlT1UOqmgak4L5N94hGHMEExiYivYCbcDGeArQGbqzruERkFPCdqn5UF68Xawl9G9DR73GKty0mqOo27+d3wIu4X+pY8q2IHAvg/fwuyvGUUtVvvT+6EuBRonjuRKQRLmEuUNUXvM0xce6CxRZL586LZxfwDjAQaCUivqUso/736hfbCK8JS1X1R+BJonPeTgVGi0gergn5LOB+InTeYi2hrwa6eT3AjYEJwNIoxwSAiDQXkWTffeAcYG3oZ9W5pcBk7/5k4KUoxlKOL1l6xhKlc+e1Xz4OfK6q9/jtivq5qyi2WDh3ItJORFp595sCw3Bt/O8AF3rFonXegsX2hd8/aMG1Udf5eVPVm1Q1RVU74/LZ26o6iUidt2j3/gbpDT4P17v/FXBztOPxi6srbtTNGmBdtGMD/o77+l2Ea4P7Ba5t7i1gI/Am0DqGYpsPfAZ8ikuex0YpttNwzSmfAjne7bxYOHchYov6uQN6A594MawFZnnbuwKrgFxgEdAkhmJ72ztva4Fn8EbCROsGDKZslEtEzptd+m+MMQki1ppcjDHGVJMldGOMSRCW0I0xJkFYQjfGmARhCd0YYxKEJXRjjEkQltCNMSZB/D+7dfuqMRYaSgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xUVf7/8deH0AWlKggK4oooNRBAwYJlLYiigK7IiohrW7urLlZQF3Vd1lW/VizYUHQt2OBnQxTFFhQQECwIihVxQRRQkM/vj3OHTMJkUphkJpP38/HII5l7z9w5c5N8cvK5n3uOuTsiIlL11Uh3B0REJDUU0EVEsoQCuohIllBAFxHJEgroIiJZQgFdRCRLKKBnMDObamYnpLptOpnZEjM7sAKO62b2h+jrO8zs8tK0LcfrDDOzF8vbzyTH7Wdmy1J93CSv19fMPjGzn83syMp6XalYCugpFv2CxD42mtnauMfDynIsdz/U3e9Pddts5+6nufvVW3ocM2sbBf+accee6O4HbemxM8BVwC3u3sDdJ2/pwcxsjJmtL/Lz3y5ufzczm2Vma6LP3ZIcq4mZPWVmv5jZUjM7Lm5fVzObb2Y/mNn5cdtrmdk7ZrbDlr6XqkwBPcWiX5AG7t4A+AI4PG7bxFi7+CAhkgZtgPnleWKSn91H43/+3X1x1L428DTwENAYuB94OtqeyK3Ab8B2wDDgdjPrGO27FrgA6ApcamYtou3nA0+4+5fleU/ZQgG9ksT+pTazv5vZt8AEM2tsZs+Z2XIz+1/0deu450w3s79EX48wszfMbFzU9nMzO7ScbXcys9fNbLWZvWxmt5rZQ8X0uzR9vNrM3oyO96KZNYvbf3w0ylphZpcmOT+9zexbM8uJ23aUmc2Nvu5lZm+Z2Uoz+8bMbikuIJjZfWb2j7jHF0bP+drMRhZpe5iZfWBmP5nZl2Y2Jm7369HnldGIc8/YuY17fh8ze8/MVkWf+5T23CRjZrtFz18ZjUiPiNvX38wWRMf8yswuiLY3i74/K83sRzObYWab/Y6b2WdAO+DZ6H3VMbPtzeyZ6HmfmtnJce3HmNnjZvaQmf0EjCjNe4jTD6gJ3Ojuv7r7zYAB+yfo21bAYOByd//Z3d8AngGOj5rsBExz96+AT4AdzaxN9Jz/lLFfWUcBvXK1AJoQRkenEM7/hOjxjsBa4JYkz+8NLAKaAdcD95iZlaPtw8C7QFNgDAW/LImUpo/HAScC2wK1CSMozGx34Pbo+NtHr9eaBNz9HeAXCv+SHxf1FeB34Lzo/ewJHAD8NUm/ifpwSNSfPwK7AEXz978Aw4FGwGHA6VaQU94n+twoGnG+VeTYTYDngZuj93YD8LyZNS3yHjY7NyX0uRbwLPBi9LyzgIlmtmvU5B7gVHdvCHQCpkXb/wYsA5oTRreXAJvN7eHuO1P4v8dfgUnRc7cHhgDXmFn892Ig8Hh0niaS2OHRH4T5ZnZ63PaOwFwvPM/I3Gh7Ue2BDe7+cdy2OXFt5wEHRYOKtsBnwE3Ahe6+vph+VRsK6JVrIzA6GqWsdfcV7v6Eu69x99XAWGDfJM9f6u53ufvvhH9bWxJ+cUvd1sx2BHoCV7j7b3EjoIRK2ccJ7v6xu68FHgNi+dEhwHPu/noUNC6PzkFxHgGGAphZQ6B/tA13n+Xub7v7BndfAtyZoB+JHBP1b567/0L4Axb//qa7+4fuvtHd50avV5rjQvgD8Im7Pxj16xFgIXB4XJvizk0yewANgOui79E04DmicwOsB3Y3s63d/X/u/n7c9pZAG3df7+4zigTRhCzknfsCf3f3de4+G7ib8Icu5i13nxydp7UJDvMYsBvhj8nJwBVmFutvA2BVkfargIYJjtMA+ClJ2wuA0wk/s+dF/V4NfG5mT5vZa2Z2dEnvOVspoFeu5e6+LvbAzOqb2Z1RSuInwr/4jeLTDkV8G/vC3ddEXzYoY9vtgR/jtgEUm3csZR+/jft6TVyfto8/dhRQVxT3WoTR+CAzqwMMAt5396VRP9pH6YRvo35cQxitl6RQH4ClRd5fbzN7NUoprQJOK+VxY8deWmTbUqBV3OPizk2JfXb3+D9+8ccdTPhjtzQKYHtG2/8FfAq8aGaLzWxU6d7Gpp+J1UneR9LctLsvcPev3f13d59JGDUPiXb/DGxd5ClbEwJxUUnbuvtSd+/v7t0JefmrCUF+HPAocARwQ/TfU7WjgF65io6W/gbsCvR2960p+Be/uDRKKnwDNDGz+nHbklUGbEkfv4k/dvSaTYtr7O4LCIHkUAqnWyCkbhYCu0T9uKQ8fSCkjeI9TBjt7eDu2wB3xB23pNHt14RUVLwdga9K0a+SjrtDkfz3puO6+3vuPpCQjplMGB3j7qvd/W/u3o4Q2M43swNK+XpNov+KinsfZZ2W1Sk4j/OBLkXSg11IfFH2Y6Cmme0St61rMW2vAO5y9++AzkC+u68ipI7KVZZa1Smgp1dDQk56ZTSiGF3RLxiNePOBMWZWOxrdHZ7kKVvSx8eBAWa2V3QB8ypK/pl7GDiH8Ifjv0X68RPws5l1IPzbXRqPASPMbPfoD0rR/jckjE7XmVkvwh+SmOWEFFE7EpsCtDez48ysppn9CdidkB7ZEu8QRvMXWSjH60f4Hk2KvmfDzGybKGf8U9RHzGyAmf0hCpyrCNcdkqW4AIgqQ2YC15pZXTPrApxEqEopFTMbaOECukXn8WzCCBpgetSXs6MLsGdG26cVPU70X9yTwFVmtpWZ9SXk7x8s8nq7Ey623h5t+hzY38y2I1wr+aK0fc8mCujpdSNQD/gBeBv4f5X0usMIFxZXAP8g/Kv6azFty91Hd58PnEEI0t8A/yOMnpKJ5bCnufsPcdsvIATb1cBdUZ9L04ep0XuYRkhHFA0ifyUEj9WEEd9jcc9dQ7hm8GZUObJHkWOvAAYQ/otZAVwEDCjS7zJz998IAfxQwnm/DRju7gujJscDS6LU02mE7yeEQPYyIW3xFnCbu79aypcdSrjI+DXwFOFaz8tl6PaxhPO7GngA+KdH90VE7+dIQk5+JTASODLajpldYmZT4471V8LP3PeEn4fTo5+leLcC50TXiAAuJvwRmQ9c4+7fUg1ZKa6ZSJYzs0eBhe5e4f8hiEjF0Qi9GjKznma2s5nViMr6BhJysSJSheluxeqpBSFP2ZSQAjnd3T9Ib5dEZEsp5SIikiWUchERyRJpS7k0a9bM27Ztm66XFxGpkmbNmvWDuzdPtC9tAb1t27bk5+en6+VFRKokMyt6d/ImSrmIiGQJBXQRkSxRYkA3s3vN7Hszm1dCu55mtsHMhiRrJyIiFaM0OfT7CPNfP1Bcg2jmvX8S5m8WkQy1fv16li1bxrp160puLGlVt25dWrduTa1atUr9nBIDuru/bmZtS2h2FvAEYZ5tEclQy5Yto2HDhrRt25bi10aRdHN3VqxYwbJly9hpp51K/bwtzqGbWSvgKApmPUvW9hQzyzez/OXLl5f5tSZOhLZtoUaN8HliceumiEhC69ato2nTpgrmGc7MaNq0aZn/k0rFRdEbCSudlGaazvHunufuec2bJyyjLNbEiXDKKbB0KbiHz6ecoqAuUlYK5lVDeb5PqQjoeYR5mpcQVii5LW5NxpS59FJYs6bwtjVrwnYREUlBQHf3ndy9rbu3JSxo8Fd3T/nMfV8UM119cdtFJPOsWLGCbt260a1bN1q0aEGrVq02Pf7tt9+SPjc/P5+zzz67xNfo06dPSvo6ffp0BgwYkJJjVZYSL4qa2SOElUGamdkywoovtQDc/Y4K7V2cHXcMaZZE20WkYkycGP4L/uKL8Ls2diwMG1by84rTtGlTZs+eDcCYMWNo0KABF1xwwab9GzZsoGbNxGEpLy+PvLy8El9j5syZ5e9gFVfiCN3dh7p7S3ev5e6t3f0ed78jUTB39xHu/nhFdHTsWKhfv/C2+vXDdhFJvcq6bjVixAhOO+00evfuzUUXXcS7777LnnvuSW5uLn369GHRokVA4RHzmDFjGDlyJP369aNdu3bcfPPNm47XoEGDTe379evHkCFD6NChA8OGDSM2u+yUKVPo0KEDPXr04Oyzzy5xJP7jjz9y5JFH0qVLF/bYYw/mzp0LwGuvvbbpP4zc3FxWr17NN998wz777EO3bt3o1KkTM2bMSO0JS6LK3Ck6bBiMH18wIt9xx/A4frSgKhiR1KnM61bLli1j5syZ3HDDDXTo0IEZM2bwwQcfcNVVV3HJJZckfM7ChQt54YUXePfdd7nyyitZv379Zm0++OADbrzxRhYsWMDixYt58803WbduHaeeeipTp05l1qxZlKbibvTo0eTm5jJ37lyuueYahg8fDsC4ceO49dZbmT17NjNmzKBevXo8/PDDHHzwwcyePZs5c+bQrVu3LTs5ZVClFrgYNgzWroWTT4bcXDjooIJ9sdFE7AcwNpqIPU9EyqYyr1sdffTR5OTkALBq1SpOOOEEPvnkE8wsYaAGOOyww6hTpw516tRh22235bvvvqN169aF2vTq1WvTtm7durFkyRIaNGhAu3btNtV3Dx06lPHjxyft3xtvvMETTzwBwP7778+KFSv46aef6Nu3L+effz7Dhg1j0KBBtG7dmp49ezJy5EjWr1/PkUceWakBvcqM0GNGjoQbboCpU6FzZ5gyJWxXFYxIahV3faoirltttdVWm76+/PLL2W+//Zg3bx7PPvtssbXYderU2fR1Tk4OGzZsKFebLTFq1Cjuvvtu1q5dS9++fVm4cCH77LMPr7/+Oq1atWLEiBE88ECxN9mnXJUL6DVqwHnnQX4+bLstHHYYnHFG4gumoCoYkfJK13WrVatW0apVKwDuu+++lB9/1113ZfHixSxZsgSARx99tMTn7L333kyMcrjTp0+nWbNmbL311nz22Wd07tyZv//97/Ts2ZOFCxeydOlStttuO04++WT+8pe/8P7776f8PRSnygX0mM6d4d134W9/g9tug2IujKsKRqScYtet2rQBs/C56HWrinDRRRdx8cUXk5ubm/IRNUC9evW47bbbOOSQQ+jRowcNGzZkm222SfqcMWPGMGvWLLp06cKoUaO4//77Abjxxhvp1KkTXbp0oVatWhx66KFMnz6drl27kpuby6OPPso555yT8vdQLHdPy0ePHj08VV55xb1xY/dwLb7go35994ceKmj30EPubdq4m4XP8ftEqoMFCxakuwsZYfXq1e7uvnHjRj/99NP9hhtuSHOPEkv0/QLyvZi4WmVH6PH23x8++wz22KNgW6tWhUcTmjpARGLuuusuunXrRseOHVm1ahWnnnpquruUElkR0AEaN4aZM+Ghh6Bhw3BBNO46iy6aisgm5513HrNnz2bBggVMnDiR+kUvFlRRWRPQIeT5hg2D99+Hdu3gqKPgrLNg3TpNHSAi2S+rAnrMH/4QRuvnnQe33AJ77gktWyZuq4umIpItsjKgA9SuHerVn30WvvwSVqwI2+Jp6gARySZZG9BjBgyA2bOhd2/47beCvHqiEixNHSAiVVnWB3SA1q3hlVfgiivChdBddw0j96LBXFUwIhVrv/3244UXXii07cYbb+T0008v9jn9+vUjPz8fgP79+7Ny5crN2owZM4Zx48Ylfe3JkyezYMGCTY+vuOIKXn755bJ0P6FMmma3WgR0CDceXXllCOyrVoUR+4QJBftVBSNS8YYOHcqkSZMKbZs0aRJDhw4t1fOnTJlCo0aNyvXaRQP6VVddxYEHHliuY2WqahPQY/bbDz74INSsjxwJJ54YAreqYEQq3pAhQ3j++ec3LWaxZMkSvv76a/bee29OP/108vLy6NixI6NHj074/LZt2/LDDz8AMHbsWNq3b89ee+21aYpdCDXmPXv2pGvXrgwePJg1a9Ywc+ZMnnnmGS688EK6devGZ599xogRI3j88TDb9yuvvEJubi6dO3dm5MiR/Prrr5teb/To0XTv3p3OnTuzcOHCpO8v3dPsVqnZFlOlRQt46SW46iq4+mp4771QBfP115u3VRWMZKtzzw3Xl1KpWze48cbi9zdp0oRevXoxdepUBg4cyKRJkzjmmGMwM8aOHUuTJk34/fffOeCAA5g7dy5dunRJeJxZs2YxadIkZs+ezYYNG+jevTs9evQAYNCgQZx88skAXHbZZdxzzz2cddZZHHHEEQwYMIAhQ4YUOta6desYMWIEr7zyCu3bt2f48OHcfvvtnHvuuQA0a9aM999/n9tuu41x48Zx9913F/v+YtPsTp48mWnTpjF8+HBmz569aZrdvn378vPPP1O3bl3Gjx/PwQcfzKWXXsrvv//OmqIpgnKodiP0mJyckIL5f/8Pvv9eVTAilSU+7RKfbnnsscfo3r07ubm5zJ8/v1B6pKgZM2Zw1FFHUb9+fbbeemuOOOKITfvmzZvH3nvvTefOnZk4cSLz589P2p9Fixax00470b59ewBOOOEEXn/99U37Bw0aBECPHj02TehVnDfeeIPjjz8eSDzN7s0338zKlSupWbMmPXv2ZMKECYwZM4YPP/yQhg0bJj12aVTLEXq8gw4KKZihQ2HGDGjQAH7+OVTBFF1uK9XLcYmkU7KRdEUaOHAg5513Hu+//z5r1qyhR48efP7554wbN4733nuPxo0bM2LEiGKnzS3JiBEjmDx5Ml27duW+++5j+vTpW9Tf2BS8WzL97qhRozjssMOYMmUKffv25YUXXtg0ze7zzz/PiBEjOP/88zctnFFe1XaEHq9VK5g2DUaNCsE8NxdefVVVMCIVoUGDBuy3336MHDly0+j8p59+YquttmKbbbbhu+++Y+rUqUmPsc8++zB58mTWrl3L6tWrefbZZzftW716NS1btmT9+vWbprwFaNiwIatXr97sWLvuuitLlizh008/BeDBBx9k3333Ldd7S/c0uwrokZo14dpr4bnn4PPPoUePgsUzQFUwIqk0dOhQ5syZsymgx6ab7dChA8cddxx9+/ZN+vzu3bvzpz/9ia5du3LooYfSs2fPTfuuvvpqevfuTd++fenQocOm7cceeyz/+te/yM3N5bPPPtu0vW7dukyYMIGjjz6azp07U6NGDU477bRyva90T7NrHi2aWtny8vI8VluaaRYvhsGDYc4cuPzyUL9eq1YYmRdlBhs3Vn4fRcrjo48+Yrfddkt3N6SUEn2/zGyWu+claq8RegLt2oW5YIYPD5UwAwaEtEwiqoIRkUyhgF6MevXCjUd33BHy6+vWQd26hduoCkZEMokCehJmcOqp8MYbIcD//js0aZJ4OS7NAyNVRbrSrFI25fk+KaCXQs+eYY71fv3gxx9DdcvHH2s1JKl66taty4oVKxTUM5y7s2LFCuoWTQuUQBdFy+D33+Gyy+C662DvveGJJ6B58zAiX7p08/Zt2kAJ9yGIVKr169ezbNmyctd4S+WpW7curVu3platWoW2J7soWu1vLCqLnJxQ2tilS5gHpmdPePppzQMjVUetWrXYaaed0t0NqSBKuZRD7K7SDRugTx9o2jRxO1XAiEhlKjGgm9m9Zva9mc0rZv8wM5trZh+a2Uwz65r6bmaevDzIz4euXeGHH8KNSfFUASMila00I/T7gEOS7P8c2NfdOwNXA+NT0K8qoUWLMEXAyJFhtF6vXtiu1ZBEJB1KzKG7++tm1jbJ/plxD98GWm95t6qOOnXg7rvDSP3886FTJ3jmGYhPU8aqYGJTB8SqYECTe4lI6qQ6h34SUOysOmZ2ipnlm1n+8uXLU/zS6WMGZ58dpuJdtiyshjQz7s+c5oERkcqQsoBuZvsRAvrfi2vj7uPdPc/d85o3b56ql84YBx4I77wDjRqFlZFiaRVVwYhIZUhJ2aKZdQHuBg519xWpOGZV1b49vP02DBkCf/4zfPQR7LBD4uCtKhgRSaUtHqGb2Y7Ak8Dx7v7xlnep6mvSJKRf/vKXUOnSokXBBdMYVcGISKqVpmzxEeAtYFczW2ZmJ5nZaWYWmzD4CqApcJuZzTazqnX7ZwWpXTtUuvz732HN0hYtwoyNieaBAVXBiMiW063/leDZZ8PNSI0aha9zcwvvL1oFA2EEXzToi4hoPvQ0O/xwePPNMPrea69Q1hhPVTAikgoK6JWka1d4913o2BGOPBJuuqlgBSRVwYhIKiigV6IWLWD69BDQzz0Xzjor3GFaXLWLqmBEpCwU0CtZ/frw+ONwwQVw660wcGCYkrd+/c3bqQpGRMpCAT0NatSAf/0Lbr8dXngBbrklTMvbpk3xVTAiIiVRQE+j006D55+HxYvhn/+EJ5+EjRvDohhFg7nKGkWkJAroaXbwwaECJicnrIL07LObt9ESdyJSGgroGaBz5zAHzG67hQumt95aeL/KGkWkNBTQM0TLlvDaa9C/P5x5Jlx4YUi/gMoaRaR0FNAzyFZbwVNPwemnw7hxcOyxsG6dyhpFpHQU0DNMzZoh5XL99fDf/4YpeUeNUlmjiJRMAT0DmYWUy6RJYWKv//wHrr46eVmjqmBEJCXzoUvF+NOfwgyNAwfCddfBc89Br16bt9MSdyICGqFnvL32CsvZNWgA/frB009v3kZVMCICCuhVwq67hlWQOneGQYPgjjsK71cVjIiAAnqVse22MG0aHHpoqIK57LKC2RpVBSMioIBepWy1FUyeDCefHCpcTjwR1q8PX6sKRkR0UbSKqVkT7rwTWreG0aPh229DeSOEnPkXX4SR+dixuiAqUt1ohF4FmcEVV8Ddd8PLL4eLpQccECb1SjS5l0oaRaoHBfQq7KSTwnJ2CxfCnnvCokWbt9HEXiLVhwJ6Fde/f1gF6ZdfoG9feOutwvtV0ihSfSigZ4GePUMgb9w4pF7ip+BVSaNI9aGAniV23jnMqx5bhPruu8N2lTSKVB8K6Flk223h1VfhoINCaeNVV8E//qGSRpHqQgE9yzRoEC6UnnBCKGucMSOsXar1SkWyn+rQs1CtWjBhAmy/fVh8+ttvYcGCzUfqMRMnqoZdJBtohJ6lzOCaa+D//i9cJD3wQFixYvN2KmsUyR4lBnQzu9fMvjezecXsNzO72cw+NbO5ZtY99d2U8jrzTHjsMZg1K8zcuHRp4f0qaxTJHqUZod8HHJJk/6HALtHHKcDtW94tSaUhQ+DFF+Gbb8INSHPmFOxTWaNI9igxoLv768CPSZoMBB7w4G2gkZm1TFUHJTX23RfeeCPc/r/PPqEaBlTWKJJNUpFDbwV8Gfd4WbRtM2Z2ipnlm1n+8uXLU/DSUhadOoUbkFq3hkMOgUcf1UyNItmkUi+Kuvt4d89z97zmzZtX5ktLZIcdwki9d2849lj4/vtQxqiyRpGqLxUB/Stgh7jHraNtkqEaNw459cGD4fzz4YMPYPHixDM1gmZrFKkqUhHQnwGGR9UuewCr3P2bFBxXKlDduiHlcsYZ8O9/w5//DL/9tnk7lTWKVB0l3lhkZo8A/YBmZrYMGA3UAnD3O4ApQH/gU2ANcGJFdVZSKycn1Km3bg0XXxzSL08+CVtvXdAmWVmj0jIimcU8tjBlJcvLy/P8/Py0vLZs7oEHwvzqnTrBlCnQMqpTqlGjYO3SeGYhRSMilcvMZrl7XqJ9ulNUABg+HJ57Dj75BPr0KVgsQ2WNIlWHArpscvDBYbGMNWvCYhlvv62yRpGqRAFdCsnLg5kzoVEj2H9/2GYblTWKVBUK6LKZnXcOQb1jRxg4ENauLX4BalBZo0imUECXhIoulnHllYkvjqqsUSRzKKBLseIXyxgzBk49FTZsKNxGszWKZA4tcCFJxRbLaNUqzK/+9dcwaVII9qDZGkUyiUboUiKzUNVyxx0wdSr06wfffRf2qaxRJHMooEupnXoqPP00fPRRmFd90SKVNYpkEgV0KZMBA0Kt+i+/hBuQ2rZVWaNIplBAlzLr2TPMq96sGRxwANSpo7JGkUyggC7l0q5dqFXv0QOOOQb+85/E7VTWKFJ5FNCl3Jo2hZdfhkGDwrzq554Lv/9euI3KGkUqjwK6bJF69cK86ueeCzfdFBakjg/gKmsUqTwK6LLFcnJCyuWmm0IVjMoaRdJDAV1S5uyz4amnYN482GOPUN6oskaRyqOALik1cCC89lqY0KtPn3CHabKyRlXAiKSObv2XlOvZM8yl3r9/mNzr3ntDOWNRsQqYWM49VgEDqmMXKQ+N0KVCtG0Lb74ZFso4/ni4+urNZ2tUBYxIaimgS4Vp3BheeCEE9CuugBNPhF9/LdivChiR1FJAlwpVuzbcf3+Yfvf+++GPf4Qffgj7VAEjkloK6FLhzGD0aHj4YXj33VABo4m9RFJPAV0qzdChMG0a/PRTCOotW2piL5FUUkCXStWnTxilt2oFBx+s9UpFUkkBXSpdrALmgAPCeqUXXrj5HDCgib1EykoBXdJim23guefgjDNg3DgYPBh+/rlwG5U1ipSNArqkTc2acMstcPPN8OyzIR0TfwOSyhpFykYBXdLurLPCWqVffhnuMn3ttbBdZY0iZVOqgG5mh5jZIjP71MxGJdi/o5m9amYfmNlcM+uf+q5KNjvoIHjnnTDH+oEHwp13qqxRpKxKDOhmlgPcChwK7A4MNbPdizS7DHjM3XOBY4HbUt1RyX7t24eg/sc/wmmnhRWRbrsteVmjqmBECpRmcq5ewKfuvhjAzCYBA4EFcW0c2Dr6ehvg61R2UqqPbbYJ+fSLL4Z//QsWLID8/LB+aVGa3EuksNKkXFoBX8Y9XhZtizcG+LOZLQOmAGclOpCZnWJm+WaWv3z58nJ0V6qDnBy4/np44IGwGHWvXmGO9aJUBSNSWKouig4F7nP31kB/4EEz2+zY7j7e3fPcPa958+YpemnJVscfHy6QrlsX7ix94onC+1UFI1JYaQL6V8AOcY9bR9vinQQ8BuDubwF1gQT/JIuUTe/e8N570KlTWK/00ksLbkJSFYxIYaUJ6O8Bu5jZTmZWm3DR85kibb4ADgAws90IAV05FUmJVq3CSP2kk+Caa+Dww2HlSlXBiBRVYkB39w3AmcALwEeEapb5ZnaVmR0RNfsbcLKZzQEeAUa4F13OQKT86tSBu+6CO+6Al18O9WziXZkAABEISURBVOpdu2pyL5F4lq64m5eX5/n5+Wl5bana3nwzpF9Wrw5zrA8eXHzbiRNDmuaLL0IqZuxYBXyp2sxslrvnJdqnO0WlyunbF2bNgs6dQ2C/5BJN7iUCCuhSRW2/PUyfHmZrvPZaGDAAVqwo3EZljVLdKKBLlVWnTsiZ33lnWDgjNzfcaRqjskapbhTQpco75ZQwTUBODuy9d5jB0V1ljVL9KKBLVujRA95/P6yCdNZZYbm7yy5TWaNULwrokjUaN4ann4brroP//jcsnHH55ZrcS6oPlS1KVnrtNfjTn8KC1HfeGaYRKKro5F4QRvCqZZdMprJFqXb23Rc++CBM7DV8eKiGKVrxoioYyTYK6JK1WrYMd5VefDHcc0+4u/TDDwv2qwpGso0CumS1mjXD/C8vvgg//hhG7LffrioYyU4K6FItHHggzJkTUjF//WvBHaaqgpFsooAu1ca228KUKWElpGeeCYH7ggtUBSPZQwFdqpUaNUIQf/PNkI4ZOxb+8hdYvx6WLNk8mGsuGKlKFNClWurVK1TBHH10qFU/4IDNL4aqCkaqGgV0qba23hoefhgmTAizN3bpAo8+WrBfVTBS1SigS7VmBiNGwOzZsNtucOyxoW79p59UBSNVjwK6CLDzzjBjBoweHXLk3brBCSeoCkaqFgV0kUjNmjBmTAjsAP/4R5jsa8cdE1fBqAJGMk3NdHdAJNP06RNSMGedBQ88AL17hztOd9mloE3ReWBiFTCgeWAkfTRCF0lg663DeqWPPgqLFoUUzG23hfJFUAWMZCYFdJEkjjkG5s0LC2eccUZIwXz5pSpgJDMpoIuUoFUrmDoV7rgjrIzUuTM0aZK4rSpgJJ0U0EVKwQxOPTXMB9OpU1iQOiencJuiFTC6aCqVTQFdpAx23jksnnH99SHI14h+gxJVwGjaAKlsCugiZZSTAxdeGKYO6NIlbNtzTzjooII2umgq6aCALlJOnTrBO+/AlVfCE0+EO00ffjiMyHXRVNJBAV1kC9SuDVdcEUbrf/hDSLkcfjhsv33i9rpoKhVJAV0kBTp2DFPy3nADTJsWLprWqlW4jaYNkIpWqoBuZoeY2SIz+9TMRhXT5hgzW2Bm883s4dR2UyTz5eTAeeeFuvU+fcIc63XqhH1aPEMqQ4kB3cxygFuBQ4HdgaFmtnuRNrsAFwN93b0jcG4F9FWkSmjXLkwVcNddIaDXrRsW0RgypKCNqmCkIpRmhN4L+NTdF7v7b8AkYGCRNicDt7r7/wDc/fvUdlOkajELQXzBAhgwICyi0a1bKHkEVcFIxShNQG8FfBn3eFm0LV57oL2ZvWlmb5vZIYkOZGanmFm+meUvX768fD0WqUJatYL//heefx7WrYN+/eDEE8OIPBFVwciWSNVF0ZrALkA/YChwl5k1KtrI3ce7e5675zVv3jxFLy2S+fr3h/nzYdQoeOihghuSilIVjGyJ0gT0r4Ad4h63jrbFWwY84+7r3f1z4GNCgBeRSP36cO21BSWOifZr6gDZEqUJ6O8Bu5jZTmZWGzgWeKZIm8mE0Tlm1oyQglmcwn6KZI1OneCjj0KOPTZSb9gQ/vMfTR0gW6bEgO7uG4AzgReAj4DH3H2+mV1lZkdEzV4AVpjZAuBV4EJ3X1FRnRap6mrUCFUw330Hp50Gv/wCl1wSZnT8/XddNJXyMY/N2F/J8vLyPD8/Py2vLZJp5syBc84JVTBdusDcuYnbmcHGjZXbN8ksZjbL3fMS7dOdoiIZoGtXePVVeOwxWLmy+Ha6aCrJKKCLZAgzOPpoWLgQBg/efL8umkpJFNBFMky9evD443DTTSGIQwjagwYV3G2qi6aSiAK6SIY6++xwsXTWLNh//1C/vuuu4fMll+iiqWxOAV0kw3XvDi+9BC++GNYyPf54zbcuiSmgi1QRf/wj5OeHtErR9UxjYhdNlV+vnmqmuwMiUno1asBxx8Fvv4Wc+fr1Bfvq1AkXTWP59VhKJpZfh8LT90r20QhdpAoaMQImTIAdokk5atSAX38N+fULLlB+vbpSQBepooYNCzlzd1i1Cq67Dt57D779NnF75deznwK6SBZo0AD+/ndYsgQabTbPaaCbkrKfArpIFmnQAG65JdSyx6tRAwYODPPEgC6aZitdFBXJMrELn5deGi6INmkCNWvCzTfDlCmw994waRKsXRva6aJp9tAIXSQLDRsW0i/usGIFfP11WDmpceNwMTUWzGN00TQ7KKCLVAM5OWHagHfeKb6NLppWfQroItWIGbRpk3hf7drw1FMhz64ce9WkHLpINTN2bOEbjwBq1QoXVAcNgmbNQhlk7KYl5dirDo3QRaqZYcNg/PgwUo+N2CdMCPXrTz4Jq1cXvgMVlGOvKhTQRaqh2EXTjRvD52HDQiXMUUeFaQUSWbo0jNxBKZlMpZSLiBSy444heCey/fbQsye8/XaYagCUkskkGqGLSCFjxxYsrBFTrx5cdVWYGOz11wuCeYxSMplBAV1ECkmUY7/rLrj88vC5uHXlly4tPl0jlUMBXUQ2kyjHHlNc2SNAy5Zw5plhNN+mjXLslU05dBEpk0Rlj/XqhUD+5ZdhdB9fJaMce+XRCF1EyqS4lMz118Mjj0CLFps/Z80aOOssWLas8vtbnSigi0iZJUvJFBe0//e/sCBHhw7QtGn4Y6B0TGopoItIShU37/r224f5ZD7+GH78MWxbuhROPDFM+StbTgFdRFIqUdlj/fohJfPee5tXyaxfH9Ixe+0FN96oScK2RKkCupkdYmaLzOxTMxuVpN1gM3Mzy0tdF0WkKkmUYx8/vmDJvOL8/DOcd15oX6dOeG6rVkrJlEWJAd3McoBbgUOB3YGhZrZ7gnYNgXOAJBN0ikh1UFyOvbh0TJs2MHs2jBsXJgqL1bN//TUcfzwccUQY3W/cWBm9r7pKM0LvBXzq7ovd/TdgEjAwQburgX8C61LYPxHJIsWlY8aODV//3/9tPjGYOzz7LPTqFf4gnHEGvPSSbmJKpDQBvRXwZdzjZdG2TcysO7CDuz+f7EBmdoqZ5ZtZ/vLly8vcWRGp2pKlYyB5SuaBB6B3b7jnHjjooJCW2WqrUOP+zTeV0/9Mt8UXRc2sBnAD8LeS2rr7eHfPc/e85s2bb+lLi0gVlKzkMVlK5vjjw3ztNeKi1po1oQZ+++2he/cwPcFbbxUshl3dlCagfwXsEPe4dbQtpiHQCZhuZkuAPYBndGFURMqqpJTMpZduvh4qQKNGYYGOa6+FPn1gu+1g6FC4995w92p1UZqA/h6wi5ntZGa1gWOBZ2I73X2Vuzdz97bu3hZ4GzjC3fMrpMcikrXKm5JZtSrMArl8eZiCYN06mDQJTjopjPp32w3OPjvk4levrrz3U9lKnMvF3TeY2ZnAC0AOcK+7zzezq4B8d38m+RFEREpv2LDi53wpbq72WKpmypQwKi+6vF7t2nD33eGia82asMcecMABsP/+IS9fp07q30c6mBc3F2YFy8vL8/x8DeJFpPQmTtx8YrD69QtG8W3bJg74bdrAokUwc2aokHnpJXj//ZDHr1cP+vYNwX3//aFHjxD0M5WZzXL3hClt3SkqIlVGeVMyX3wRRuH77QcdO4bUzMaN0Lw57LMPfPcdXHJJGLk3aQIDBsA//wlvvBHSN1WFRugikjWSjdCXLEk+wj/oIJg+HaZNg1dfDSN6COmanj3D1AR77RUuujZpUglvphgaoYtItVCaKpn4YA4Fy+c1bx5uVpo6NUwg1rp1mIrg7LNDGeQNN8Dhh4eZIjt2hFNPDbXxn31W/CpOlU0jdBHJKhMnhgD9xRfhYunYsQUpmRo1EgdfM3jwweT5+bVrw/QDb7wRPmbODNU1EMok+/Yt+MjNDSP7ipBshK6ALiLVRrKUDCRP1xS1cSMsWABvvlnwsXhx2FenTrjRqXfvgo+2bcMfji2llIuICMlTMskuqMZMnBgCc40a0K4dzJlTOPXy9dfw+OOhFj4nB+68M9zg1K5dGMUffjj84x8wa1bFvL8MLs4REUmtWOolUUrm0kuT17gXvaCaaK3Uli1h8ODwAWGisXnz4J13Cj6eey5UzvTokfr3p5SLiAhbVuMeS8kky9/HrFoVAn2zZuXrZ7KUi0boIiIkH71DySmZ0ozgAbbZJvV9j1EOXUQkUp6ZIGPbk5VExsTn4CtigWwFdBGRUiipxr20I/ilS0PpZGwEn8qgroAuIlIKJU07kIoR/JZSQBcRKaVkKZktHcGnggK6iEgKbOkIPhUU0EVEUmRLRvCpoIAuIlIJShrBp4Lq0EVEKkmy1ZhSQSN0EZEsoYAuIpIlFNBFRLKEArqISJZQQBcRyRJpmz7XzJYDCSaj3KQZ8EMldaes1LfyUd/KR30rn2ztWxt3b55oR9oCeknMLL+4OX/TTX0rH/WtfNS38qmOfVPKRUQkSyigi4hkiUwO6OPT3YEk1LfyUd/KR30rn2rXt4zNoYuISNlk8ghdRETKQAFdRCRLZFxAN7NDzGyRmX1qZqPS3Z94ZrbEzD40s9lmlp/mvtxrZt+b2by4bU3M7CUz+yT63DiD+jbGzL6Kzt1sM+ufpr7tYGavmtkCM5tvZudE29N+7pL0Le3nzszqmtm7ZjYn6tuV0fadzOyd6Pf1UTOrnUF9u8/MPo87b90qu29xfcwxsw/M7LnoccWcN3fPmA8gB/gMaAfUBuYAu6e7X3H9WwI0S3c/or7sA3QH5sVtux4YFX09CvhnBvVtDHBBBpy3lkD36OuGwMfA7plw7pL0Le3nDjCgQfR1LeAdYA/gMeDYaPsdwOkZ1Lf7gCHp/pmL+nU+8DDwXPS4Qs5bpo3QewGfuvtid/8NmAQMTHOfMpK7vw78WGTzQOD+6Ov7gSMrtVORYvqWEdz9G3d/P/p6NfAR0IoMOHdJ+pZ2HvwcPawVfTiwP/B4tD1d5624vmUEM2sNHAbcHT02Kui8ZVpAbwV8Gfd4GRnyAx1x4EUzm2Vmp6S7Mwls5+7fRF9/C2yXzs4kcKaZzY1SMmlJB8Uzs7ZALmFEl1HnrkjfIAPOXZQ2mA18D7xE+G96pbtviJqk7fe1aN/cPXbexkbn7T9mVicdfQNuBC4CNkaPm1JB5y3TAnqm28vduwOHAmeY2T7p7lBxPPwvlzGjFOB2YGegG/AN8O90dsbMGgBPAOe6+0/x+9J97hL0LSPOnbv/7u7dgNaE/6Y7pKMfiRTtm5l1Ai4m9LEn0AT4e2X3y8wGAN+7+6zKeL1MC+hfATvEPW4dbcsI7v5V9Pl74CnCD3Um+c7MWgJEn79Pc382cffvol+6jcBdpPHcmVktQsCc6O5PRpsz4twl6lsmnbuoPyuBV4E9gUZmFlvKMu2/r3F9OyRKYbm7/wpMID3nrS9whJktIaSQ9wduooLOW6YF9PeAXaIrwLWBY4Fn0twnAMxsKzNrGPsaOAiYl/xZle4Z4ITo6xOAp9PYl0JiwTJyFGk6d1H+8h7gI3e/IW5X2s9dcX3LhHNnZs3NrFH0dT3gj4Qc/6vAkKhZus5bor4tjPsDbYQcdaWfN3e/2N1bu3tbQjyb5u7DqKjzlu6rvwmuBvcnXN3/DLg03f2J61c7QtXNHGB+uvsGPEL493s9IQd3EiE39wrwCfAy0CSD+vYg8CEwlxA8W6apb3sR0ilzgdnRR/9MOHdJ+pb2cwd0AT6I+jAPuCLa3g54F/gU+C9QJ4P6Ni06b/OAh4gqYdL1AfSjoMqlQs6bbv0XEckSmZZyERGRclJAFxHJEgroIiJZQgFdRCRLKKCLiGQJBXQRkSyhgC4ikiX+P76k5cJg+s8bAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "trainacc = []\n",
    "trainloss = []\n",
    "valacc = []\n",
    "valloss = []\n",
    "for i in range (len(history)):\n",
    "    trainacc.append(history[i].history['accuracy'])\n",
    "    trainloss.append(history[i].history['loss'])\n",
    "    valacc.append(history[i].history['val_accuracy'])\n",
    "    valloss.append(history[i].history['val_loss'])\n",
    "\n",
    "acc = np.mean(trainacc, axis=0)\n",
    "val_acc = np.mean(valacc, axis=0)\n",
    "loss = np.mean(trainloss, axis=0)\n",
    "val_loss = np.mean(valloss, axis=0)\n",
    "epochs = range(1, len(acc) + 1)\n",
    "\n",
    "plt.plot(epochs, acc, 'bo', label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy for {}%'.format(fraction_of_data*100))\n",
    "plt.legend()\n",
    "plt.figure()\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss for {}%'.format(fraction_of_data*100))\n",
    "plt.legend()\n",
    "plt.savefig(\"./results/original/Train - {}%.png\".format(fraction_of_data*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "M0wCrUPaM-Lx"
   },
   "source": [
    "# Generating Synthetic Data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "BOSmX2NaNAAK"
   },
   "outputs": [],
   "source": [
    "def build_generator():\n",
    "    model = Sequential()\n",
    "    \n",
    "    model.add(Dense(256, input_dim=latent_dim))\n",
    "    model.add(LeakyReLU(alpha=0.2))\n",
    "    model.add(BatchNormalization(momentum=0.8))\n",
    "    model.add(Dense(512))\n",
    "    model.add(LeakyReLU(alpha=0.2))\n",
    "    model.add(BatchNormalization(momentum=0.8))\n",
    "    model.add(Dense(1024))\n",
    "    model.add(LeakyReLU(alpha=0.2))\n",
    "    model.add(BatchNormalization(momentum=0.8))\n",
    "    model.add(Dense(np.prod(data_shape), activation='tanh'))\n",
    "    model.add(Reshape(data_shape))\n",
    "\n",
    "    #model.summary()\n",
    "\n",
    "    noise = Input(shape=(latent_dim,))\n",
    "    gendata = model(noise)\n",
    "    \n",
    "    return Model(noise, gendata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "tSG_iHoLNJbB"
   },
   "outputs": [],
   "source": [
    "def build_discriminator():\n",
    "\n",
    "    model = Sequential()\n",
    "\n",
    "    model.add(Flatten(input_shape=data_shape))\n",
    "    model.add(Dense(512))\n",
    "    model.add(LeakyReLU(alpha=0.2))\n",
    "    model.add(Dense(256))\n",
    "    model.add(LeakyReLU(alpha=0.2))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "    #model.summary()\n",
    "\n",
    "    data = Input(shape=data_shape)\n",
    "    validity = model(data)\n",
    "\n",
    "    return Model(data, validity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "CvGAwkWgNLep"
   },
   "outputs": [],
   "source": [
    "def train(epochs, features, batch_size=128):\n",
    "\n",
    "    valid = np.ones((batch_size, 1))\n",
    "    fake = np.zeros((batch_size, 1))\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "            idx = np.random.randint(0, features.shape[0], batch_size)\n",
    "            data = features[idx]\n",
    "\n",
    "            noise = np.random.normal(0, 1, (batch_size, latent_dim))\n",
    "            gen_data = generator.predict(noise)\n",
    "\n",
    "            d_loss_real = discriminator.train_on_batch(data, valid)\n",
    "            d_loss_fake = discriminator.train_on_batch(gen_data, fake)\n",
    "            d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)\n",
    "\n",
    "            noise = np.random.normal(0, 1, (batch_size, latent_dim))\n",
    "            g_loss = combined.train_on_batch(noise, valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ajXvrtYHNfuo"
   },
   "source": [
    "**Training GAN**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oS3jdf5HNVsp"
   },
   "outputs": [],
   "source": [
    "gen_data = []\n",
    "\n",
    "for i in tqdm(range(num_of_classes)):\n",
    "    discriminator = build_discriminator()\n",
    "    discriminator.compile(loss='binary_crossentropy',\n",
    "                          optimizer=Adam(0.0002, 0.5),\n",
    "                          metrics=['accuracy'])\n",
    "    \n",
    "    generator = build_generator()\n",
    "    noise = Input(shape=(latent_dim,))\n",
    "    gendata = generator(noise)\n",
    "    discriminator.trainable = False\n",
    "    validity = discriminator(gendata)\n",
    "    combined = Model(noise, validity)\n",
    "    combined.compile(loss='binary_crossentropy',\n",
    "                    optimizer=Adam(0.0002, 0.5))\n",
    "\n",
    "    minimaxscaler = MinMaxScaler((-1,1))\n",
    "    Z_train_transformed = minimaxscaler.fit_transform(Z_train[i])\n",
    "    Z_train_transformed = np.expand_dims(Z_train_transformed, axis=2)\n",
    "\n",
    "    train(epochs=gan_epochs,\n",
    "          features=Z_train_transformed,\n",
    "          batch_size=64)\n",
    "        \n",
    "    noise = np.random.normal(0, 1, (data_to_gen, latent_dim))\n",
    "    gen_data_temp = generator.predict(noise)\n",
    "    gen_data_temp = np.asarray(gen_data_temp, dtype=np.float32)\n",
    "    gen_data_temp = np.squeeze(gen_data_temp)\n",
    "    gen_data_temp = minimaxscaler.inverse_transform(gen_data_temp)\n",
    "\n",
    "    gen_data.append(gen_data_temp)\n",
    "\n",
    "    clear_session()\n",
    "    del(discriminator)\n",
    "    del(generator)\n",
    "    del(combined)\n",
    "\n",
    "gen_data = np.asarray(gen_data, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sdGw4QXJNvp6"
   },
   "source": [
    "# Classification with MLP for Real + Synthetic Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Uwjms7RUwAT9"
   },
   "source": [
    "**Preprocess synthetic data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "ssYq_Q_z6hZ-"
   },
   "outputs": [],
   "source": [
    "gen_label = []\n",
    "for i in range(4):\n",
    "    gen_label_temp = np.tile(i, data_to_gen)\n",
    "    gen_label.extend(gen_label_temp)\n",
    "\n",
    "gen_label = np.asarray(gen_label, dtype=np.float32)\n",
    "gen_label_encoded = to_categorical(gen_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "VbZDEbHCNzES"
   },
   "outputs": [],
   "source": [
    "gen_data_reshaped = gen_data.reshape(num_of_classes*data_to_gen, data_shape[0])\n",
    "\n",
    "X_train_gan, Y_train_gan = shuffle(gen_data_reshaped,\n",
    "                                   gen_label_encoded, \n",
    "                                   random_state=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "WttAbdqfBxif"
   },
   "outputs": [],
   "source": [
    "new_x_train = np.concatenate((X_train, X_train_gan), axis=0)\n",
    "new_y_train = np.concatenate((Y_train_encoded, Y_train_gan), axis=0)\n",
    "\n",
    "new_x_train, new_y_train = shuffle(new_x_train, new_y_train, random_state=15)\n",
    "new_x_train_transformed = scaler.fit_transform(new_x_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "f8axOzjLPoYL"
   },
   "source": [
    "**Train a new MLP on Real+Synthetic Data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vERjg_OLPav6"
   },
   "outputs": [],
   "source": [
    "all_test_loss_gan =[]\n",
    "all_test_acc_gan = []\n",
    "ganhistory = []\n",
    "\n",
    "for i in tqdm_notebook(range(50)):\n",
    "    seed(i*seed_multiplier)\n",
    "    set_random_seed(i*seed_multiplier)\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Dense(128, input_shape=(7,), activation='relu'))\n",
    "    model.add(Dense(64, activation='relu'))\n",
    "    model.add(Dense(32, activation='relu'))\n",
    "    model.add(Dense(16, activation='relu'))\n",
    "    model.add(Dense(8, activation='relu'))\n",
    "    model.add(Dense(4, activation='softmax'))\n",
    "\n",
    "    model.compile(optimizer=Adam(0.0002, 0.5),\n",
    "                  loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "    ganhistorytemp = model.fit(new_x_train_transformed,\n",
    "                    new_y_train,\n",
    "                    epochs=mlp_epochs,\n",
    "                    batch_size=64,\n",
    "                    validation_split=valid_split,\n",
    "                    verbose = 0)\n",
    "    ganhistory.append(ganhistorytemp)\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(X_test_transformed, \n",
    "                                         Y_test_encoded, \n",
    "                                         verbose=0)\n",
    "    print(\"#{} Test acc:\".format(i), test_acc)\n",
    "\n",
    "    all_test_acc_gan.append(test_acc)\n",
    "    all_test_loss_gan.append(test_loss)\n",
    "    del(model)\n",
    "    clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZcAwt0sVqAvc"
   },
   "outputs": [],
   "source": [
    "gantrainacc = []\n",
    "gantrainloss = []\n",
    "ganvalacc = []\n",
    "ganvalloss = []\n",
    "for i in range (len(ganhistory)):\n",
    "    gantrainacc.append(ganhistory[i].history['acc'])\n",
    "    gantrainloss.append(ganhistory[i].history['loss'])\n",
    "    ganvalacc.append(ganhistory[i].history['val_acc'])\n",
    "    ganvalloss.append(ganhistory[i].history['val_loss'])\n",
    "\n",
    "gan_acc = np.mean(gantrainacc, axis=0)\n",
    "gan_val_acc = np.mean(ganvalacc, axis=0)\n",
    "gan_loss = np.mean(gantrainloss, axis=0)\n",
    "gan_val_loss = np.mean(ganvalloss, axis=0)\n",
    "epochs = range(1, len(gan_acc) + 1)\n",
    "plt.plot(epochs, gan_acc, 'bo', label='Training acc')\n",
    "plt.plot(epochs, gan_val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy for {}%'.format(fraction_of_data*100))\n",
    "plt.legend()\n",
    "plt.figure()\n",
    "plt.plot(epochs, gan_loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, gan_val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss for {}%'.format(fraction_of_data*100))\n",
    "plt.legend()\n",
    "plt.savefig(\"./Results/GAN/Train - {}%.png\".format(fraction_of_data*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0ngBfCAl2sSW"
   },
   "source": [
    "# Save Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "vWOLAHG8xWu2"
   },
   "outputs": [],
   "source": [
    "AccMean = np.mean(all_test_acc)\n",
    "LossMean = np.mean(all_test_loss)\n",
    "AccStd = np.std(all_test_acc)\n",
    "LossStd = np.std(all_test_loss)\n",
    "\n",
    "GanAccMean = np.mean(all_test_acc_gan)\n",
    "GanLossMean = np.mean(all_test_loss_gan)\n",
    "GanAccStd = np.std(all_test_acc_gan)\n",
    "GanLossStd = np.std(all_test_loss_gan)\n",
    "\n",
    "lines  = list()\n",
    "lines.append(\"Original Data (Each Class: {} Real):\".format(num_of_data))\n",
    "lines.append(\"Accuracy mean: {}\".format(AccMean))\n",
    "lines.append(\"Loss mean: {}\".format(LossMean))\n",
    "lines.append(\"Accuracy STD: {}\".format(AccStd))\n",
    "lines.append(\"Loss STD: {} \\n\".format(LossStd))\n",
    "lines.append(\"Maximum Accuracy: {}\".format(np.max(all_test_acc)))\n",
    "lines.append(\"Loss of Maximum Accuracy: {}\".format(\n",
    "    all_test_loss[np.argmax(all_test_acc)]))\n",
    "\n",
    "lines.append(\"\\n ================== \\n\")\n",
    "lines.append(\"Original + GAN Data\" +\n",
    "             \"(Each Class: {} Real + {} GAN):\".format(num_of_data, \n",
    "                                                      data_to_gen))\n",
    "lines.append(\"Accuracy mean: {}\".format(GanAccMean))\n",
    "lines.append(\"Loss mean: {}\".format(GanLossMean))\n",
    "lines.append(\"Accuracy STD: {}\".format(GanAccStd))\n",
    "lines.append(\"Loss STD: {} \\n\".format(GanLossStd))\n",
    "lines.append(\"Maximum Accuracy: {}\".format(np.max(all_test_acc_gan)))\n",
    "lines.append(\"Loss of Maximum Accuracy: {}\".format(\n",
    "    all_test_loss_gan[np.argmax(all_test_acc_gan)]))\n",
    "\n",
    "\n",
    "file_dir = \"./results/Test - {}%.txt\".format(fraction_of_data*100)\n",
    "with open(file_dir, \"w\") as filehandle:\n",
    "    for items in lines:\n",
    "        filehandle.write('%s\\n' % items)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Localization.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "07ca9dda197f46e391f911867de47da6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "149f19108d984a6e8ec835992c0f75a2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "187a15988ac943e6a3d1855725d1115a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_99a69a686ef44dd3bac52523cfa3febe",
      "placeholder": "​",
      "style": "IPY_MODEL_149f19108d984a6e8ec835992c0f75a2",
      "value": " 50/50 [03:20&lt;00:00,  4.00s/it]"
     }
    },
    "1c12d3a5f44f4a61bbeb2aa7a499fcfb": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "242638032f3d448497a89e566a0729cc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_c8784995eaa54ed084acc9f12a746c59",
       "IPY_MODEL_187a15988ac943e6a3d1855725d1115a"
      ],
      "layout": "IPY_MODEL_1c12d3a5f44f4a61bbeb2aa7a499fcfb"
     }
    },
    "99a69a686ef44dd3bac52523cfa3febe": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ad1a4cf935e544fa91265778f15ff117": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c8784995eaa54ed084acc9f12a746c59": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ad1a4cf935e544fa91265778f15ff117",
      "max": 50,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_07ca9dda197f46e391f911867de47da6",
      "value": 50
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}