Named Entity Recognition using LSTMs with Keras.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow version:  2.3.0-dev20200605\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print('Tensorflow version: ', tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and explore NER data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sentence #</th>\n",
       "      <th>Word</th>\n",
       "      <th>POS</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>NNS</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>of</td>\n",
       "      <td>IN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>NNS</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>have</td>\n",
       "      <td>VBP</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>marched</td>\n",
       "      <td>VBN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>through</td>\n",
       "      <td>IN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>London</td>\n",
       "      <td>NNP</td>\n",
       "      <td>B-geo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>to</td>\n",
       "      <td>TO</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>protest</td>\n",
       "      <td>VB</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>the</td>\n",
       "      <td>DT</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>war</td>\n",
       "      <td>NN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>in</td>\n",
       "      <td>IN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>Iraq</td>\n",
       "      <td>NNP</td>\n",
       "      <td>B-geo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>and</td>\n",
       "      <td>CC</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>demand</td>\n",
       "      <td>VB</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>the</td>\n",
       "      <td>DT</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>withdrawal</td>\n",
       "      <td>NN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>of</td>\n",
       "      <td>IN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>British</td>\n",
       "      <td>JJ</td>\n",
       "      <td>B-gpe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>troops</td>\n",
       "      <td>NNS</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Sentence #           Word  POS    Tag\n",
       "0   Sentence: 1      Thousands  NNS      O\n",
       "1   Sentence: 1             of   IN      O\n",
       "2   Sentence: 1  demonstrators  NNS      O\n",
       "3   Sentence: 1           have  VBP      O\n",
       "4   Sentence: 1        marched  VBN      O\n",
       "5   Sentence: 1        through   IN      O\n",
       "6   Sentence: 1         London  NNP  B-geo\n",
       "7   Sentence: 1             to   TO      O\n",
       "8   Sentence: 1        protest   VB      O\n",
       "9   Sentence: 1            the   DT      O\n",
       "10  Sentence: 1            war   NN      O\n",
       "11  Sentence: 1             in   IN      O\n",
       "12  Sentence: 1           Iraq  NNP  B-geo\n",
       "13  Sentence: 1            and   CC      O\n",
       "14  Sentence: 1         demand   VB      O\n",
       "15  Sentence: 1            the   DT      O\n",
       "16  Sentence: 1     withdrawal   NN      O\n",
       "17  Sentence: 1             of   IN      O\n",
       "18  Sentence: 1        British   JJ  B-gpe\n",
       "19  Sentence: 1         troops  NNS      O"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/ner_dataset.csv', encoding='latin1')\n",
    "df = df.fillna(method='ffill')\n",
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique words in corpus:  35178\n",
      "Unique tags in corpus:  17\n"
     ]
    }
   ],
   "source": [
    "print('Unique words in corpus: ', df['Word'].nunique())\n",
    "print('Unique tags in corpus: ', df['Tag'].nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = list(set(df['Word'].values))\n",
    "words.append('ENDPAD')\n",
    "num_words = len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = list(set(df['Tag'].values))\n",
    "num_tags = len(tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(35179, 17)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_words, num_tags"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieve sentences and corresponding tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SentenceGetter(object):\n",
    "    def __init__(self, df):\n",
    "        self.n_sent = 1\n",
    "        self.df = df\n",
    "        agg_func = lambda s: [(w, p, t) for w, p, t in zip(s['Word'].values.tolist(),\n",
    "                                                          s['POS'].values.tolist(),\n",
    "                                                          s['Tag'].values.tolist())]\n",
    "        self.grouped = self.df.groupby('Sentence #').apply(agg_func)\n",
    "        self.sentences = [s for s in self.grouped]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "getter = SentenceGetter(df)\n",
    "sentences = getter.sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Thousands', 'NNS', 'O'),\n",
       " ('of', 'IN', 'O'),\n",
       " ('demonstrators', 'NNS', 'O'),\n",
       " ('have', 'VBP', 'O'),\n",
       " ('marched', 'VBN', 'O'),\n",
       " ('through', 'IN', 'O'),\n",
       " ('London', 'NNP', 'B-geo'),\n",
       " ('to', 'TO', 'O'),\n",
       " ('protest', 'VB', 'O'),\n",
       " ('the', 'DT', 'O'),\n",
       " ('war', 'NN', 'O'),\n",
       " ('in', 'IN', 'O'),\n",
       " ('Iraq', 'NNP', 'B-geo'),\n",
       " ('and', 'CC', 'O'),\n",
       " ('demand', 'VB', 'O'),\n",
       " ('the', 'DT', 'O'),\n",
       " ('withdrawal', 'NN', 'O'),\n",
       " ('of', 'IN', 'O'),\n",
       " ('British', 'JJ', 'B-gpe'),\n",
       " ('troops', 'NNS', 'O'),\n",
       " ('from', 'IN', 'O'),\n",
       " ('that', 'DT', 'O'),\n",
       " ('country', 'NN', 'O'),\n",
       " ('.', '.', 'O')]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define mappings between sentences and tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2idx = {w: i+1 for i, w in enumerate(words)}\n",
    "tag2idx = {t: i for i, t in enumerate(tags)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Padding input sentences and creating train/test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAQpUlEQVR4nO3df6zddX3H8edr4G83W6RrWNusLDYzuEQxN4DRLA42KGAsf6jBmNGZJv2HZbqYONiWEH8lNVlETCZJA8xinMBQR4NG11WM2R8iRRzyQ8ZVy2gD9moL6ow/iu/9cT41x3oP995yem57Ps9HcnO/38/3c875fPK5ed3P/ZzP/Z5UFZKkPvzOcjdAkjQ5hr4kdcTQl6SOGPqS1BFDX5I6cupyN+DZnH766bV+/frlboYknVTuvffeH1TVqvmuLSr0k+wFfgw8AxyuqpkkpwG3AuuBvcDbqupQkgDXAZcAPwX+qqq+0Z5nM/CP7Wk/WFU7nu11169fz549exbTRElSk+SxUdeWsrzzZ1X1mqqaaedXAburagOwu50DXAxsaF9bgetbI04DrgHOBc4BrkmycikdkSQ9N89lTX8TcGSmvgO4bKj85hr4GrAiyRnARcCuqjpYVYeAXcDG5/D6kqQlWmzoF/AfSe5NsrWVra6qJ9rxk8DqdrwGeHzosfta2ajy35Bka5I9SfbMzc0tsnmSpMVY7Bu5b6iq/Ul+H9iV5NvDF6uqkozlfg5VtR3YDjAzM+M9IiRpjBY106+q/e37AeBzDNbkv9+WbWjfD7Tq+4F1Qw9f28pGlUuSJmTB0E/ykiS/e+QYuBB4ANgJbG7VNgN3tOOdwBUZOA94ui0DfQm4MMnK9gbuha1MkjQhi1neWQ18brATk1OBf62qLya5B7gtyRbgMeBtrf4XGGzXnGWwZfOdAFV1MMkHgHtavfdX1cGx9USStKCcyLdWnpmZKffpS9LSJLl3aHv9b/A2DJLUkRP6Ngya3/qrPj9v+d5tl064JZJONs70Jakjhr4kdcTQl6SOGPqS1BFDX5I64u6dKTJqVw+4s0fSgDN9SeqIoS9JHTH0Jakjhr4kdcTQl6SOuHvnBPZsu3Ek6Vg405ekjhj6ktQRQ1+SOmLoS1JHfCO3E37wiiRwpi9JXTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHFh36SU5Jcl+SO9v5mUnuTjKb5NYkz2/lL2jns+36+qHnuLqVP5LkonF3RpL07JYy038X8PDQ+YeBa6vqFcAhYEsr3wIcauXXtnokOQu4HHgVsBH4eJJTnlvzJUlLsajQT7IWuBS4oZ0HOB+4vVXZAVzWjje1c9r1C1r9TcAtVfXzqvoeMAucM45OSJIWZ7Ez/Y8C7wV+1c5fDjxVVYfb+T5gTTteAzwO0K4/3er/unyex/xakq1J9iTZMzc3t4SuSJIWsuBn5CZ5E3Cgqu5N8sbj3aCq2g5sB5iZmanj/XonglGfXytJ47aYD0Z/PfDmJJcALwR+D7gOWJHk1DabXwvsb/X3A+uAfUlOBV4G/HCo/Ijhx0iSJmDB5Z2qurqq1lbVegZvxH65qt4B3AW8pVXbDNzRjne2c9r1L1dVtfLL2+6eM4ENwNfH1hNJ0oIWM9Mf5e+AW5J8ELgPuLGV3wh8MskscJDBLwqq6sEktwEPAYeBK6vqmefw+pKkJVpS6FfVV4CvtOPvMs/um6r6GfDWEY//EPChpTZSkjQe/keuJHXE0JekjjyXNX1NgVHbRfduu3TCLZE0Cc70Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRb7g2QX4WrqTl5kxfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRP0RF8xr1gS97t1064ZZIGidn+pLUkQVDP8kLk3w9yX8neTDJ+1r5mUnuTjKb5NYkz2/lL2jns+36+qHnurqVP5LkouPVKUnS/BYz0/85cH5VvRp4DbAxyXnAh4Frq+oVwCFgS6u/BTjUyq9t9UhyFnA58CpgI/DxJKeMszOSpGe3YOjXwE/a6fPaVwHnA7e38h3AZe14UzunXb8gSVr5LVX186r6HjALnDOWXkiSFmVRa/pJTknyTeAAsAv4DvBUVR1uVfYBa9rxGuBxgHb9aeDlw+XzPGb4tbYm2ZNkz9zc3NJ7JEkaaVGhX1XPVNVrgLUMZuevPF4NqqrtVTVTVTOrVq06Xi8jSV1a0u6dqnoKuAt4HbAiyZEtn2uB/e14P7AOoF1/GfDD4fJ5HiNJmoDF7N5ZlWRFO34R8BfAwwzC/y2t2mbgjna8s53Trn+5qqqVX95295wJbAC+Pq6OSJIWtph/zjoD2NF22vwOcFtV3ZnkIeCWJB8E7gNubPVvBD6ZZBY4yGDHDlX1YJLbgIeAw8CVVfXMeLsjSXo2C4Z+Vd0PnD1P+XeZZ/dNVf0MeOuI5/oQ8KGlN1OSNA7+R64kdcTQl6SOeMO142DUzcokabk505ekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakj3k9fSzLqswL2brt0wi2RdCyc6UtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR3x3jvPwaj70EjSicqZviR1xNCXpI4sGPpJ1iW5K8lDSR5M8q5WflqSXUkebd9XtvIk+ViS2ST3J3nt0HNtbvUfTbL5+HVLkjSfxcz0DwPvqaqzgPOAK5OcBVwF7K6qDcDudg5wMbChfW0FrofBLwngGuBc4BzgmiO/KCRJk7Fg6FfVE1X1jXb8Y+BhYA2wCdjRqu0ALmvHm4Cba+BrwIokZwAXAbuq6mBVHQJ2ARvH2htJ0rNa0pp+kvXA2cDdwOqqeqJdehJY3Y7XAI8PPWxfKxtVfvRrbE2yJ8meubm5pTRPkrSARYd+kpcCnwHeXVU/Gr5WVQXUOBpUVduraqaqZlatWjWOp5QkNYsK/STPYxD4n6qqz7bi77dlG9r3A618P7Bu6OFrW9mocknShCxm906AG4GHq+ojQ5d2Akd24GwG7hgqv6Lt4jkPeLotA30JuDDJyvYG7oWtTJI0IYv5j9zXA38JfCvJN1vZ3wPbgNuSbAEeA97Wrn0BuASYBX4KvBOgqg4m+QBwT6v3/qo6OJZeSJIWZcHQr6r/AjLi8gXz1C/gyhHPdRNw01IaKEkaH/8jV5I6YuhLUkcMfUnqiKEvSR3xfvo6rkZ95sDebZdOuCWSwJm+JHXF0Jekjri8o7HwoyOlk4MzfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEW+tvAjeNljStHCmL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdWTB0E9yU5IDSR4YKjstya4kj7bvK1t5knwsyWyS+5O8dugxm1v9R5NsPj7dkSQ9m8XM9D8BbDyq7Cpgd1VtAHa3c4CLgQ3taytwPQx+SQDXAOcC5wDXHPlFIUmanAVDv6q+Chw8qngTsKMd7wAuGyq/uQa+BqxIcgZwEbCrqg5W1SFgF7/9i0SSdJwd65r+6qp6oh0/Caxux2uAx4fq7Wtlo8p/S5KtSfYk2TM3N3eMzZMkzec5v5FbVQXUGNpy5Pm2V9VMVc2sWrVqXE8rSeLYQ//7bdmG9v1AK98PrBuqt7aVjSqXJE3QsYb+TuDIDpzNwB1D5Ve0XTznAU+3ZaAvARcmWdnewL2wlUmSJmjBj0tM8mngjcDpSfYx2IWzDbgtyRbgMeBtrfoXgEuAWeCnwDsBqupgkg8A97R676+qo98cliQdZwuGflW9fcSlC+apW8CVI57nJuCmJbVOkjRW/keuJHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdWXCffk/WX/X55W6CJB1XzvQlqSPO9LUsRv1VtXfbpRNuidQXZ/qS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXEffo6obh/Xzq+nOlLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQR9+nrpOD+fWk8nOlLUkcMfUnqiKEvSR3pck1/1PqwTj6u9UtL40xfkjpi6EtSR7pc3tH0c9lHmp8zfUnqyMRn+kk2AtcBpwA3VNW2SbdB/TqWN/H960DTZKIz/SSnAP8MXAycBbw9yVmTbIMk9WzSM/1zgNmq+i5AkluATcBDx+PF3JqpcRjXz9Govxh8/0GTNOnQXwM8PnS+Dzh3uEKSrcDWdvqTJI8s8TVOB35wzC08udjXk0g+vOiqpwM/WEL9k9lJP65LMMm+/uGoCyfc7p2q2g5sP9bHJ9lTVTNjbNIJy75OJ/s6nU6Uvk56985+YN3Q+dpWJkmagEmH/j3AhiRnJnk+cDmwc8JtkKRuTXR5p6oOJ/lr4EsMtmzeVFUPjvlljnlp6CRkX6eTfZ1OJ0RfU1XL3QZJ0oT4H7mS1BFDX5I6MjWhn2RjkkeSzCa5arnbM05J1iW5K8lDSR5M8q5WflqSXUkebd9XLndbxyXJKUnuS3JnOz8zyd1tfG9tGwFOeklWJLk9ybeTPJzkddM6rkn+tv38PpDk00leOE3jmuSmJAeSPDBUNu9YZuBjrd/3J3ntpNo5FaHfwe0dDgPvqaqzgPOAK1v/rgJ2V9UGYHc7nxbvAh4eOv8wcG1VvQI4BGxZllaN33XAF6vqlcCrGfR56sY1yRrgb4CZqvoTBhs5Lme6xvUTwMajykaN5cXAhva1Fbh+Qm2cjtBn6PYOVfUL4MjtHaZCVT1RVd9oxz9mEAxrGPRxR6u2A7hseVo4XknWApcCN7TzAOcDt7cqU9HXJC8D/hS4EaCqflFVTzGl48pgt+CLkpwKvBh4gika16r6KnDwqOJRY7kJuLkGvgasSHLGJNo5LaE/3+0d1ixTW46rJOuBs4G7gdVV9US79CSwepmaNW4fBd4L/Kqdvxx4qqoOt/NpGd8zgTngX9pS1g1JXsIUjmtV7Qf+CfhfBmH/NHAv0zmuw0aN5bJl1rSEfheSvBT4DPDuqvrR8LUa7L096fffJnkTcKCq7l3utkzAqcBrgeur6mzg/zhqKWeKxnUlg9ntmcAfAC/ht5dCptqJMpbTEvpTf3uHJM9jEPifqqrPtuLvH/mTsH0/sFztG6PXA29OspfBMt35DNa9V7RlAZie8d0H7Kuqu9v57Qx+CUzjuP458L2qmquqXwKfZTDW0ziuw0aN5bJl1rSE/lTf3qGtad8IPFxVHxm6tBPY3I43A3dMum3jVlVXV9XaqlrPYBy/XFXvAO4C3tKqTUtfnwQeT/LHregCBrcZn7pxZbCsc16SF7ef5yN9nbpxPcqosdwJXNF28ZwHPD20DHR8VdVUfAGXAP8DfAf4h+Vuz5j79gYGfxbeD3yzfV3CYK17N/Ao8J/Aacvd1jH3+43Ane34j4CvA7PAvwEvWO72jamPrwH2tLH9d2DltI4r8D7g28ADwCeBF0zTuAKfZvB+xS8Z/BW3ZdRYAmGw4/A7wLcY7GqaSDu9DYMkdWRalnckSYtg6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SO/D9u+G926OILUgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist([len(s) for s in sentences], bins=50)\n",
    "plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "max_len = 50\n",
    "X = [[word2idx[w[0]] for w in s] for s in sentences]\n",
    "X = pad_sequences(maxlen = max_len, sequences = X, padding='post', value=num_words-1)\n",
    "\n",
    "y = [[tag2idx[w[2]] for w in s] for s in sentences]\n",
    "y = pad_sequences(maxlen = max_len, sequences = y, padding = 'post', value = tag2idx['O'])\n",
    "y = [to_categorical(i, num_classes=num_tags) for i in y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build and compile a Bidirectional LSTM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import Model, Input\n",
    "from tensorflow.keras.layers import LSTM, Embedding, Dense\n",
    "from tensorflow.keras.layers import TimeDistributed, SpatialDropout1D, Bidirectional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 50)]              0         \n",
      "_________________________________________________________________\n",
      "embedding (Embedding)        (None, 50, 50)            1758950   \n",
      "_________________________________________________________________\n",
      "spatial_dropout1d (SpatialDr (None, 50, 50)            0         \n",
      "_________________________________________________________________\n",
      "bidirectional (Bidirectional (None, 50, 200)           120800    \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 50, 17)            3417      \n",
      "=================================================================\n",
      "Total params: 1,883,167\n",
      "Trainable params: 1,883,167\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "input_word = Input(shape=(max_len, ))\n",
    "model = Embedding(input_dim = num_words, output_dim = max_len, input_length = max_len)(input_word)\n",
    "model = SpatialDropout1D(0.1)(model)\n",
    "model = Bidirectional(LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(model)\n",
    "out = TimeDistributed(Dense(num_tags, activation='softmax'))(model)\n",
    "model = Model(input_word, out)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from livelossplot.tf_keras import PlotLossesCallback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "1080/1080 [==============================] - 86s 80ms/step - loss: 0.1906 - accuracy: 0.9570 - val_loss: 0.0675 - val_accuracy: 0.9806\n",
      "Epoch 2/3\n",
      "1080/1080 [==============================] - 85s 79ms/step - loss: 0.0527 - accuracy: 0.9846 - val_loss: 0.0499 - val_accuracy: 0.9849\n",
      "Epoch 3/3\n",
      "1080/1080 [==============================] - 83s 77ms/step - loss: 0.0381 - accuracy: 0.9885 - val_loss: 0.0477 - val_accuracy: 0.9855\n"
     ]
    }
   ],
   "source": [
    "early_stopping = EarlyStopping(monitor='val_accuracy', patience=1, verbose=0, mode='max', restore_best_weights=False)\n",
    "callbacks = [PlotLossesCallback(), early_stopping]\n",
    "\n",
    "history = model.fit(\n",
    "    x_train, np.array(y_train),\n",
    "    validation_split=0.2,\n",
    "    batch_size = 32,\n",
    "    epochs = 3,\n",
    "    verbose = 1,\n",
    "#    callbacks = callbacks\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate Named Entity Recognition model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXgV5dn48e9NVkgCZCPs+xoEARPcWkGsFeuCoIjUBSKC1r2+tm6t8lOs1uV1eV2REsANKYpaa1VU3KpIwr5DZJEAkrAkECBku39/nElyCAk5gZxMknN/rutcmTPzzJx7Jidz53meeWZEVTHGGGN81cTtAIwxxjQsljiMMcbUiCUOY4wxNWKJwxhjTI1Y4jDGGFMjljiMMcbUiF8Th4gMF5H1IpIhIvdWsryTiHwhIitE5CsRae+17O8issp5jfGa30VEfnS2+Y6IhPpzH4wxxhzNb4lDRIKAF4ELgURgrIgkVij2FDBLVfsDDwOPOeteBAwCBgCnA3eLSHNnnb8Dz6hqd2AfMMFf+2CMMeZY/qxxDAYyVHWTqhYAs4ERFcokAl860wu8licC36hqkaoeBFYAw0VEgGHAXKfcTOAyP+6DMcaYCoL9uO12wDav95l4ag/elgOjgOeAkUCUiMQ68x8SkaeBZsC5wBogFshR1SKvbbarLpC4uDjt3Lnzie+JMcYEoMWLF+9W1fiK8/2ZOHxxN/CCiIwHvgG2A8Wq+pmIJAPfA9nAD0BxTTYsIpOASQAdO3YkPT29NuM2xphGT0S2Vjbfn01V24EOXu/bO/PKqOoOVR2lqgOBB5x5Oc7PR1V1gKqeDwiwAdgDtBSR4Kq26bXtqaqapKpJ8fHHJExjjDEnyJ+JIw3o4VwFFQpcBXzoXUBE4kSkNIb7gOnO/CCnyQoR6Q/0Bz5Tzx0ZFwBXOOuMAz7w4z4YY4ypwG+Jw+mHuBX4FFgLzFHV1SLysIhc6hQbCqwXkQ1AAvCoMz8E+FZE1gBTgWu8+jXuAe4SkQw8fR7/8Nc+GGOMOZYEwm3Vk5KStGIfR2FhIZmZmeTn57sUlfEWHh5O+/btCQkJcTsUY4xDRBaralLF+W53jrsmMzOTqKgoOnfujOcqX+MWVWXPnj1kZmbSpUsXt8MxxlQjYG85kp+fT2xsrCWNekBEiI2NtdqfMQ1EwCYOwJJGPWK/C2MajoBtqjLGmMboQH4hGVl5nld2HrcN60FkWO2e6i1xGGNMA6Oq7M4rKEsOP2XlsTHrABlZeezaf6SsXGhQE0YObEfv1s2Ps7Was8TRyBUVFREcbL9mYxqikhJle87hsuSQkZXHRudn7uHCsnIRoUF0bxXJ2d3j6N4qku7xkXRvFUnHmGYEB9V+j4SdUVx02WWXsW3bNvLz87njjjuYNGkSn3zyCffffz/FxcXExcXxxRdfkJeXx2233UZ6ejoiwkMPPcTll19OZGQkeXl5AMydO5ePPvqIGTNmMH78eMLDw1m6dClnn302V111FXfccQf5+fk0bdqU1NRUevXqRXFxMffccw+ffPIJTZo0YeLEifTt25fnn3+e999/H4D58+fz0ksvMW/ePDcPlTGNWmFxCVv3HCxrYipNDpuyD3K4sPxuS7ERoXRrFclF/duUJYceCZG0bh5ep/2EljiA//ev1azZsb9Wt5nYtjkPXdL3uGWmT59OTEwMhw8fJjk5mREjRjBx4kS++eYbunTpwt69ewF45JFHaNGiBStXrgRg37591X5+ZmYm33//PUFBQezfv59vv/2W4OBgPv/8c+6//37effddpk6dypYtW1i2bBnBwcHs3buX6Ohobr75ZrKzs4mPjyc1NZXrr7/+5A+IMYZDBUVsyj7oJIcDZYli655DFJWUj6lr17Ip3VpFcnqXWE8NwnnFRNSPxw9Z4nDR888/X/af/LZt25g6dSrnnHNO2ViGmJgYAD7//HNmz55dtl50dHS12x49ejRBQUEA5ObmMm7cODZu3IiIUFhYWLbdm266qawpq/Tzrr32Wt544w1SUlL44YcfmDVrVi3tsTGBIedQwTG1h4ysPLbnHC4rE9RE6BTbjO7xkVzQt7Wn9tAqiq7xEUTUcmd2bavf0dWR6moG/vDVV1/x+eef88MPP9CsWTOGDh3KgAEDWLdunc/b8K6aVhwDERERUTb917/+lXPPPZd58+axZcsWhg4detztpqSkcMkllxAeHs7o0aOtj8SYSqgqu/YfOab28FN2HrvzCsrKhYc0oWtcJKd1imZMcgd6OLWHTrERhAY3zBERdkZwSW5uLtHR0TRr1ox169axcOFC8vPz+eabb9i8eXNZU1VMTAznn38+L774Is8++yzgaaqKjo4mISGBtWvX0qtXL+bNm0dUVFSVn9WuneexJTNmzCibf/755/Pqq69y7rnnljVVxcTE0LZtW9q2bcuUKVP4/PPP/X4sjKnPikuUbXsPHV17yM5jU1YeB44UlZVrHh5M91aRDOvdih6tosqal9q1bEqTJo1rnJIlDpcMHz6cV155hT59+tCrVy/OOOMM4uPjmTp1KqNGjaKkpIRWrVoxf/58/vKXv3DLLbdwyimnEBQUxEMPPcSoUaN4/PHHufjii4mPjycpKamso7yiP//5z4wbN44pU6Zw0UUXlc2/4YYb2LBhA/379yckJISJEydy6623AnD11VeTnZ1Nnz596uR4GOO2I0XFbN59kI278o66zHXT7oMUFJWUlWsVFUb3VpGMHNSOHq0i6eYkiPjIsIAZyBqwNzlcu3atnRSP49Zbb2XgwIFMmFB3j3S334mpCwfyC/kp+yAbdx046jLXn/ceorR/WgQ6RDdz+h3Kk0O3+EhaNA2cG3HaTQ6Nz0477TQiIiJ4+umn3Q7FmBOiquw5WOCpPXglh4ysPH7ZX94fGBIkdImLILFtcy4d0K5sDETX+AjCQ4Jc3IP6zRKHOcbixYvdDsEYn5SUKDtyD7Mx6+jkkJGdR86howfIdWsVyVndY+tkgFxjZ4nDGFPvVRwgV94HcfQAuZiIULrHR/K7fuUD5Lq3iqRNi7odINfYWeIwxtQbhwuK+Sk775gEsWX3waMGyLVtEU73hCgGD66fA+QaO0scxpg65z1ArjQ5lA6QK71ex3uA3G8TE8qSQ7f4yHo/QK6xs6NvjPEL7wFyGVkHypJDRtbRA+TCgpvQLT6SQR2juTKpQ1mC6NyAB8g1dpY4jDEnxXuAnHdy+Ok4A+TKmpfio2gX3ZSgRjZArrGzxNFAeN8J1xg3lA6QO6qJqZoBct5XMMVHBc4AucbOEoepEXu+R+NXOkDu6ARxoMoBcuf0jKd7fPkguUAaIFcvqEJJERQdgeKC8ldRARQfgdjuEBxWqx9pZwCA/9wLv6ys3W227gcXPl7l4nvvvZcOHTpwyy23ADB58mSCg4NZsGAB+/bto7CwkClTpjBixIhqPyovL48RI0ZUut6sWbN46qmnEBH69+/P66+/zq5du7jpppvYtGkTAC+//DJt27bl4osvZtWqVQA89dRT5OXlMXny5LIbMH733XeMHTuWnj17MmXKFAoKCoiNjeXNN98kISGh0ueG5ObmsmLFirL7bL322musWbOGZ5555qQOrzk5pQPkKtYejjtA7tS2R42gDrgBciXFzgnZ6wR91HTpSfuI13Rl5U5kHafMUet4leE4dwC5NR3ietTqofBr4hCR4cBzQBAwTVUfr7C8EzAdiAf2Ateoaqaz7AngIqAJMB+4Q1VVRL4C2gCl9yf+rapm+XM//GHMmDHceeedZYljzpw5fPrpp9x+++00b96c3bt3c8YZZ3DppZdWW70PDw9n3rx5x6y3Zs0apkyZwvfff09cXFzZ8z1uv/12hgwZwrx58yguLiYvL6/aZ3wUFBRQetuWffv2sXDhQkSEadOm8cQTT/D0009X+tyQkJAQHn30UZ588klCQkJITU3l1VdfPdnDZ3xUOkDO+86tpaOpvQfINXOeIHdWt9iy5NDDjQFyqlWcaAsrnFwrTBcXHmedKtavdJ3jnNC1uPr4ayIoFILCICjEUyMICvW8jpoOhdAIT7ng0MrLlJYLCvOaDi1fJzKhduPGj4lDRIKAF4HzgUwgTUQ+VNU1XsWeAmap6kwRGQY8BlwrImcBZwP9nXLfAUOAr5z3V6vq0TefOhnHqRn4y8CBA8nKymLHjh1kZ2cTHR1N69at+eMf/8g333xDkyZN2L59O7t27aJ169bH3Zaqcv/99x+z3pdffsno0aOJi4sDyp+38eWXX5Y9YyMoKIgWLVpUmzjGjBlTNp2ZmcmYMWPYuXMnBQUFZc8Pqeq5IcOGDeOjjz6iT58+FBYW0q9fvxoeLVMdzwC5Q17J4UDVA+TiIriobzw9Y0PpHhtCt5hQEpqBFBc5J8r9ULwH8o5Arvd/u6UnWu/pSk7OxZWt42MSKCk8zl6egCbBFU7O3tMhzsk1DMKioFlcFeW8T9YhFU7iFaaPmwS81g8K8bT3NVD+rHEMBjJUdROAiMwGRgDeiSMRuMuZXgC870wrEA6EAgKEALv8GKsrRo8ezdy5c/nll18YM2YMb775JtnZ2SxevJiQkBA6d+58zHM2KnOi63kLDg6mpKS8g/N4z/e47bbbuOuuu7j00kv56quvmDx58nG3fcMNN/C3v/2N3r17k5KSUqO4Gi1VyN0GOdu8TpzV/Rd8hKLCI+TmHeTAwUMcPHSQw4cOk38kn6KCfIK1kBgpojWFnB9UQkRQMc2aFhPWrIgQigjSQpoUF8CuAthVmzc3lfKT7TH/FZeeKEMhpCmEt6zBybnCibbG64RCE7uc1x/8mTjaAdu83mcCp1cosxwYhac5ayQQJSKxqvqDiCwAduJJHC+o6lqv9VJFpBh4F5iildziV0QmAZMAOnbsWEu7VLvGjBnDxIkT2b17N19//TVz5syhVatWhISEsGDBArZu3erTdnJzcytdb9iwYYwcOZK77rqL2NjYsudtnHfeebz88svceeedZU1VCQkJZGVlsWfPHiIjI/noo48YPnx4lZ9X+nyPmTNnls2v6rkhp59+Otu2bWPJkiWsWLHiZA5Zw5SfC7vWQNZqz89dqyFrDRyp2eOKCwjmiIYAwYQTTBNCiGoSgoSEERQVTmhYGGHhLWga3ozgkJo2bVRs5qjsv+Uq1g+yrtJA4/Zv/G7gBREZD3wDbAeKRaQ70Ado75SbLyK/VtVv8TRTbReRKDyJ41rgmGebqupUYCp4bqvu9z05AX379uXAgQO0a9eONm3acPXVV3PJJZfQr18/kpKS6N27t0/bqWq9vn378sADDzBkyBCCgoIYOHAgM2bM4LnnnmPSpEn84x//ICgoiJdffpkzzzyTBx98kMGDB9OuXbvjfvbkyZMZPXo00dHRDBs2jM2bNwNU+dwQgCuvvJJly5b59NjbBqu4EPZkeBJDaXLYtdpTsygV1gIS+kL/KyGhL/vC2rHtgPJzbjGbcwrYtK+QjD1F7DpYQgHBFBKMBIfRIa4F3ROi6B4fSY+E0ifINSMsOMA6qE294LfncYjImcBkVb3AeX8fgKo+VkX5SGCdqrYXkT8B4ar6iLPsQSBfVZ+osM54IElVbz1eLPY8DvddfPHF/PGPf+S8886rskyD+Z2owoFfnBqEVy1i93rnChc8betxPaFVoidRlL6atwMRVJWHPlzNrB/Ka5VRzgA57+RgA+SMm9x4Hkca0ENEuuCpSVwF/L5CUHHAXlUtAe7Dc4UVwM/ARBF5DE9T1RDgWREJBlqq6m4RCQEuBuzZpvVYTk4OgwcP5tRTTz1u0qi3Cg5C1jrYtaq8BrFrNRzeW14mqq0nKXQfBgmneJJFXE9Pc04VXvrqJ2b9sJWxgztySf82NkDONCh+SxyqWiQitwKf4rkcd7qqrhaRh4F0Vf0QGAo8JiKKp6nqFmf1ucAwYCWejvJPVPVfIhIBfOokjSA8SeM1f+1DfbNy5Uquvfbao+aFhYXx448/uhRR9Vq2bMmGDRvcDqN6JcWwd7NXP4STKPZupuwa+ZAISEiEPpd4EkRCoidJNIup0UfNW5rJk5+u57IBbfnbyFMsWZgGJ6AfHdu7d2/7o60nVJV169bVTVPVwd1e/RClP9dBkTM0SJpATDdPYiitQST0hZadTvoqne8zdjMudRFJnWKYef1gu4mfqdfs0bEVhIeHs2fPHmJjYy15uExV2bNnD+Hh4bW74cJ8T79DaZIo7bDO87qyOyLekxSSrnf6IRIhvrfn0tFatu6X/dz4+mK6xEXwyrWnWdIwDVbAJo727duTmZlJdna226EYPIm8ffv21ResjCrkbPW61NVpbtqTUT7aNzjckxC6/8aTIEprEZGtam8njmNn7mFSUtNoFhbEjJTBdj8n06AFbOIICQkpG/FsGpDDOUd3Uu9aDVlroeBAeZnoztCqLySOKG9uiukKTdy5dPVAfiEpqWkcyC9izo1n0rZl7ddmjKlLAZs4TD1XXAi7Nx7dD7FrDezPLC8T3tJTaxgw1qlBnAKtentuH1FPFBaXcPObS8jIymP6+GQS2zZ3OyRjTpolDuMuVTiw89h+iOz15fctahIC8b2g01lHj4mIalOv7/ejqtz77kq+3bibJ6/ozzk9490OyZhaYYnD1J0jeZ5mpaMGzq2C/JzyMs3be5qXepzvaW5K6Os8T6DqMRH11TOfb+TdJZnc+ZsejE7q4HY4xtQaSxym9pUUw95NnqTg3WG9b0t5mdBIT/NS35HlNYhWfaBp47glyTtpP/P8Fxu5Mqk9d5xXu89CMMZtljjMycnLPnZUdfY6KHLuritNPDWGtgNhwDXll7y26Nho71z61fos7p+3inN6xvPoyH52ubdpdCxxGN8UHvYkhNImptLmpoNelzNHJnhqEck3lF/yGt/LL2Mi6qtV23O55c0l9EqI4qWrBxFSlw9BMqaOWOIwRyspccZElN7d1Wlu2vsTqPO8juCmnquXel5Q3g+R0Bci4tyN3WWZ+w6RMiONls1CSU1JJjLM/rxM42Tf7EB2aK+THLzuzZS1FgrynALiGROR0BdOGeXUIvpCTBfXxkTUV7mHChmfmkZ+YTFv3nA6Cc1reRS8MfWIJY5AUFQAuzccXYPYtRoO7Cgv0zTaMw5iwNXlNYj43hAW6V7cDcSRomImvZ7Oz3sOMfP6wfRMqD/jSIzxB0scjYkq7N9+7JiI3RugpMhTJigU4npBl3Ocu7uWjoloXa/HRNRXJSXK3f9cwY+b9/LcVQM4s1us2yEZ43eWOBqq/P3HjonIWu15TGmpFh08SaHn8PJaRGx3zzOZTa34+6fr+NfyHdwzvDcjBrRzOxxj6oQljvquuMjTMX3U40hXQc7P5WVCo5x+iMvL+yFa9YGmLd2LOwDM+mELr369iWvO6MhNQ7q6HY4xdcYSR32hCnlZXmMinASRvR6Kj3jKSBDE9YB2STBoXHktokUHa2aqY/PX7GLyh6s5r3crJl/S18ZqmIBiicMNBYcge22F24CvhkN7ystEtvb0QXQdUt4PEdcTQuxqHbct25bDbW8voV+7Fvzf7wcSbGM1TICxxOFPJSWwb/OxtwHfu4nyx5E28zQr9fqd1+NI+0KEdbLWR1v3HGTCjDTio8KYNi6ZZqH2J2QCj33ra8uhveWXuno/jrTwoFNAPM+ESEiE/leWP0goukujvfVGY7P3YAHjU9MoVmVGymDio8LcDskYV1jiqKmiI55+h4pjIvJ+KS/TNMaTFAZd5zxIyBkTERrhXtzmpOQXFnPDzDS25xzmrRtOp1u8jW8xgcsSx/HkZsIvq46+id/ujeWPIw0K9dyLqdu55TWIhL6eezZZZ2mjUVyi3Dl7GUu35fDS7weR1DnG7ZCMcZUljuP54BbY9JVnumVHT99D74vL+yFiu0OQHcLGbsq/1/DJ6l/468WJXNivjdvhGOM6v571RGQ48BwQBExT1ccrLO8ETAfigb3ANaqa6Sx7ArgIaALMB+5QVRWR04AZQFPg49L5ftmBofd7Xq36QLg98jMQTft2E6n/3cL1Z3dhwq/sGfXGgOek7BciEgS8CFwIJAJjRSSxQrGngFmq2h94GHjMWfcs4GygP3AKkAwMcdZ5GZgI9HBew/21D3Q83fOypBGQPl65k0c/XsuFp7TmLxf1cTscY+oNf17OMxjIUNVNqloAzAZGVCiTCHzpTC/wWq5AOBAKhAEhwC4RaQM0V9WFTi1jFnCZH/fBBKi0LXu5851lDOoYzTNjBtCkifVZGVPKn4mjHbDN632mM8/bcmCUMz0SiBKRWFX9AU8i2em8PlXVtc76mdVsEwARmSQi6SKSnp2dXVkRYyr1U3YeE2el065lU6Zdl0R4iN1C3hhvbg8guBsYIiJL8TRFbQeKRaQ70AdojycxDBORX9dkw6o6VVWTVDUpPj6+tuM2jVT2gSOMT11EkAgzUpKJjgh1OyRj6h1/do5vBzp4vW/vzCujqjtwahwiEglcrqo5IjIRWKiqec6y/wBnAq8726lym8acqEMFRUyYmUb2gSPMnnQmnWJt3I0xlfFnjSMN6CEiXUQkFLgK+NC7gIjEiUhpDPfhucIK4Gc8NZFgEQnBUxtZq6o7gf0icoZ47ip3HfCBH/fBBIii4hJue2spq7bn8sLYQQzoYHcWNqYqfkscqloE3Ap8CqwF5qjqahF5WEQudYoNBdaLyAYgAXjUmT8X+AlYiacfZLmq/stZdjMwDchwyvzHX/tgAoOq8tCHq/liXRb/b8Qp/CYxwe2QjKnXxF9DIOqTpKQkTU9PdzsMU0+99FUGT3yynpuGdOPeC3u7HY4x9YaILFbVpIrz3e4cN8ZVHyzbzhOfrOfSU9vy5wt6uR2OMQ2CJQ4TsL7/aTd3/3M5Z3SN4cnR/W2shjE+ssRhAtL6Xw5w4+uL6RwbwavXJhEWbGM1jPGVJQ4TcHbtzycldRFNQ4KYcf1gWjQNcTskYxoUSxwmoBzIL2R8ahq5hwtJTUmmXcumbodkTINj9wQ3AaOwuISb31zChl0HmD4+mb5tW7gdkjENktU4TEBQVe57byXfbtzNYyP7MaSn3YbGmBNlicMEhGc/38jcxZncfl4PrkzuUP0KxpgqWeIwjd6ctG0898VGrjitPX/8TQ+3wzGmwbPEYRq1rzdkc9+8lfy6RxyPjeqH2LPgjTlpljhMo7V6Ry43v7GYnglRvHT1IEKC7OtuTG2wvyTTKG3POUxKahrNm4aQOj6ZqHAbq2FMbbHLcU2jk3u4kJTURRwuLGbuTWfRukW42yEZ06hYjcM0KkeKirnx9XQ27z7Iq9eeRq/WUW6HZEyjYzUO02iUlCh/nruChZv28uyYAZzVLc7tkIxplKzGYRqNJz9bzwfLdvCnC3px2cB2bodjTKNlicM0Cm8s3MrLX/3E70/vyM1Du7kdjjGNmiUO0+B9vmYXD36wimG9W/HwpX1trIYxfmaJwzRoy7flcNvbS+nbtgX/N3YgwTZWwxi/s78y02D9vOcQE2amERsZyj/GJxERZtd6GFMXfEocIvKeiFwkIpZoTL2w72AB41MXUViszEgZTKsoG6thTF3xNRG8BPwe2Cgij4tILz/GZMxx5RcWc8OsdDJzDjNtXBLdW0W6HZIxAcWnxKGqn6vq1cAgYAvwuYh8LyIpImL3cjB1pqREuWvOMpb8vI9nrhxAcucYt0MyJuD43PQkIrHAeOAGYCnwHJ5EMv846wwXkfUikiEi91ayvJOIfCEiK0TkKxFp78w/V0SWeb3yReQyZ9kMEdnstWxAjfbYNGiPfryWj1f+wgO/68NF/du4HY4xAcmn3kQRmQf0Al4HLlHVnc6id0QkvYp1goAXgfOBTCBNRD5U1TVexZ4CZqnqTBEZBjwGXKuqC4ABznZigAzgM6/1/qSqc33dSdM4TP9uM//4bjPjz+rMhF91cTscYwKWr5ehPO+czI+hqklVrDMYyFDVTQAiMhsYAXgnjkTgLmd6AfB+Jdu5AviPqh7yMVbTCP1n5U4e+fcaLuibwF8vTrSxGsa4yNemqkQRaVn6RkSiReTmatZpB2zzep/pzPO2HBjlTI8EopwmMW9XAW9XmPeo07z1jIiEVfbhIjJJRNJFJD07O7uaUE19tnjrXu58ZxkDO7TkuasGEtTEkoYxbvI1cUxU1ZzSN6q6D5hYC59/NzBERJYCQ4DtQHHpQhFpA/QDPvVa5z6gN5AMxAD3VLZhVZ2qqkmqmhQfH18LoRo3bMrO44aZ6bRt2ZRp45IJDwlyOyRjAp6vTVVBIiKqqlDWfxFazTrbgQ5e79s788qo6g6cGoeIRAKXeyco4EpgnqoWeq1T2r9yRERS8SQf0wjtzjvC+NQ0mogwIyWZmIjqvnLGmLrga43jEzwd4eeJyHl4mo4+qWadNKCHiHQRkVA8TU4fehcQkTivQYX3AdMrbGMsFZqpnFoI4mnkvgxY5eM+mAbkUEERE2akkXUgn2njkugUG+F2SMYYh681jnuAG4E/OO/nA9OOt4KqFonIrXiamYKA6aq6WkQeBtJV9UNgKPCYiCjwDXBL6foi0hlPjeXrCpt+U0TiAQGWATf5uA+mgSgqLuH2t5eycnsur1xzGgM7RrsdkjHGizitT41aUlKSpqdXetWwqWdUlb9+sIo3Fv7MwyP6ct2Znd0OyZiAJSKLK7ty1tdxHD3wjLFIBMpuCqSqXWstQmOAV77exBsLf+bGc7pa0jCmnvK1jyMVeBkoAs4FZgFv+CsoE5g+WLadv3+yjktObcs9w3u7HY4xpgq+Jo6mqvoFnqatrao6GbjIf2GZQLNw0x7+9M8VDO4Sw1Oj+9PExmoYU2/52jl+xLn6aaPT4b0dsFuSmlqxcdcBJs1Kp2NsM167NomwYBurYUx95muN4w6gGXA7cBpwDTDOX0GZwLFrfz7jU9MICwliRkoyLZrZzZaNqe+qrXE4g/3GqOrdQB6Q4veoTEDIO1JESmoa+w4VMOfGM2kf3cztkIwxPqi2xqGqxcCv6iAWE0AKi0u4+c0lrN91gJeuHsQp7Vq4HZIxxke+9nEsFZEPgX8CB0tnqup7fonKNGqqygPzVvLNhmz+foxmHBsAABfNSURBVHk/hvZq5XZIxpga8DVxhAN7gGFe8xSwxGFq7PkvMpiTnsntw7ozJrmj2+EYY2rIp8ShqtavYWrFP9O38cznGxg1qB1/PL+n2+EYY06AryPHU/HUMI6iqtfXekSm0fpmQzb3vbeSX3WP4/FR/e1hTMY0UL42VX3kNR2O56FLO2o/HNNYrdmxn5vfXEL3VpG8dM0gQoN9fty9Maae8bWp6l3v9yLyNvCdXyIyjc6OnMOkzFhEZFgwqSnJNA+3sRrGNGQn+m9fD8AuhTHVyj1cyPjURRw6UsyM65Np06Kp2yEZY06Sr30cBzi6j+MXqnhkqzGlCopKuOn1xWzefZAZKYPp3bq52yEZY2qBr01VUf4OxDQuqso9767gh017+N8rT+Xs7nFuh2SMqSU+NVWJyEgRaeH1vqWIXOa/sExD99Rn65m3dDt3/7Ynowa1dzscY0wt8rWP4yFVzS19o6o5wEP+Cck0dG/9+DMvLviJsYM7cMu53d0OxxhTy3xNHJWV8/VSXhNAvly3i7+8v5Jze8XzyIhTbKyGMY2Qr4kjXUT+V0S6Oa//BRb7MzDT8KzIzOGWN5eS2LY5L/x+EMFBNlbDmMbI17/s24AC4B1gNpAP3OKvoEzDs23vIa6fkUZMRCjTxycTEWYVUmMaK1+vqjoI3OvnWEwDte9gAeNSF1FYrMyelEyrqHC3QzLG+JGvV1XNF5GWXu+jReRTH9YbLiLrRSRDRI5JPCLSSUS+EJEVIvKViLR35p8rIsu8XvmlV3GJSBcR+dHZ5jsiEur77prall9YzMRZ6WTuPcxr1yXRvZVduW1MY+drU1WccyUVAKq6j2pGjjtPDnwRuBBIBMaKSGKFYk8Bs1S1P/Aw8Jiz/QWqOkBVB+C5lfsh4DNnnb8Dz6hqd2AfMMHHfTC1rKRE+Z85y0nfuo+nrzyVwV1i3A7JGFMHfE0cJSJS9uAEEelMJXfLrWAwkKGqm1S1AE/fyIgKZRKBL53pBZUsB7gC+I+qHhLPJTrDgLnOspmAjSdxyd8+Xsu/V+7k/t/15pJT27odjjGmjviaOB4AvhOR10XkDeBr4L5q1mkHbPN6n+nM87YcGOVMjwSiRCS2QpmrgLed6VggR1WLjrNNAERkkoiki0h6dnZ2NaGamkr972amfbeZcWd2YuKvu7odjjGmDvmUOFT1EyAJWI/nJP4/wOFa+Py7gSEishQYAmwHiksXikgboB9QbX9KJTFPVdUkVU2Kj4+vhVBNqU9W/cLDH63ht4kJPHhJXxurYUyA8fUmhzcAdwDtgWXAGcAPHP0o2Yq2Ax283rd35pVR1R04NQ4RiQQu9+5LAa4E5qlqofN+D9BSRIKdWscx2zT+tXjrPu6YvZQBHVry3FUDCWpiScOYQONrU9UdQDKwVVXPBQYCOcdfhTSgh3MVVCieJqcPvQuISJyIlMZwHzC9wjbGUt5Mhaoqnr6QK5xZ44APfNwHc5I27z7IDTPTaNMinGnXJdE0NMjtkIwxLvA1ceSraj6AiISp6jqg1/FWcGoEt+JpZloLzFHV1SLysIhc6hQbCqwXkQ1AAvBo6fpOB3wHPP0p3u4B7hKRDDx9Hv/wcR/MSdidd4TxqYsQEWakDCY2MsztkIwxLvF1eG+mM47jfWC+iOwDtla3kqp+DHxcYd6DXtNzKb9CquK6W6ik41tVN+G5YsvUkcMFxUyYmc6u/fm8NfEMOsdFuB2SMcZFvo4cH+lMThaRBUAL4BO/RWXqjeIS5fbZS1mRmcMr15zGoI7RbodkjHFZjW8opKoVm45MI6Wq/L9/rWb+ml1MviSRC/q2djskY0w9YLcvNVWa+s0mZv2wlYm/7sL4s7u4HY4xpp6wxGEq9eHyHTz2n3Vc1L8N913Yx+1wjDH1iCUOc4yFm/Zw95zlDO4cw9OjT6WJjdUwxnixxGGOsnHXASbNSqdDTFOmXnca4SE2VsMYczRLHKZM1v58xqemERocxIyUwbRsZnesN8YcyxKHASDvSBEpM9LYd6iA1PHJdIhp5nZIxph6yp7vaSgqLuGWN5ew7pcDTLsuiX7tW7gdkjGmHrMaR4BTVf7y/iq+3pDNlMtO4dzex30+lzHGWOIIdC98mcHstG3cem53xg7uWP0KxpiAZ4kjgL27OJOn529g1MB2/M9ve7odjjGmgbDEEaC+27ibe95dwVndYnn88v72MCZjjM8scQSgtTv3c9Mbi+kWH8kr155GaLB9DYwxvrMzRoDZmXuYlNQ0IsOCSU1Jpnl4iNshGWMaGLscN4Dszy9k/PQ08o4U8c+bzqRty6Zuh2SMaYCsxhEgCopK+MMbi/kpO4+XrxlEnzbN3Q7JGNNAWY0jAKgq9767gv9m7OGp0afy6x7xbodkjGnArMYRAJ7+bAPvLd3OXef35IrT2rsdjjGmgbPE0ci99ePPvLAgg6uSO3DbsO5uh2OMaQQscTRiC9Zl8dcPVjGkZzyPXHaKjdUwxtQKSxyN1MrMXG55awm9W0fx4tWDCAmyX7Uxpnb49WwiIsNFZL2IZIjIvZUs7yQiX4jIChH5SkTaey3rKCKfichaEVkjIp2d+TNEZLOILHNeA/y5Dw3Rtr2HSJmRRnSzUFLHJxMZZtdAGGNqj98Sh4gEAS8CFwKJwFgRSaxQ7Clglqr2Bx4GHvNaNgt4UlX7AIOBLK9lf1LVAc5rmb/2oSHKOVTA+NRFFBQVMyMlmVbNw90OyRjTyPizxjEYyFDVTapaAMwGRlQokwh86UwvKF3uJJhgVZ0PoKp5qnrIj7E2CvmFxUyatZhtew8z9bokeiREuR2SMaYR8mfiaAds83qf6czzthwY5UyPBKJEJBboCeSIyHsislREnnRqMKUedZq3nhGRMH/tQENSUqL8zz+Xs2jLXp668lTO6BrrdkjGmEbK7R7Tu4EhIrIUGAJsB4rxDEz8tbM8GegKjHfWuQ/o7cyPAe6pbMMiMklE0kUkPTs725/7UC88/sk6/r1iJ/de2JtLT23rdjjGmEbMn4ljO9DB6317Z14ZVd2hqqNUdSDwgDMvB0/tZJnTzFUEvA8McpbvVI8jQCqeJrFjqOpUVU1S1aT4+MY9Unrm91uY+s0mrj2jEzee09XtcIwxjZw/E0ca0ENEuohIKHAV8KF3ARGJE5HSGO4Dpnut21JESs/4w4A1zjptnJ8CXAas8uM+1Hufrv6Fyf9azW/6JDD50r42VsMY43d+SxxOTeFW4FNgLTBHVVeLyMMicqlTbCiwXkQ2AAnAo866xXiaqb4QkZWAAK8567zpzFsJxAFT/LUP9d2Sn/dx+9tL6d++Jf83diBBTSxpGGP8T1TV7Rj8LikpSdPT090Oo1Zt2X2QUS9/T2RYMO/dfBZxkXaNgDGmdonIYlVNqjjf7c5xcwL25B1hXOoiVJWZ1w+2pGGMqVM2pLiBOVxQzISZ6fySm89bE8+gS1yE2yEZYwKMJY4GpLhEuWP2UpZn5vDy1adxWqdot0MyxgQga6pqIFSVRz5aw2drdvHgxYkMP6W12yEZYwKUJY4GYtq3m5nx/RYm/KoLKWd3cTscY0wAs8TRAHy0YgePfryW3/VrzQO/6+N2OMaYAGeJo55btHkvd72znKRO0fzvlQNoYmM1jDEus8RRj2VkHWDirHTaxzTlteuSCA8Jqn4lY4zxM0sc9VTWgXzGTU8jJEiYmTKY6IhQt0MyxhjALsetlw4eKeL6GWnsPVjAOzeeQYeYZm6HZIwxZazGUc8UFZdw61tLWLNjPy/8fiD927d0OyRjjDmK1TjqEVXlrx+sYsH6bB4deQrn9UlwOyRjjDmG1TjqkRcXZPD2om3cPLQbV5/eye1wjDGmUpY46on3lmTy1GcbuGxAW/50QS+3wzHGmCpZ4qgH/puxmz/PXcGZXWN54opT7WFMxph6zRKHy9b9sp+bXl9M1/gIXrn2NEKD7VdijKnf7Czlop25h0lJTaNZWBCpKYNp0TTE7ZCMMaZaljhcciC/kJTUNPYfLmT6+GTatWzqdkjGGOMTuxzXBQVFJfzhjSVkZOUxfXwyfdu2cDskY4zxmSWOOqaq3PveCr7L2M2TV/TnnJ7xbodkjDE1Yk1VdeyZ+Rt4b8l27vxND0YndXA7HGOMqTFLHHVo9qKfef7LDK5Mas8d5/VwOxxjjDkhljjqyIL1WTzw/irO6RnPoyP72VgNY0yD5dfEISLDRWS9iGSIyL2VLO8kIl+IyAoR+UpE2nst6ygin4nIWhFZIyKdnfldRORHZ5vviEi9v9/4qu253PLmEnolRPHS1YMICbJ8bYxpuPx2BhORIOBF4EIgERgrIokVij0FzFLV/sDDwGNey2YBT6pqH2AwkOXM/zvwjKp2B/YBE/y1D7Vh295DpMxII7pZKKkpyUSG2fUIxpiGzZ//+g4GMlR1k6oWALOBERXKJAJfOtMLSpc7CSZYVecDqGqeqh4ST/vOMGCus85M4DI/7sNJyT1USMqMNPILi0lNSSahebjbIRljzEnzZ+JoB2zzep/pzPO2HBjlTI8EokQkFugJ5IjIeyKyVESedGowsUCOqhYdZ5sAiMgkEUkXkfTs7Oxa2iXfHSkqZuLr6Wzdc5Cp1ybRMyGqzmMwxhh/cLux/W5giIgsBYYA24FiPONLfu0sTwa6AuNrsmFVnaqqSaqaFB9ft2MlSkqUu/+5gkWb9/LU6FM5s1tsnX6+Mcb4kz8Tx3bAe6BCe2deGVXdoaqjVHUg8IAzLwdPTWKZ08xVBLwPDAL2AC1FJLiqbdYHf/90Hf9avoM/D+/FiAGVVoiMMabB8mfiSAN6OFdBhQJXAR96FxCROBEpjeE+YLrXui1FpLSqMAxYo6qKpy/kCmf+OOADP+5Djc36YQuvfr2Ja87oyB+GdHM7HGOMqXV+SxxOTeFW4FNgLTBHVVeLyMMicqlTbCiwXkQ2AAnAo866xXiaqb4QkZWAAK8569wD3CUiGXj6PP7hr32oqflrdjH5w9Wc17sVky/pa2M1jDGNknj+iW/ckpKSND093a+fsfTnfYx9bSG9EqJ4e9IZNAu1y26NMQ2biCxW1aSK893uHG8Utu45yA0z04mPCmPauGRLGsaYRs0Sx0nae7CA8alpFKsyI2Uw8VFhbodkjDF+Zf8an4T8wmJumJnG9pzDvHXD6XSLj3Q7JGOM8TurcZyg4hLljtlLWboth+fGDCCpc4zbIRljTJ2wxHECVJVHPlrDp6t38ZeLErmwXxu3QzLGmDpjieME/OO7zcz4fgvXn92FCb/q4nY4xhhTpyxx1NC/V+xkyr/XMrxvax64qI/b4RhjTJ2zxFEDaVv28sc5yzitUzTPXjWAoCY2wM8YE3gscfjop+w8Js5Kp13Lprx2XRLhIUFuh2SMMa6wxOGD7ANHGJ+6iCARZqQkExNR7x86aIwxfmPjOKpxqKCICTPTyD5whNmTzqRTbITbIRljjKusxnEcRcUl3PrWUlZtz+WFsYMY0KGl2yEZY4zrrMZRBVXlwQ9X8+W6LB657BR+k5jgdkjGGFMvWI2jCiJCt/hIbh7ajWvP6OR2OMYYU29YjeM4bHCfMcYcy2ocxhhjasQShzHGmBqxxGGMMaZGLHEYY4ypEUscxhhjasQShzHGmBqxxGGMMaZGLHEYY4ypEVFVt2PwOxHJBrae4OpxwO5aDKe2WFw1Y3HVjMVVM401rk6qGl9xZkAkjpMhIumqmuR2HBVZXDVjcdWMxVUzgRaXNVUZY4ypEUscxhhjasQSR/Wmuh1AFSyumrG4asbiqpmAisv6OIwxxtSI1TiMMcbUSEAnDhEZLiLrRSRDRO6tZHmYiLzjLP9RRDp7LbvPmb9eRC6o47juEpE1IrJCRL4QkU5ey4pFZJnz+rCO4xovItlen3+D17JxIrLReY2r47ie8Yppg4jkeC3zy/ESkekikiUiq6pYLiLyvBPzChEZ5LXMn8equriuduJZKSLfi8ipXsu2OPOXiUh6Hcc1VERyvX5XD3otO+7v389x/ckrplXO9ynGWebP49VBRBY454HVInJHJWX89x1T1YB8AUHAT0BXIBRYDiRWKHMz8IozfRXwjjOd6JQPA7o42wmqw7jOBZo5038ojct5n+fi8RoPvFDJujHAJudntDMdXVdxVSh/GzC9Do7XOcAgYFUVy38H/AcQ4AzgR38fKx/jOqv084ALS+Ny3m8B4lw6XkOBj07291/bcVUoewnwZR0drzbAIGc6CthQyd+j375jgVzjGAxkqOomVS0AZgMjKpQZAcx0pucC54mIOPNnq+oRVd0MZDjbq5O4VHWBqh5y3i4E2tfSZ59UXMdxATBfVfeq6j5gPjDcpbjGAm/X0mdXSVW/AfYep8gIYJZ6LARaikgb/Husqo1LVb93Phfq7rvly/Gqysl8L2s7rjr5bgGo6k5VXeJMHwDWAu0qFPPbdyyQE0c7YJvX+0yOPfBlZVS1CMgFYn1c159xeZuA57+KUuEiki4iC0XkslqKqSZxXe5Ui+eKSIcaruvPuHCa9LoAX3rN9tfxqk5VcfvzWNVUxe+WAp+JyGIRmeRCPGeKyHIR+Y+I9HXm1YvjJSLN8Jx83/WaXSfHSzxN6AOBHyss8tt3zJ453oCJyDVAEjDEa3YnVd0uIl2BL0Vkpar+VEch/Qt4W1WPiMiNeGprw+ros31xFTBXVYu95rl5vOotETkXT+L4ldfsXznHqhUwX0TWOf+R14UleH5XeSLyO+B9oEcdfbYvLgH+q6retRO/Hy8RicSTrO5U1f21ue3jCeQax3agg9f79s68SsuISDDQAtjj47r+jAsR+Q3wAHCpqh4pna+q252fm4Cv8PwnUidxqeoer1imAaf5uq4/4/JyFRWaEvx4vKpTVdz+PFY+EZH+eH5/I1R1T+l8r2OVBcyj9ppnq6Wq+1U1z5n+GAgRkTjqwfFyHO+75ZfjJSIheJLGm6r6XiVF/Pcd80fHTUN44altbcLTdFHaqda3QplbOLpzfI4z3ZejO8c3UXud477ENRBPh2CPCvOjgTBnOg7YSC11FPoYVxuv6ZHAQi3vjNvsxBftTMfUVVxOud54OiulLo6Xs83OVN3ZexFHd1wu8vex8jGujnj67M6qMD8CiPKa/h4YXodxtS793eE5Af/sHDuffv/+istZ3gJPP0hEXR0vZ99nAc8ep4zfvmO1dnAb4gvPVQcb8JyEH3DmPYznv3iAcOCfzh/SIqCr17oPOOutBy6s47g+B3YBy5zXh878s4CVzh/PSmBCHcf1GLDa+fwFQG+vda93jmMGkFKXcTnvJwOPV1jPb8cLz3+fO4FCPG3IE4CbgJuc5QK86MS8Ekiqo2NVXVzTgH1e3610Z35X5zgtd37HD9RxXLd6fbcW4pXYKvv911VcTpnxeC6W8V7P38frV3j6UFZ4/a5+V1ffMRs5bowxpkYCuY/DGGPMCbDEYYwxpkYscRhjjKkRSxzGGGNqxBKHMcaYGrHEYcwJqnBn3WW1eWdWEelc1R1ZjXGb3XLEmBN3WFUHuB2EMXXNahzG1DLnOQxPOM9iWCQi3Z35nUXkSyl/jkpHZ36CiMxzbuC3XETOcjYVJCKvOc9b+ExEmjrlb5fy57HMdmk3TQCzxGHMiWtaoalqjNeyXFXtB7wAPOvM+z9gpqr2B94EnnfmPw98raqn4nn2w2pnfg/gRVXtC+QAlzvz7wUGOtu5yV87Z0xVbOS4MSdIRPJUNbKS+VuAYaq6ybkR3S+qGisiu/Hcz6vQmb9TVeNEJBtor143q3RulT1fVXs47+8BQlR1ioh8AuThuUPs++rc/M+YumI1DmP8Q6uYrokjXtPFlPdJXoTnHkSDgDTnzs3G1BlLHMb4xxivnz8409/jucsywNXAt870F3geAYyIBIlIi6o2KiJNgA6qugC4B8+dWY+p9RjjT/afijEnrqmILPN6/4mqll6SGy0iK/DUGsY6824DUkXkT0A2kOLMvwOYKiIT8NQs/oDnjqyVCQLecJKLAM+rak6t7ZExPrA+DmNqmdPHkaSqu92OxRh/sKYqY4wxNWI1DmOMMTViNQ5jjDE1YonDGGNMjVjiMMYYUyOWOIwxxtSIJQ5jjDE1YonDGGNMjfx/gVTURlcfaWMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3yV9fn/8deVQRYzgxlCwhIZgpDgqlj9tYq2Sh3IcIBQUVy1tlZb7bdU7ddWq7ZWFFEUxIEUR7Faqatft02AyBQIO8wkhJGEkHX9/jg3cJKchAPkzp2cXM/H4zxyn3ucc+XOSd65P5/7vj+iqhhjjDE1hXldgDHGmKbJAsIYY0xAFhDGGGMCsoAwxhgTkAWEMcaYgCK8LqChJCYmampqqtdlGGNMs7J48eJ8VU0KtCxkAiI1NZWsrCyvyzDGmGZFRDbXtcyamIwxxgRkAWGMMSYgCwhjjDEBhUwfhDGmZSovLyc3N5fS0lKvS2nSoqOjSU5OJjIyMuhtLCCMMc1abm4ubdq0ITU1FRHxupwmSVUpKCggNzeXtLS0oLezJiZjTLNWWlpKQkKChUM9RISEhITjPsqygDDGNHsWDsd2IvuoxQdEeWUVD7+3mm17D3pdijHGNCktPiC2FR7k1W+2MHl2JgdKy70uxxjTDLVu3drrElzR4gMiNTGOp68dyrrdRdz+2lIqKqu8LskYY5qEFh8QAOf2SeLBUQP5z5o8HvznKq/LMcY0U6rK3XffzcCBAxk0aBCvv/46ADt27GDEiBEMGTKEgQMH8tlnn1FZWcnEiROPrPvEE094XH1tdpqrY/wZKWzML+K5zzaSlhjHxHOCPxXMGNM0/P6dlazavr9BX7N/17b87tIBQa375ptvkp2dzbfffkt+fj4ZGRmMGDGCV199lYsuuoj77ruPyspKSkpKyM7OZtu2baxYsQKAvXv3NmjdDcGOIPzce/Gp/LB/Jx745yo+/m6X1+UYY5qZzz//nHHjxhEeHk6nTp0477zzyMzMJCMjgxdffJFp06axfPly2rRpQ8+ePdmwYQO3334777//Pm3btvW6/FrsCMJPeJjw17FDGD3jK25/dSl/v/ls+ndtej80Y0xgwf6n39hGjBjBp59+yrvvvsvEiRO56667uP766/n2229ZtGgRM2bMYP78+bzwwgtel1qNHUHUENsqglkTMmgTHcnkOZns3m+X7xtjgnPuuefy+uuvU1lZSV5eHp9++inDhw9n8+bNdOrUiRtvvJGf/vSnLFmyhPz8fKqqqrjyyit56KGHWLJkidfl12JHEAF0bhfN8xPSufrZr5g8J4vXbzqT2Fa2q4wx9bv88sv56quvGDx4MCLCI488QufOnZkzZw6PPvookZGRtG7dmpdeeolt27Zxww03UFXlO3Py4Ycf9rj62kRVva6hQaSnp2tDDxj04apd3Dg3iwv7d+KZa4YRFmZXaxrT1KxevZpTTz3V6zKahUD7SkQWq2p6oPWtiakeP+jfift/1J9FK3fxp0XfeV2OMcY0Kms3OYZJ56SyMb+IZ/9vA2kJcYwdnuJ1ScYY0yhcPYIQkZEiskZEckTk3gDLR4jIEhGpEJGraix7RERWishqEXlSPLobl4gw7dIBnNsnkfvfXsEXOflelGGMMY3OtYAQkXBgOnAx0B8YJyL9a6y2BZgIvFpj27OBc4DTgIFABnCeW7UeS0R4GNOvGUrPpDhufnkxObsPeFWKMcY0GjePIIYDOaq6QVXLgHnAKP8VVHWTqi4Dat4ASYFooBUQBUQCnl651jY6klkTMoiKCGPS7CwKig55WY4xxrjOzYDoBmz1e57rzDsmVf0K+ATY4TwWqerqmuuJyBQRyRKRrLy8vAYouX7d42OZeX06u/aXctPcxZSWV7r+nsYY45UmeRaTiPQGTgWS8YXKBSJybs31VHWmqqaranpSUlKj1DY0pQOPXT2YrM2F3PPGMkLlNGFjjKnJzYDYBnT3e57szAvG5cDXqlqkqkXAv4CzGri+E/bj07py90Wn8I/s7fz1o3Vel2OMaUbqGzti06ZNDBw4sBGrqZ+bAZEJ9BGRNBFpBYwFFga57RbgPBGJEJFIfB3UtZqYvHTL93tx5dBk/vLhOt5eGmzuGWNM8+HadRCqWiEitwGLgHDgBVVdKSIPAFmqulBEMoC3gA7ApSLye1UdACwALgCW4+uwfl9V33Gr1hMhIjx8xSByC0v41YJlJHeIIT013uuyjGnZ/nUv7FzesK/ZeRBc/Mc6F9977710796dW2+9FYBp06YRERHBJ598QmFhIeXl5Tz00EOMGjWqztcIpLS0lKlTp5KVlUVERASPP/44559/PitXruSGG26grKyMqqoq3njjDbp27crVV19Nbm4ulZWV/Pa3v2XMmDEn9W2DyxfKqep7wHs15v2P33QmvqanmttVAje5WVtDaBURxoxrh3HFM18yZe5i3rrlbHokxHldljGmEY0ZM4Y777zzSEDMnz+fRYsWcccdd9C2bVvy8/M588wzueyyyziey7mmT5+OiLB8+XK+++47LrzwQtauXcuMGTP42c9+xjXXXENZWRmVlZW89957dO3alXfffReAffv2Ncj3ZldSn6QOca14YWIGlz/9BZNmZ/Lm1HNoFxvpdVnGtEz1/KfvltNPP53du3ezfft28vLy6NChA507d+bnP/85n376KWFhYWzbto1du3bRuXPnoF/3888/5/bbbwegX79+9OjRg7Vr13LWWWfxhz/8gdzcXK644gr69OnDoEGD+MUvfsE999zDj3/8Y849t9Y5PSekSZ7F1NykJcYx49phbNlTwtRXFlNu41ob06KMHj2aBQsW8PrrrzNmzBheeeUV8vLyWLx4MdnZ2XTq1InS0oYZOmD8+PEsXLiQmJgYLrnkEj7++GP69u3LkiVLGDRoEPfffz8PPPBAg7yXBUQDObNnAg9fcRpfri/g/rdW2OmvxrQgY8aMYd68eSxYsIDRo0ezb98+OnbsSGRkJJ988gmbN28+7tc899xzeeWVVwBYu3YtW7Zs4ZRTTmHDhg307NmTO+64g1GjRrFs2TK2b99ObGws1157LXfffXeDjS1hTUwN6KphyWzKL+apT3LomRTHTef18rokY0wjGDBgAAcOHKBbt2506dKFa665hksvvZRBgwaRnp5Ov379jvs1b7nlFqZOncqgQYOIiIhg9uzZREVFMX/+fObOnUtkZCSdO3fmN7/5DZmZmdx9992EhYURGRnJM8880yDfl40H0cCqqpTb5y3lveU7eOaaoYwc2MXrkowJaTYeRPBsPAiPhYUJj40ezODk9tz5ejbLcvd6XZIxxpwQCwgXREeG89z16STERTF5Thbb9x70uiRjTBOyfPlyhgwZUu1xxhlneF1WLdYH4ZKkNlG8eEMGVz79JZNmZ7Jg6tm0jrLdbYwbVPW4rjHw2qBBg8jOzm7U9zyR7gQ7gnBR305teOqaoazbXcTtry6hwk5/NabBRUdHU1BQYGcO1kNVKSgoIDo6+ri2s39pXXZe3ySmXTaA3769gofeXc20ywZ4XZIxISU5OZnc3Fwa45b/zVl0dDTJybVuXFEvC4hGcN2ZPdiUX8yszzeSlhjHhLNTvS7JmJARGRlJWlqa12WEJAuIRvKbS05lc0Exv39nJSnxsZzfr6PXJRljTL2sD6KRhIcJfx17Ov06t+W2V5ewesd+r0syxph6WUA0orioCGZNTKd1dASTZ2ey+0DD3JvFGGPcYAHRyLq0i2HWhAwKS8q5cU4WB8tsXGtjTNNkAeGBgd3a8dexQ1i2bR93zc+mqspOzzPGND0WEB65cEBn7rvkVP61YieP/nuN1+UYY0wtrgaEiIwUkTUikiMi9wZYPkJElohIhYhcVWNZioj8W0RWi8gqEUl1s1YvTP5eGuPPSOGZ/6xnfuZWr8sxxphqXAsIEQkHpgMXA/2BcSLSv8ZqW4CJwKsBXuIl4FFVPRUYDux2q1aviAi/v2wA5/ZJ5DdvLefL9flel2SMMUe4eQQxHMhR1Q2qWgbMA6qN2q2qm1R1GVDtHhROkESo6gfOekWqWuJirZ6JDA/jqfFDSUuM4+a5i1mfV+R1ScYYA7gbEN0A/3aTXGdeMPoCe0XkTRFZKiKPOkck1YjIFBHJEpGs5nyZfbuYSF6YmEFkeBiTZmeyp7jM65KMMabJdlJHAOcCvwQygJ74mqKqUdWZqpququlJSUmNW2ED6x4fy8zr09mxr5Sb5mZxqMJOfzXGeMvNgNgGdPd7nuzMC0YukO00T1UAbwNDG7i+JmdYjw48NnowmZsKufeN5XZ3SmOMp9wMiEygj4ikiUgrYCyw8Di2bS8ihw8LLgBWuVBjk3Pp4K784od9eWvpNv72cY7X5RhjWjDXAsL5z/82YBGwGpivqitF5AERuQxARDJEJBcYDTwrIiudbSvxNS99JCLLAQGec6vWpua2C3pzxendePyDtfwjO9iDLmOMaVgSKs0Y6enpmpWV5XUZDeZQRSXXPf9fsnP38tqNZzCsR7zXJRljQpCILFbV9EDLmmondYsXFRHOs9cNo2u7aKa8tJgtBSF5lq8xpgmzgGjCOsS1YtbEDCqqlElzMtl3sNzrkowxLYgFRBPXK6k1M64dxqb8Ym59ZQnlNq61MaaRWEA0A2f1SuB/rxjE5zn5/M8/Vtjpr8aYRmFDjjYTV6d3Z1N+MU//Zz09E1tz44ieXpdkjAlxFhDNyC8vPIVNBcX8779Wk5IQy0UDOntdkjEmhFkTUzMSFiY8fvUQTktuz53zslmeu8/rkowxIcwCopmJjgznueuHER/XislzMtmx76DXJRljQpQFRDPUsU00syamU1JWyaTZWRQfqvC6JGNMCLKAaKb6dW7LU+NPZ83O/dzx2lIqbVxrY0wDs4Boxr5/Skd+f9kAPvpuN394d7XX5RhjQoydxdTMXXdWKhvyi3nhi42kJcZy3VmpXpdkjAkRFhAh4P4f9WdzQQnT3llF9/hYvn9KR69LMsaEAGtiCgHhYcKT406nb6c23PbqUtbsPOB1ScaYEGABESJaR0Uwa0I6sa3CmTQ7k90HSr0uyRjTzFlAhJCu7WOYNSGDPcVl3PjSYkrLbVxrY8yJs4AIMYOS2/GXsUNYlruXX8z/lio7/dUYc4JcDQgRGSkia0QkR0TuDbB8hIgsEZEKEbkqwPK2IpIrIk+5WWeouWhAZ359cT/eXb6Dxz5Y43U5xphmyrWAEJFwYDpwMdAfGCci/WustgWYCLxax8s8CHzqVo2h7MZzezJueHemf7Kev2dt9bocY0wz5OYRxHAgR1U3qGoZMA8Y5b+Cqm5S1WVArVFwRGQY0An4t4s1hiwR4YFRAzmndwK/eWs5X60v8LokY0wz42ZAdAP8/3XNdeYdk4iEAY8BvzzGelNEJEtEsvLy8k640FAVGR7G09cMIyU+lptfXsyGvCKvSzLGNCNNtZP6FuA9Vc2tbyVVnamq6aqanpSU1EilNS/tYiJ5ceJwwsOESbMzKSwu87okY0wz4WZAbAO6+z1PduYF4yzgNhHZBPwZuF5E/tiw5bUcKQmxPHf9MLbvK+WmuYs5VGGnvxpjjs3NgMgE+ohImoi0AsYCC4PZUFWvUdUUVU3F18z0kqrWOgvKBG9Yj3geveo0/rtpD79+c7mNa22MOSbXAkJVK4DbgEXAamC+qq4UkQdE5DIAEckQkVxgNPCsiKx0qx4Do4Z04+c/6MubS7Yx/ZMcr8sxxjRxEir/Saanp2tWVpbXZTR5qspd87/lraXb+Nu407l0cFevSzLGeEhEFqtqeqBlTbWT2rhERPjjlYPISO3AL/7+LYs3F3pdkjGmibKAaIGiIsJ59rp0OreNZspLWWzdU+J1ScaYJsgCooWKj2vFCxMzKK+sYtLsTPaXlntdkjGmibGAaMF6d2zNjGuHsTG/mFtfWUJ5Za0L2o0xLZgFRAt3du9E/nD5QD5bl8/vFq6001+NMUfYkKOGMRkpbMwvYcb/radnYhw/Pben1yUZY5oACwgDwK8uOoVN+cX84b3V9EiI44f9O3ldkjHGY9bEZAAICxOeGDOEQd3accdrS1mxbZ/XJRljPGYBYY6IaRXO89en0yE2kslzMtm5z8a1NqYls4Aw1XRsG82siRkUlVYweU4mxYcqvC7JGOMRCwhTy6ld2vLU+KGs3rGfn83LptLGtTamRbKAMAGd368jv7t0AB+u3sXD7632uhxjjAfsLCZTpwlnp7Ixv5jnP99IamIc157Zw+uSjDGNyALC1Ov+H53K5oJifrdwJSnxsYzoayP3GdNSWBOTqVdEeBh/Gz+UPh1bc+srS1i764DXJRljGokFhDmm1lERzJqYQXSrcG54MZO8A4e8LskY0wgsIExQurWP4fnr0ykoPsSUuVmUltu41saEOlcDQkRGisgaEckRkVpjSovICBFZIiIVInKV3/whIvKViKwUkWUiMsbNOk1wBndvz1/GDGHplr388u/fUmWnvxoT0lwLCBEJB6YDFwP9gXEi0r/GaluAicCrNeaXANer6gBgJPAXEWnvVq0meCMHduHei/vxz2U7eOLDtV6XY4xxkZtnMQ0HclR1A4CIzANGAasOr6Cqm5xl1QYiUNW1ftPbRWQ3kATsdbFeE6SbRvRkY14xf/s4h9SEOK4clux1ScYYF7jZxNQN2Or3PNeZd1xEZDjQClgfYNkUEckSkay8vLwTLtQcHxHhwZ8M5OxeCdz75jK+2VDgdUnGGBcEFRAi8jMRaSs+s5x+gwvdLk5EugBzgRtUtdZwZ6o6U1XTVTU9KcnOz29MrSLCeOaaYXSPj+WmlxezMb/Y65KMMQ0s2COISaq6H7gQ6ABcB/zxGNtsA7r7PU925gVFRNoC7wL3qerXwW5nGk+72EhenJiBAJNmZ7K3pMzrkowxDSjYgBDn6yXAXFVd6TevLplAHxFJE5FWwFhgYVBv5lv/LeAlVV0QZI3GAz0S4ph5fTrbCg9y09zFlFXYuNbGhIpgA2KxiPwbX0AsEpE2QL1/CVS1ArgNWASsBuar6koReUBELgMQkQwRyQVGA8+KyEpn86uBEcBEEcl2HkOO+7szjSIjNZ5HrjqNbzbu4ddvLrdxrY0JERLML7OIhAFDgA2quldE4oFkVV3mdoHBSk9P16ysLK/LaNGe+GAtf/1oHXdfdAq3nt/b63KMMUEQkcWqmh5oWbBHEGcBa5xwuBa4H7AxKU01d/6gD5cN7sqji9bw7rIdXpdjjDlJwQbEM0CJiAwGfoHvlNOXXKvKNEsiwiNXncawHh24a342S7cUel2SMeYkBBsQFeprixoFPKWq04E27pVlmqvoyHBmXjeMjm2juPGlLLbuKfG6JGPMCQo2IA6IyK/xnd76rtMnEeleWaY5S2gdxYsTMzhUUcXkOZnsLy33uiRjzAkINiDGAIfwXQ+xE981DY+6VpVp9np3bMOMa4exIa+Y215dSkWlnf5qTHMTVEA4ofAK0E5EfgyUqqr1QZh6ndM7kQd/MpBP1+Yx7Z2VdvqrMc1MsLfauBr4L77rFa4GvvG/PbcxdRk3PIWbRvTk5a+38OIXm7wuxxhzHIK9m+t9QIaq7gYQkSTgQ8CucjbHdM/IfmzML+bBd1eREh/LD/p38rokY0wQgu2DCDscDo6C49jWtHBhYcJfxg5hYNd23DFvKSu32yU0xjQHwf6Rf19EFonIRBGZiO8meu+5V5YJNbGtInh+QjrtYiKZPDuLXftLvS7JGHMMwXZS3w3MBE5zHjNV9R43CzOhp1PbaGZNyOBAaTmT52RSUlbhdUnGmHoE3Uykqm+o6l3O4y03izKhq3/Xtvxt/Oms2r6fn83LptLGtTamyao3IETkgIjsD/A4ICL7G6tIE1ou6NeJ3/64Px+s2sWf3v/O63KMMXWo9ywmVbXbaRhXTDw7lY35xcz8dAOpCXGMPyPF65KMMTUEe5qrMQ1KRPifH/dnc0EJv/3HClLiY/len0SvyzLG+LFTVY1nIsLDeGr86fROas3UVxazbtcBr0syxvixgDCeahMdyayJ6URFhDNpTib5RYe8LskY43A1IERkpIisEZEcEbk3wPIRIrJERCpq3rpDRCaIyDrnMcHNOo23kjvE8vyEdHbvP8SUl7IoLa/0uiRjDC4GhIiEA9OBi4H+wDgR6V9jtS3ARODVGtvGA78DzgCGA78TkQ5u1Wq8N6R7e54YM4QlW/Zy94JldmM/Y5oAN48ghgM5qrpBVcuAefgGHDpCVTc541rXvBf0RcAHqrpHVQuBD4CRLtZqmoBLBnXhVyNP4Z1vt/PEh+u8LseYFs/NgOgGbPV7nuvMa7BtRWSKiGSJSFZeXt4JF2qajqnn9WL0sGSe/Ggdby3N9bocY1q0Zt1JraozVTVdVdOTkpK8Lsc0ABHhD5cP4sye8dyzYDn/3bjH65KMabHcDIhtQHe/58nOPLe3Nc1cq4gwZlw7jOQOMdw0N4tN+cVel2RMi+RmQGQCfUQkTURaAWOBhUFuuwi4UEQ6OJ3TFzrzTAvRPrYVL0zMQIFJszPZV2LjWhvT2FwLCFWtAG7D94d9NTBfVVeKyAMichmAiGSISC6+keqeFZGVzrZ7gAfxhUwm8IAzz7QgqYlxzLwuna2FJdz88mLKKmxca2Mak4TK6YTp6emalZXldRnGBW8uyeWu+d9ydXoyf7ryNETE65KMCRkislhV0wMts3sxmSbviqHJbMov5smPc0hLbM3U7/fyuiRjWgQLCNMs/PyHfdlYUMKf3v+OHgmxXDKoi9clGRPymvVprqblEBEeveo0hqa05+evZ5O9da/XJRkT8iwgTLMRHRnOzOvTSWoTxU/nZLFt70GvSzImpFlAmGYlsXUUL07M4FB5JZNnZ3Kg1E5/NcYtFhCm2enTqQ1PXzuUdbuLuP21pVRU2umvxrjBAsI0S+f2SeLBUQP5z5o8HvznKq/LMSYk2VlMptkaf0YKG/OLeO6zjaQlxjHxnDSvSzImpFhAmGbt3otPZVNBCQ/8cxUpCbFc0K+T1yUZEzKsick0a+Fhwl/HDuHULm25/dWlrNq+3+uSjAkZFhCm2YttFcGsCRm0iY5k8pxMdu8v9bokY0KCBYQJCZ3bRfP8hHT2HSxn8pwsSsoqvC7JmGbPAsKEjIHd2vHk2NNZsX0fP389m6qq0LgRpTFesYAwIeUH/Ttx/4/6s2jlLv606DuvyzGmWbOzmEzImXROKhvzi3j2/zaQlhDH2OEpXpdkTLNkAWFCjogw7dIBbC4o4f63V9A9PpZzeid6XZYxzY6rTUwiMlJE1ohIjojcG2B5lIi87iz/RkRSnfmRIjJHRJaLyGoR+bWbdZrQExEexvRrhtIzKY6bX15Mzu4DXpdkTLPjWkCISDgwHbgY6A+ME5H+NVabDBSqam/gCeBPzvzRQJSqDgKGATcdDg9jgtU2OpJZEzKIighj0uwsCooOeV2SMc2Km0cQw4EcVd2gqmXAPGBUjXVGAXOc6QXA/xPfeJIKxIlIBBADlAF2BZQ5bt3jY5l5fTq79pdy09zFlJZXel2SMc2GmwHRDdjq9zzXmRdwHVWtAPYBCfjCohjYAWwB/qyqe1ys1YSwoSkdeOzqwWRtLuSeN5YRKuOwG+O2ptpJPRyoBLoCHYDPRORDVd3gv5KITAGmAKSk2Jkqpm4/Pq0rmwtKeHTRGtIS47jzB329LsmYJs/NI4htQHe/58nOvIDrOM1J7YACYDzwvqqWq+pu4AsgveYbqOpMVU1X1fSkpCQXvgUTSm75fi+uHJrMXz5cx9tLa34UjTE1uRkQmUAfEUkTkVbAWGBhjXUWAhOc6auAj9V3/L8FuABAROKAMwG76smcFBHh4SsGcUZaPL9asIysTdZqaUx9XAsIp0/hNmARsBqYr6orReQBEbnMWW0WkCAiOcBdwOFTYacDrUVkJb6geVFVl7lVq2k5WkWEMePaYXTrEMOUuYvZXFDsdUnGNFkSKh126enpmpWV5XUZppnYmF/M5U9/QUJcK96ceg7tYiO9LskYT4jIYlWt1YQPdi8m00KlJcYx49phbNlTwtRXFlNu41obU4sFhGmxzuyZwMNXnMaX6wu4/60VdvqrMTU01dNcjWkUVw1LZlN+MU99kkPPpDhuOq+X1yUZ02RYQJgW764f9mVjQTF/fP87eiTEMnJgF69LMqZJsCYm0+KFhQmPjR7M4OT23Pl6Nsty93pdkjFNggWEMUB0ZDjPXZ9OQlwUk+dksX3vQa9LMsZzFhDGOJLaRPHiDRmUllUyaXYmRYdsXGvTsllAGOOnb6c2PHXNUNbtLuL2V5dQYae/mhbMAsKYGs7rm8S0ywbwyZo8Hnp3tdflGOMZO4sJQBVEvK7CNCHXndmDTfnFzPp8I2mJcUw4O9XrkoxpdBYQh4rgz30gvick9PY9Evs4070gpoPXFRqP/OaSU9lcUMzv31lJSnws5/fr6HVJxjQquxdTyR749M9QkAMF66BwM6jfqGOxiU5o9D4aIAl9ID4NIqIa7hswTVLxoQpGz/iKzQXFLJh6Nqd2aet1ScY0qPruxWQBUVNFGRRucgLDCY2C9ZC/Dop3H11PwqBdd7+jDb9H224QZt07oWLHvoP8ZPoXhIvw9m3n0LFNtNclGdNgLCAaSuk+X1gcCY8cX3AUrIdyv9tGR8T4mqf8QyOxjzVZNWMrtu1j9Iyv6NupNfOmnEVMq3CvSzKmQVhAuE0VDuzwC42cepqsEnxNVDWbreJ7WpNVE/fvlTu56eXFjBzQmenjhxIWZic2mObPAsJLFWWwd7NzpJFT/VG06+h6h5usqnWSW5NVU/P8Zxt46N3VTP1+L+4Z2c/rcow5afUFhJ3F5LaIVr4/+Il9ai870mS13unrcIJjydd1NFn1Onr0cfgIxJqsGtXk76WxIb+YZ/6znrSEOK7O6H7sjYxpplwNCBEZCfwVCAeeV9U/1lgeBbwEDAMKgDGquslZdhrwLNAWqAIyVLXUzXobXXQ76DbU9/CnCgd2+oWG00m+cwWs/mfdTVYJvY4efXRIg0jrTG1oIsLvLxvA1oQjhpEAABFcSURBVD0l/Oat5STHx3B2r0SvyzLGFa41MYlIOLAW+CGQi29s6XGquspvnVuA01T1ZhEZC1yuqmNEJAJYAlynqt+KSAKwV9X/L2N1TbaJqaEdbrI60kFeR5MVAu1Tal/XkdDHmqwawL6D5Vz1zJfs2l/KW7eeQ6+k1l6XZMwJ8aQPQkTOAqap6kXO818DqOrDfussctb5ygmFnUAScDEwXlWvDfb9WkxA1Kd0/9EjjiOn6DrPy4qOrletyap39SOQ2Hjv6m9mtu4p4SfTv6B1dARv3XIO8XGtvC7JmOPmVR9EN2Cr3/Nc4Iy61lHVChHZByQAfQF1AiQJmKeqj7hYa2iIbnuMJqsa13XU2WR1ODR6HT0CsSarWrrHxzLz+nTGPfc1N83N4uWfnkFUhJ3+akJHU+2kjgC+B2QAJcBHTsp95L+SiEwBpgCkpKQ0epHNhgi07eJ7pJ1bfVllefULAw9f15HzAWS/7P8i0L673ym6fgHSNrnFNlkN69GBx0YP5vbXlnLvG8t5/OrBiN3Xy4QINwNiG+B/ikeyMy/QOrlOE1M7fJ3VucCnqpoPICLvAUOBagGhqjOBmeBrYnLhewh94ZH1nGW1H/asr35dR0EObP2mRpNVNMT3qn07khbSZHXp4K5syi/msQ/WkpYYxx3/L8C+NKYZcjMgMoE+IpKGLwjGAuNrrLMQmAB8BVwFfKyqh5uWfiUisUAZcB7whIu1mkCi20LX030Pf9WarPwegZqsYuID344kvmdINVnddkFvNuYX8/gHa+mREMuoId28LsmYk+ZaQDh9CrcBi/Cd5vqCqq4UkQeALFVdCMwC5opIDrAHX4igqoUi8ji+kFHgPVV9161azXE6ZpPV5qNHG0earD6C7Ff8X6R6k5X/leXNsMlKRHj4ykHkFh7k7gXLSO4Qw7AeoX/0ZEKbXUltGk+tJiu/TvNATVb+13UcfjTxJqvC4jIuf/oLDpRW8NYt55CSEOt1ScbUy261YZo2Vd81HIFuR1K4Car8xoau1mTld2V5E2qyWp9XxBVPf0lSmyjemHo27WIivS7JmDpZQJjm60iTlV8n+eEjkKKdfisebrKqcV1HYh9Pmqy+Wl/AdbO+4cyeCbx4QwaR4c2rycy0HBYQJjQdOlD9ViT+Rx51NVnVvBmii01W87O28qsFyxg3vDv/e/kgO/3VNEl2sz4TmqLa1H2WVaAmq92rYM17tZusal7XkdCnQZqsrk7vzqb8Yp7+z3rW5xXTKymO5A6xJHeIcR6xJLWOstuGmybLAsKEHhFo09n3qPMsqxq3IqnzLKsa13Uk9Pbdlj3IJqtfXngKAF+sL+CDVbvILyqrtrxVeBhd20fXCo7DXzu2sQAx3rEmJmMO82+yqnkzxFpNVj0DjBh47CarkrIKtu89yNbCg+QWHiS3sITcwoNsc57nFx2qtn7NAOnWPobk+KMh0rFNNOEWIOYkWB+EMSfjcJNVtdBwxvAIdJZVzes6Enr75odHQlg4hEU60xG+534OllWybe/R4PAPkUABEhkudG3vHHm0d448nADp1j6GTm0tQEz9LCCMcUtlOezd4hcc644egRzYEcQLiF9YOAESHhlgOgLCI6iSCA5VhVFaFcbBSqGkQiiuEIrK4UAZFFcI5RpOJeGUE06VRBAd1YrY6GjiYqJpHRtNm9gY2sbF0i4uhjZxMYQdfv8jddQ1HQnhEXVMHzv8TNNkndTGuCU88uit02s6dOBoWBzaD5UVvqONqnJfsFRV+k1XHH34P68s961TVQmV5YRVlRNTVUlMZTkdqsogssJ5Xd82WllOZUU5VZXlaGU5UulbHl5UQXhRVSPvHAkyXI4VNHWFVuAQDTzdkNu2nPCzgDDGLVFtoOsQ36ORCPX8UqtSeugQO/YcYPueA+woLGJX4QF27S0ib38xefuK2Ft0kEgqiXAeUWFVdGodRqe4SDq1DqdjXBiJseEkxoaRECO0iwojXCuCC7+A00fD78h0RRlUFTvrHQ2/6tM1XqvuscRc0oDhdzJHa4e3iUuEXhc0+HdpAWFMSyFCdHQ0aV2jSeuaFHCV0vJKduwr9ev38H1dXHiQ3J0l7NpfvQ8kPEzo0i661tlXhzvUu7SLJqIxLhKsqvKFRI0jrrrD5TgC7GTCr9q2Fe6FX7d0CwhjjLuiI8NJS4wjLTEu4PJDFZXs2Ftao/Pc9/XzdfnsOlCKf7dmeJjQuW3NADk63WABEhYGhPn+mw41wYRfuDt/yi0gjDFBi4oIJzUxjtQgA8R3RpZv+sv1+ezcHzhAutW6BiSG7h1i6dwu2m5T4mH4WUAYYxrMsQKkrKKKHfsCXwPy9foCduzfVi1AwgS6tIvxBUj72hcSdmlvAeImCwhjTKNpFRFGj4Q4eiTUHSA7A/SB5BYe5OsNBezcX0pVjQDxNWEdPfLoVq0JK4ZWERYgJ8oCwhjTZLSKCCMlIbbOcTTKK30BsjXAhYTfbNzD29kHqwWIHAmQuvpALEDqYwFhjGk2IsPD6B4fS/f4+gOk5tFHbmEJ/924h38ECJBObaJrBceRAGkfTVREy7jmIRBXA0JERgJ/xTfk6POq+scay6OAl4BhQAEwRlU3+S1PAVYB01T1z27Waoxp/qoHSEKt5YEC5PCtTbI2F/LOsh1U+iWIf4DU7kiPpWuIB4hrASEi4cB04IdALpApIgtVdZXfapOBQlXtLSJjgT8BY/yWPw78y60ajTEty7ECpKKyip37S2s1X+UWlrB4cyH/rBEgAJ3aRlXvA2l/dLpr+xiiI5tvgLh5BDEcyFHVDQAiMg8Yhe+I4LBRwDRnegHwlIiIqqqI/ATYCBS7WKMxxhwRER7m/LEP3IRVUVnFrgOHyN1Tuw9kyZZC3l22g4oaAdKxTVTA5qvmECBuBkQ3YKvf81zgjLrWUdUKEdkHJIhIKXAPvqOPX9b1BiIyBZgCkJKS0nCVG2NMABHhYXRr77tKvOYfMzgaINsCXEiYvXUv7y2vHSBJtQKk+tXoXgZIU+2kngY8oapF9Q3TqKozgZngu5tr45RmjDGB+QfI8LTaY4NUVim79vtdSHj4KGRvCcty9/L+ih2UVwYOkG7tA4eImwHiZkBsA7r7PU925gVaJ1dEIoB2+DqrzwCuEpFHgPZAlYiUqupTLtZrjDGuCg/zjd/RtZ4A2X3ArxN9z8EjHekrtu1j0cqdtQIksXUUZ/VK4G/jTq/1eifLzYDIBPqISBq+IBgLjK+xzkJgAvAVcBXwsfoGqDgyTqSITAOKLByMMaHOd/ND3/UZGamBAyTvwKFazVfxca1cqce1gHD6FG4DFuE7zfUFVV0pIg8AWaq6EJgFzBWRHGAPvhAxxhgTQHiY0LldNJ3bRZOe6v772YhyxhjTgtU3opxdY26MMSYgCwhjjDEBWUAYY4wJyALCGGNMQBYQxhhjArKAMMYYE5AFhDHGmIBC5joIEckDNp/ESyQC+Q1UTkOyuo6P1XV8rK7jE4p19VDVpEALQiYgTpaIZNV1sYiXrK7jY3UdH6vr+LS0uqyJyRhjTEAWEMYYYwKygDhqptcF1MHqOj5W1/Gxuo5Pi6rL+iCMMcYEZEcQxhhjArKAMMYYE1DIB4SIjBSRNSKSIyL3BlgeJSKvO8u/EZFUv2W/duavEZGLGrmuu0RklYgsE5GPRKSH37JKEcl2Hgsbua6JIpLn9/4/9Vs2QUTWOY8JjVzXE341rRWRvX7L3NxfL4jIbhFZUcdyEZEnnbqXichQv2Vu7q9j1XWNU89yEflSRAb7LdvkzM8WkQYdZCWIur4vIvv8fl7/47es3s+Ay3Xd7VfTCuczFe8sc3N/dReRT5y/BStF5GcB1nHvM6aqIfvAN5LdeqAn0Ar4FuhfY51bgBnO9FjgdWe6v7N+FJDmvE54I9Z1PhDrTE89XJfzvMjD/TUReCrAtvHABudrB2e6Q2PVVWP92/GNYOjq/nJeewQwFFhRx/JLgH8BApwJfOP2/gqyrrMPvx9w8eG6nOebgESP9tf3gX+e7Gegoeuqse6l+IZHboz91QUY6ky3AdYG+J107TMW6kcQw4EcVd2gqmXAPGBUjXVGAXOc6QXA/xMRcebPU9VDqroRyHFer1HqUtVPVLXEefo1kNxA731SddXjIuADVd2jqoXAB8BIj+oaB7zWQO9dL1X9FN9wuXUZBbykPl8D7UWkC+7ur2PWpapfOu8Ljff5CmZ/1eVkPpsNXVdjfr52qOoSZ/oAsBroVmM11z5joR4Q3YCtfs9zqb1zj6yjqhXAPiAhyG3drMvfZHz/IRwWLSJZIvK1iPykgWo6nrqudA5lF4hI9+Pc1s26cJri0oCP/Wa7tb+CUVftbu6v41Xz86XAv0VksYhM8aCes0TkWxH5l4gMcOY1if0lIrH4/si+4Te7UfaX+Jq/Twe+qbHItc9YxPEWaRqXiFwLpAPn+c3uoarbRKQn8LGILFfV9Y1U0jvAa6p6SERuwnf0dUEjvXcwxgILVLXSb56X+6tJE5Hz8QXE9/xmf8/ZXx2BD0TkO+c/7MawBN/Pq0hELgHeBvo00nsH41LgC1X1P9pwfX+JSGt8oXSnqu5vyNeuT6gfQWwDuvs9T3bmBVxHRCKAdkBBkNu6WRci8gPgPuAyVT10eL6qbnO+bgD+g++/ikapS1UL/Gp5HhgW7LZu1uVnLDUO/13cX8Goq3Y391dQROQ0fD/DUapacHi+3/7aDbxFwzWtHpOq7lfVImf6PSBSRBJpAvvLUd/ny5X9JSKR+MLhFVV9M8Aq7n3G3OhYaSoPfEdIG/A1ORzu2BpQY51bqd5JPd+ZHkD1TuoNNFwndTB1nY6vU65PjfkdgChnOhFYRwN11gVZVxe/6cuBr/Voh9hGp74OznR8Y9XlrNcPX4ehNMb+8nuPVOrudP0R1TsQ/+v2/gqyrhR8/Wpn15gfB7Txm/4SGNmIdXU+/PPD94d2i7PvgvoMuFWXs7wdvn6KuMbaX873/hLwl3rWce0z1mA7t6k+8PXwr8X3x/Y+Z94D+P4rB4gG/u78svwX6Om37X3OdmuAixu5rg+BXUC281jozD8bWO78giwHJjdyXQ8DK533/wTo57ftJGc/5gA3NGZdzvNpwB9rbOf2/noN2AGU42vjnQzcDNzsLBdgulP3ciC9kfbXsep6Hij0+3xlOfN7OvvqW+fnfF8j13Wb3+fra/wCLNBnoLHqctaZiO/EFf/t3N5f38PXx7HM72d1SWN9xuxWG8YYYwIK9T4IY4wxJ8gCwhhjTEAWEMYYYwKygDDGGBOQBYQxxpiALCCMOYYad4PNbsg7iYpIal13EDXGa3arDWOO7aCqDvG6CGMamx1BGHOCnHEAHnHGAviviPR25qeKyMdydCyPFGd+JxF5y7kR3bcicrbzUuEi8pxzv/9/i0iMs/4dcnRMkHkefZumBbOAMObYYmo0MY3xW7ZPVQcBTwF/ceb9DZijqqcBrwBPOvOfBP5PVQfjG3tgpTO/DzBdVQcAe4Ernfn3Aqc7r3OzW9+cMXWxK6mNOQYRKVLV1gHmbwIuUNUNzg3Vdqpqgojk47tnVbkzf4eqJopIHpCsfjdedG7h/IGq9nGe3wNEqupDIvI+UITvjqZvq3MTO2Maix1BGHNytI7p43HIb7qSo32DP8J3j52hQKZzt2FjGo0FhDEnZ4zf16+c6S/x3RkY4BrgM2f6I3zDxyIi4SLSrq4XFZEwoLuqfgLcg+9OorWOYoxxk/1HYsyxxYhItt/z91X18KmuHURkGb6jgHHOvNuBF0XkbiAPuMGZ/zNgpohMxnekMBXfHUQDCQdedkJEgCdVdW+DfUfGBMH6IIw5QU4fRLqq5ntdizFusCYmY4wxAdkRhDHGmIDsCMIYY0xAFhDGGGMCsoAwxhgTkAWEMcaYgCwgjDHGBPT/AWOR1dsUEivMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_graphs(history, string):\n",
    "    plt.plot(history.history[string])\n",
    "    plt.plot(history.history['val_'+string])\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(string)\n",
    "    plt.legend([string, 'val_'+string])\n",
    "    plt.show()\n",
    "\n",
    "plot_graphs(history, \"accuracy\")\n",
    "plot_graphs(history, \"loss\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150/150 [==============================] - 2s 12ms/step - loss: 0.0503 - accuracy: 0.9851\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.050299596041440964, 0.9851334691047668]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(x_test, np.array(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word           True \t Pred\n",
      "\n",
      "------------------------------\n",
      "Two            O\tO\n",
      "Sudanese       B-gpe\tB-gpe\n",
      "rebel          O\tO\n",
      "factions       O\tO\n",
      "joined         O\tO\n",
      "forces         O\tO\n",
      "in             O\tO\n",
      "February       B-tim\tB-tim\n",
      "to             O\tO\n",
      "form           O\tO\n",
      "the            O\tO\n",
      "Eastern        B-org\tB-org\n",
      "Front          I-org\tI-org\n",
      ".              O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n",
      "Tachileik      O\tO\n"
     ]
    }
   ],
   "source": [
    "i = np.random.randint(0, x_test.shape[0])\n",
    "p = model.predict(np.array([x_test[i]]))\n",
    "p = np.argmax(p, axis=-1)\n",
    "\n",
    "y_true = np.argmax(np.array(y_test), axis=-1)[i]\n",
    "\n",
    "print('{:15}{:5}\\t {}\\n'.format('Word', 'True', 'Pred'))\n",
    "print('-'*30)\n",
    "for w, true, pred in zip(x_test[i], y_true, p[0]):\n",
    "    print('{:15}{}\\t{}'.format(words[w-1], tags[true], tags[pred]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}