Kaggle NER dataset - Annotated Corpus for Named Entity Recognition-crf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Understanding the Data \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libaries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import nltk, pprint\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "\n",
    "# supress warnings\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import sklearn\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# !pip install sklearn_crfsuite\n",
    "import sklearn_crfsuite\n",
    "from sklearn_crfsuite import metrics\n",
    "from sklearn_crfsuite import scorers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 281837: expected 25 fields, saw 34\\n'\n"
     ]
    }
   ],
   "source": [
    "data_features=pd.read_csv('ner.csv',encoding=\"ISO-8859-1\" ,error_bad_lines=False)\n",
    "data=pd.read_csv(\"ner_dataset.csv\",encoding=\"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1048575 entries, 0 to 1048574\n",
      "Data columns (total 4 columns):\n",
      "Sentence #    47959 non-null object\n",
      "Word          1048575 non-null object\n",
      "POS           1048575 non-null object\n",
      "Tag           1048575 non-null object\n",
      "dtypes: object(4)\n",
      "memory usage: 32.0+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1050795 entries, 0 to 1050794\n",
      "Data columns (total 25 columns):\n",
      "Unnamed: 0         1050795 non-null int64\n",
      "lemma              1050795 non-null object\n",
      "next-lemma         1050795 non-null object\n",
      "next-next-lemma    1050795 non-null object\n",
      "next-next-pos      1050795 non-null object\n",
      "next-next-shape    1050795 non-null object\n",
      "next-next-word     1050795 non-null object\n",
      "next-pos           1050795 non-null object\n",
      "next-shape         1050794 non-null object\n",
      "next-word          1050794 non-null object\n",
      "pos                1050794 non-null object\n",
      "prev-iob           1050794 non-null object\n",
      "prev-lemma         1050794 non-null object\n",
      "prev-pos           1050794 non-null object\n",
      "prev-prev-iob      1050794 non-null object\n",
      "prev-prev-lemma    1050794 non-null object\n",
      "prev-prev-pos      1050794 non-null object\n",
      "prev-prev-shape    1050794 non-null object\n",
      "prev-prev-word     1050794 non-null object\n",
      "prev-shape         1050794 non-null object\n",
      "prev-word          1050794 non-null object\n",
      "sentence_idx       1050794 non-null float64\n",
      "shape              1050794 non-null object\n",
      "word               1050794 non-null object\n",
      "tag                1050794 non-null object\n",
      "dtypes: float64(1), int64(1), object(23)\n",
      "memory usage: 200.4+ MB\n"
     ]
    }
   ],
   "source": [
    "data_features.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sentence #</th>\n",
       "      <th>Word</th>\n",
       "      <th>POS</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>NNS</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>of</td>\n",
       "      <td>IN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>NNS</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>have</td>\n",
       "      <td>VBP</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>marched</td>\n",
       "      <td>VBN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Sentence #           Word  POS Tag\n",
       "0  Sentence: 1      Thousands  NNS   O\n",
       "1          NaN             of   IN   O\n",
       "2          NaN  demonstrators  NNS   O\n",
       "3          NaN           have  VBP   O\n",
       "4          NaN        marched  VBN   O"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.columns=['Sentence', 'Word', 'POS', 'Tag']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.Sentence.fillna(method=\"pad\",inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.Sentence=data.Sentence.apply(lambda x: x[10:]).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47959"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.Sentence.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# there are 47959 sentences in total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data():\n",
    "    dataset=[]\n",
    "    X=[]\n",
    "    y=[]\n",
    "    for index in range(1,data.Sentence.max()+1):\n",
    "        sentence=data[data.Sentence == index]\n",
    "        dataset.append(list(zip(sentence.Word.values,sentence.POS.values,sentence.Tag.values)))\n",
    "        X.append(list(zip(sentence.Word.values,sentence.POS.values)))\n",
    "        y.append(list(sentence.Tag.values))\n",
    "    return dataset,X,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset,X,y=preprocess_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([('Thousands', 'NNS'),\n",
       "  ('of', 'IN'),\n",
       "  ('demonstrators', 'NNS'),\n",
       "  ('have', 'VBP'),\n",
       "  ('marched', 'VBN'),\n",
       "  ('through', 'IN'),\n",
       "  ('London', 'NNP'),\n",
       "  ('to', 'TO'),\n",
       "  ('protest', 'VB'),\n",
       "  ('the', 'DT'),\n",
       "  ('war', 'NN'),\n",
       "  ('in', 'IN'),\n",
       "  ('Iraq', 'NNP'),\n",
       "  ('and', 'CC'),\n",
       "  ('demand', 'VB'),\n",
       "  ('the', 'DT'),\n",
       "  ('withdrawal', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('British', 'JJ'),\n",
       "  ('troops', 'NNS'),\n",
       "  ('from', 'IN'),\n",
       "  ('that', 'DT'),\n",
       "  ('country', 'NN'),\n",
       "  ('.', '.')],\n",
       " ['O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-geo',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-geo',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'B-gpe',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O',\n",
       "  'O'])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[0],y[0]\n",
    "# dataset[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(33571, 14388)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X_train),len(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extracts features for the word at index i in a sentence \n",
    "def word_features(sentence):\n",
    "    features=[]\n",
    "    for i in range(0, len(sentence)):\n",
    "        word,pos=sentence[i]\n",
    "        word_pos=i\n",
    "        # first word\n",
    "        if i==0:\n",
    "            prevword = '<START>'\n",
    "            prevpos = '<START>'\n",
    "        else:\n",
    "            prevword = sentence[i-1][0]\n",
    "            prevpos = sentence[i-1][1]\n",
    "\n",
    "        # last word\n",
    "        if i == len(sentence)-1:\n",
    "            nextword = '<END>'\n",
    "            nextpos = '<END>'\n",
    "        else:\n",
    "            nextword = sentence[i+1][0]\n",
    "            nextpos = sentence[i+1][1]\n",
    "\n",
    "\n",
    "\n",
    "        features.append( {'word':word,\n",
    "                'pos': pos,\n",
    "                'word_pos':word_pos,\n",
    "                'prevword': prevword,\n",
    "                'prevpos': prevpos,  \n",
    "                'nextword': nextword, \n",
    "                'nextpos': nextpos\n",
    "\n",
    "               })\n",
    "        \n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'word': 'The',\n",
       "  'pos': 'DT',\n",
       "  'word_pos': 0,\n",
       "  'prevword': '<START>',\n",
       "  'prevpos': '<START>',\n",
       "  'nextword': 'price',\n",
       "  'nextpos': 'NN'},\n",
       " {'word': 'price',\n",
       "  'pos': 'NN',\n",
       "  'word_pos': 1,\n",
       "  'prevword': 'The',\n",
       "  'prevpos': 'DT',\n",
       "  'nextword': 'of',\n",
       "  'nextpos': 'IN'},\n",
       " {'word': 'of',\n",
       "  'pos': 'IN',\n",
       "  'word_pos': 2,\n",
       "  'prevword': 'price',\n",
       "  'prevpos': 'NN',\n",
       "  'nextword': 'oil',\n",
       "  'nextpos': 'NN'},\n",
       " {'word': 'oil',\n",
       "  'pos': 'NN',\n",
       "  'word_pos': 3,\n",
       "  'prevword': 'of',\n",
       "  'prevpos': 'IN',\n",
       "  'nextword': 'has',\n",
       "  'nextpos': 'VBZ'},\n",
       " {'word': 'has',\n",
       "  'pos': 'VBZ',\n",
       "  'word_pos': 4,\n",
       "  'prevword': 'oil',\n",
       "  'prevpos': 'NN',\n",
       "  'nextword': 'nearly',\n",
       "  'nextpos': 'RB'},\n",
       " {'word': 'nearly',\n",
       "  'pos': 'RB',\n",
       "  'word_pos': 5,\n",
       "  'prevword': 'has',\n",
       "  'prevpos': 'VBZ',\n",
       "  'nextword': 'doubled',\n",
       "  'nextpos': 'VBN'},\n",
       " {'word': 'doubled',\n",
       "  'pos': 'VBN',\n",
       "  'word_pos': 6,\n",
       "  'prevword': 'nearly',\n",
       "  'prevpos': 'RB',\n",
       "  'nextword': 'over',\n",
       "  'nextpos': 'IN'},\n",
       " {'word': 'over',\n",
       "  'pos': 'IN',\n",
       "  'word_pos': 7,\n",
       "  'prevword': 'doubled',\n",
       "  'prevpos': 'VBN',\n",
       "  'nextword': 'the',\n",
       "  'nextpos': 'DT'},\n",
       " {'word': 'the',\n",
       "  'pos': 'DT',\n",
       "  'word_pos': 8,\n",
       "  'prevword': 'over',\n",
       "  'prevpos': 'IN',\n",
       "  'nextword': 'past',\n",
       "  'nextpos': 'JJ'},\n",
       " {'word': 'past',\n",
       "  'pos': 'JJ',\n",
       "  'word_pos': 9,\n",
       "  'prevword': 'the',\n",
       "  'prevpos': 'DT',\n",
       "  'nextword': 'year',\n",
       "  'nextpos': 'NN'},\n",
       " {'word': 'year',\n",
       "  'pos': 'NN',\n",
       "  'word_pos': 10,\n",
       "  'prevword': 'past',\n",
       "  'prevpos': 'JJ',\n",
       "  'nextword': '.',\n",
       "  'nextpos': '.'},\n",
       " {'word': '.',\n",
       "  'pos': '.',\n",
       "  'word_pos': 11,\n",
       "  'prevword': 'year',\n",
       "  'prevpos': 'NN',\n",
       "  'nextword': '<END>',\n",
       "  'nextpos': '<END>'}]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_features=[word_features(sen) for sen in X_train]\n",
    "X_test_features=[word_features(sen) for sen in X_test]\n",
    "X_train_features[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CRF(algorithm='lbfgs', all_possible_states=None, all_possible_transitions=True,\n",
       "    averaging=None, c=None, c1=0.01, c2=0.1, calibration_candidates=None,\n",
       "    calibration_eta=None, calibration_max_trials=None, calibration_rate=None,\n",
       "    calibration_samples=None, delta=None, epsilon=None, error_sensitive=None,\n",
       "    gamma=None, keep_tempfiles=None, linesearch=None, max_iterations=100,\n",
       "    max_linesearch=None, min_freq=None, model_filename=None, num_memories=None,\n",
       "    pa_type=None, period=None, trainer_cls=None, variance=None, verbose=False)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  fitting crf with arbitrary hyperparameters\n",
    "crf = sklearn_crfsuite.CRF(\n",
    "    algorithm='lbfgs',\n",
    "    c1=0.01,\n",
    "    c2=0.1,\n",
    "    max_iterations=100,\n",
    "    all_possible_transitions=True\n",
    ")\n",
    "crf.fit(X_train_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['B-tim',\n",
       " 'B-org',\n",
       " 'I-org',\n",
       " 'B-geo',\n",
       " 'B-per',\n",
       " 'I-per',\n",
       " 'B-gpe',\n",
       " 'I-tim',\n",
       " 'I-geo',\n",
       " 'I-gpe',\n",
       " 'B-eve',\n",
       " 'B-art',\n",
       " 'I-art',\n",
       " 'B-nat',\n",
       " 'I-nat',\n",
       " 'I-eve']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove 'O' from the labels\n",
    "#  This is because we'll only measure the metrics for other less frequent classes.\n",
    "labels = list(crf.classes_)\n",
    "labels.remove('O')\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train  : \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "       B-tim      0.968     0.923     0.945     14224\n",
      "       B-org      0.920     0.868     0.893     14247\n",
      "       I-org      0.945     0.943     0.944     11796\n",
      "       B-geo      0.916     0.954     0.934     26156\n",
      "       B-per      0.958     0.936     0.947     11849\n",
      "       I-per      0.941     0.967     0.954     12050\n",
      "       B-gpe      0.983     0.948     0.965     11083\n",
      "       I-tim      0.938     0.897     0.917      4497\n",
      "       I-geo      0.912     0.926     0.919      5178\n",
      "       I-gpe      0.947     0.667     0.783       135\n",
      "       B-eve      0.863     0.752     0.804       218\n",
      "       B-art      0.966     0.722     0.826       277\n",
      "       I-art      0.951     0.836     0.889       207\n",
      "       B-nat      0.886     0.614     0.726       127\n",
      "       I-nat      0.885     0.742     0.807        31\n",
      "       I-eve      0.839     0.747     0.790       174\n",
      "\n",
      "   micro avg      0.940     0.931     0.935    112249\n",
      "   macro avg      0.926     0.840     0.878    112249\n",
      "weighted avg      0.940     0.931     0.935    112249\n",
      "\n",
      "test  : \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "       B-tim      0.930     0.867     0.898      6109\n",
      "       B-org      0.789     0.735     0.761      5896\n",
      "       I-org      0.811     0.804     0.807      4988\n",
      "       B-geo      0.857     0.906     0.881     11488\n",
      "       B-per      0.854     0.820     0.837      5141\n",
      "       I-per      0.851     0.905     0.877      5201\n",
      "       B-gpe      0.972     0.921     0.946      4787\n",
      "       I-tim      0.830     0.741     0.783      2031\n",
      "       I-geo      0.823     0.803     0.813      2236\n",
      "       I-gpe      0.814     0.556     0.660        63\n",
      "       B-eve      0.474     0.300     0.367        90\n",
      "       B-art      0.452     0.112     0.179       125\n",
      "       I-art      0.133     0.022     0.038        90\n",
      "       B-nat      0.679     0.257     0.373        74\n",
      "       I-nat      0.429     0.150     0.222        20\n",
      "       I-eve      0.333     0.228     0.271        79\n",
      "\n",
      "   micro avg      0.859     0.843     0.851     48418\n",
      "   macro avg      0.689     0.570     0.607     48418\n",
      "weighted avg      0.856     0.843     0.848     48418\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# make predictions \n",
    "y_pred = crf.predict(X_train_features)\n",
    "print(\"train  : \")\n",
    "# print(metrics.flat_f1_score(y_train, y_pred,\n",
    "#                       average='weighted', labels=labels))\n",
    "print(metrics.flat_classification_report(\n",
    "    y_train, y_pred, labels=labels, digits=3\n",
    "))\n",
    "print(\"test  : \")\n",
    "y_pred = crf.predict(X_test_features)\n",
    "# print(metrics.flat_f1_score(y_test, y_pred,\n",
    "#                       average='weighted', labels=labels))\n",
    "print(metrics.flat_classification_report(\n",
    "    y_test, y_pred, labels=labels, digits=3\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyperparameter tuning\n",
    "\n",
    "# define fixed parameters and parameters to search\n",
    "crf = sklearn_crfsuite.CRF(\n",
    "    algorithm='lbfgs',\n",
    "    max_iterations=100,\n",
    "    all_possible_transitions=True\n",
    ")\n",
    "\n",
    "# parameters to tune\n",
    "params_space = {\n",
    "    'c1': [0.01, 0.1, 1],\n",
    "    'c2': [0.01, 0.1, 1]\n",
    "}\n",
    "\n",
    "# use the same metric for evaluation\n",
    "f1_scorer = scorers.make_scorer(metrics.flat_f1_score,\n",
    "                        average='weighted', labels=labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 9 candidates, totalling 27 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  27 out of  27 | elapsed: 35.4min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score=nan,\n",
       "             estimator=CRF(algorithm='lbfgs', all_possible_states=None,\n",
       "                           all_possible_transitions=True, averaging=None,\n",
       "                           c=None, c1=None, c2=None,\n",
       "                           calibration_candidates=None, calibration_eta=None,\n",
       "                           calibration_max_trials=None, calibration_rate=None,\n",
       "                           calibration_samples=None, delta=None, epsilon=None,\n",
       "                           error_sensitive=None, gamma=None,\n",
       "                           keep_tempfi...\n",
       "                           trainer_cls=None, variance=None, verbose=False),\n",
       "             iid='deprecated', n_jobs=-1,\n",
       "             param_grid={'c1': [0.01, 0.1, 1], 'c2': [0.01, 0.1, 1]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "             scoring=make_scorer(flat_f1_score, average=weighted, labels=['B-tim', 'B-org', 'I-org', 'B-geo', 'B-per', 'I-per', 'B-gpe', 'I-tim', 'I-geo', 'I-gpe', 'B-eve', 'B-art', 'I-art', 'B-nat', 'I-nat', 'I-eve']),\n",
       "             verbose=1)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# instantiate a GridSearchCV object\n",
    "rs = GridSearchCV(crf, \n",
    "                  params_space,\n",
    "                  cv=3,\n",
    "                  verbose=1,\n",
    "                  n_jobs=-1,\n",
    "                  scoring=f1_scorer, \n",
    "                  return_train_score=True)\n",
    "# fit\n",
    "rs.fit(X_train_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_c1</th>\n",
       "      <th>param_c2</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_train_score</th>\n",
       "      <th>split1_train_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>205.126506</td>\n",
       "      <td>11.023804</td>\n",
       "      <td>5.492002</td>\n",
       "      <td>0.064191</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>{'c1': 0.01, 'c2': 0.01}</td>\n",
       "      <td>0.832791</td>\n",
       "      <td>0.835960</td>\n",
       "      <td>0.830906</td>\n",
       "      <td>0.833219</td>\n",
       "      <td>0.002085</td>\n",
       "      <td>5</td>\n",
       "      <td>0.950367</td>\n",
       "      <td>0.951869</td>\n",
       "      <td>0.952889</td>\n",
       "      <td>0.951708</td>\n",
       "      <td>0.001036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>214.131017</td>\n",
       "      <td>6.854912</td>\n",
       "      <td>6.234002</td>\n",
       "      <td>1.181869</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.1</td>\n",
       "      <td>{'c1': 0.01, 'c2': 0.1}</td>\n",
       "      <td>0.839287</td>\n",
       "      <td>0.841692</td>\n",
       "      <td>0.839372</td>\n",
       "      <td>0.840117</td>\n",
       "      <td>0.001114</td>\n",
       "      <td>2</td>\n",
       "      <td>0.942061</td>\n",
       "      <td>0.939559</td>\n",
       "      <td>0.940113</td>\n",
       "      <td>0.940578</td>\n",
       "      <td>0.001073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>266.842001</td>\n",
       "      <td>27.772338</td>\n",
       "      <td>8.383666</td>\n",
       "      <td>1.193904</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1</td>\n",
       "      <td>{'c1': 0.01, 'c2': 1}</td>\n",
       "      <td>0.827796</td>\n",
       "      <td>0.832280</td>\n",
       "      <td>0.824553</td>\n",
       "      <td>0.828210</td>\n",
       "      <td>0.003168</td>\n",
       "      <td>7</td>\n",
       "      <td>0.881836</td>\n",
       "      <td>0.877324</td>\n",
       "      <td>0.880013</td>\n",
       "      <td>0.879724</td>\n",
       "      <td>0.001854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>318.880337</td>\n",
       "      <td>24.992727</td>\n",
       "      <td>6.987665</td>\n",
       "      <td>0.696614</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>{'c1': 0.1, 'c2': 0.01}</td>\n",
       "      <td>0.837140</td>\n",
       "      <td>0.840152</td>\n",
       "      <td>0.837361</td>\n",
       "      <td>0.838218</td>\n",
       "      <td>0.001371</td>\n",
       "      <td>3</td>\n",
       "      <td>0.949872</td>\n",
       "      <td>0.947776</td>\n",
       "      <td>0.948142</td>\n",
       "      <td>0.948597</td>\n",
       "      <td>0.000914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>224.509330</td>\n",
       "      <td>14.575797</td>\n",
       "      <td>6.295670</td>\n",
       "      <td>0.840501</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>{'c1': 0.1, 'c2': 0.1}</td>\n",
       "      <td>0.839524</td>\n",
       "      <td>0.842287</td>\n",
       "      <td>0.839345</td>\n",
       "      <td>0.840385</td>\n",
       "      <td>0.001347</td>\n",
       "      <td>1</td>\n",
       "      <td>0.938383</td>\n",
       "      <td>0.936229</td>\n",
       "      <td>0.936791</td>\n",
       "      <td>0.937134</td>\n",
       "      <td>0.000912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>227.505649</td>\n",
       "      <td>2.021429</td>\n",
       "      <td>7.166668</td>\n",
       "      <td>1.621771</td>\n",
       "      <td>0.1</td>\n",
       "      <td>1</td>\n",
       "      <td>{'c1': 0.1, 'c2': 1}</td>\n",
       "      <td>0.826967</td>\n",
       "      <td>0.830926</td>\n",
       "      <td>0.823649</td>\n",
       "      <td>0.827180</td>\n",
       "      <td>0.002975</td>\n",
       "      <td>8</td>\n",
       "      <td>0.875463</td>\n",
       "      <td>0.873063</td>\n",
       "      <td>0.874631</td>\n",
       "      <td>0.874385</td>\n",
       "      <td>0.000995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>229.349335</td>\n",
       "      <td>3.258423</td>\n",
       "      <td>5.480002</td>\n",
       "      <td>0.185939</td>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>{'c1': 1, 'c2': 0.01}</td>\n",
       "      <td>0.831840</td>\n",
       "      <td>0.840347</td>\n",
       "      <td>0.832166</td>\n",
       "      <td>0.834784</td>\n",
       "      <td>0.003936</td>\n",
       "      <td>4</td>\n",
       "      <td>0.873904</td>\n",
       "      <td>0.873331</td>\n",
       "      <td>0.872819</td>\n",
       "      <td>0.873351</td>\n",
       "      <td>0.000443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>206.296664</td>\n",
       "      <td>4.300135</td>\n",
       "      <td>5.200665</td>\n",
       "      <td>0.738015</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>{'c1': 1, 'c2': 0.1}</td>\n",
       "      <td>0.831417</td>\n",
       "      <td>0.837533</td>\n",
       "      <td>0.830624</td>\n",
       "      <td>0.833191</td>\n",
       "      <td>0.003087</td>\n",
       "      <td>6</td>\n",
       "      <td>0.867803</td>\n",
       "      <td>0.866468</td>\n",
       "      <td>0.866752</td>\n",
       "      <td>0.867008</td>\n",
       "      <td>0.000574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>166.449002</td>\n",
       "      <td>22.228568</td>\n",
       "      <td>3.564668</td>\n",
       "      <td>0.585834</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>{'c1': 1, 'c2': 1}</td>\n",
       "      <td>0.817229</td>\n",
       "      <td>0.823722</td>\n",
       "      <td>0.815745</td>\n",
       "      <td>0.818899</td>\n",
       "      <td>0.003464</td>\n",
       "      <td>9</td>\n",
       "      <td>0.840285</td>\n",
       "      <td>0.836782</td>\n",
       "      <td>0.840217</td>\n",
       "      <td>0.839094</td>\n",
       "      <td>0.001636</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  mean_score_time  std_score_time param_c1  \\\n",
       "0     205.126506     11.023804         5.492002        0.064191     0.01   \n",
       "1     214.131017      6.854912         6.234002        1.181869     0.01   \n",
       "2     266.842001     27.772338         8.383666        1.193904     0.01   \n",
       "3     318.880337     24.992727         6.987665        0.696614      0.1   \n",
       "4     224.509330     14.575797         6.295670        0.840501      0.1   \n",
       "5     227.505649      2.021429         7.166668        1.621771      0.1   \n",
       "6     229.349335      3.258423         5.480002        0.185939        1   \n",
       "7     206.296664      4.300135         5.200665        0.738015        1   \n",
       "8     166.449002     22.228568         3.564668        0.585834        1   \n",
       "\n",
       "  param_c2                    params  split0_test_score  split1_test_score  \\\n",
       "0     0.01  {'c1': 0.01, 'c2': 0.01}           0.832791           0.835960   \n",
       "1      0.1   {'c1': 0.01, 'c2': 0.1}           0.839287           0.841692   \n",
       "2        1     {'c1': 0.01, 'c2': 1}           0.827796           0.832280   \n",
       "3     0.01   {'c1': 0.1, 'c2': 0.01}           0.837140           0.840152   \n",
       "4      0.1    {'c1': 0.1, 'c2': 0.1}           0.839524           0.842287   \n",
       "5        1      {'c1': 0.1, 'c2': 1}           0.826967           0.830926   \n",
       "6     0.01     {'c1': 1, 'c2': 0.01}           0.831840           0.840347   \n",
       "7      0.1      {'c1': 1, 'c2': 0.1}           0.831417           0.837533   \n",
       "8        1        {'c1': 1, 'c2': 1}           0.817229           0.823722   \n",
       "\n",
       "   split2_test_score  mean_test_score  std_test_score  rank_test_score  \\\n",
       "0           0.830906         0.833219        0.002085                5   \n",
       "1           0.839372         0.840117        0.001114                2   \n",
       "2           0.824553         0.828210        0.003168                7   \n",
       "3           0.837361         0.838218        0.001371                3   \n",
       "4           0.839345         0.840385        0.001347                1   \n",
       "5           0.823649         0.827180        0.002975                8   \n",
       "6           0.832166         0.834784        0.003936                4   \n",
       "7           0.830624         0.833191        0.003087                6   \n",
       "8           0.815745         0.818899        0.003464                9   \n",
       "\n",
       "   split0_train_score  split1_train_score  split2_train_score  \\\n",
       "0            0.950367            0.951869            0.952889   \n",
       "1            0.942061            0.939559            0.940113   \n",
       "2            0.881836            0.877324            0.880013   \n",
       "3            0.949872            0.947776            0.948142   \n",
       "4            0.938383            0.936229            0.936791   \n",
       "5            0.875463            0.873063            0.874631   \n",
       "6            0.873904            0.873331            0.872819   \n",
       "7            0.867803            0.866468            0.866752   \n",
       "8            0.840285            0.836782            0.840217   \n",
       "\n",
       "   mean_train_score  std_train_score  \n",
       "0          0.951708         0.001036  \n",
       "1          0.940578         0.001073  \n",
       "2          0.879724         0.001854  \n",
       "3          0.948597         0.000914  \n",
       "4          0.937134         0.000912  \n",
       "5          0.874385         0.000995  \n",
       "6          0.873351         0.000443  \n",
       "7          0.867008         0.000574  \n",
       "8          0.839094         0.001636  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# store CV results in a DF\n",
    "cv_results = pd.DataFrame(rs.cv_results_)\n",
    "cv_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA8AAAAGHCAYAAAB/FDtmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XeYVOXd//H3dwu9I8aChRiNBQQVCzFqjCWAEQQUK0VjjEmMMXnkiT55Eo0xv5hmi1EfjdIUUKwkGjW2GGMsGHvHFlFjQ5Be798fZ9AFFlhgZ8/uzPt1XXPtzJkzZ76zCR/3O/d97hMpJSRJkiRJKnUVeRcgSZIkSVJDsAGWJEmSJJUFG2BJkiRJUlmwAZYkSZIklQUbYEmSJElSWbABliRJkiSVBRtgSZIkSVJZsAFWoxcRh0TEgxExMyL+ExFXRkTb9ThO84i4OiI+KRznh2vZ/weF/WYVXte8xnM/j4hnImJJRJy9Hh9LktYoj+yLiO4RcWdEfBgRacM+gSStv3rMwKER8VBEzIuI+4tQqpoYG2A1Be2Bc4HNgB2ArsBv1uM4ZwPbAlsB+wP/HRF9a9sxIr4GnAEcAGwNfB74WY1dpgH/Ddy2HnVIUl00ePYBi4HrgW+sx/tIUn2qrwycAVwInFd/pakpswFWoxIRW0TETRHxQUR8FBGXpJQmpJTuSCnNSyl9DFwJ7L0ehx8O/Dyl9HFK6YXCcUauZt8RwFUppecK7/nzmvumlMamlP4CzF6POiRpBY0l+1JKL6WUrgKeW9/PIknrqpgZmFK6O6V0PfBOvReuJskGWI1GRFQCfwbeJBt13RyYVMuu+1Ljj7OIuLQwPaa229OFfTqSfYP4VI3jPAXstJpydqpl389FROf1+3SSVLtGln2S1KCKmYFSbaryLkCqYQ+yP9RGpZSWFLY9WHOHiDiIbHR2z+XbUkrfAb6zlmO3KfycVWPbLGB155K0qWVfCvt/tJb3kqR10ZiyT5IaWjEzUFqFI8BqTLYA3qwRfiuIiL2ACcDhKaWX1/HYcwo/29XY1o7VT2GeU8u+rGF/SVpfjSn7JKmhFTMDpVXYAKsxeQvYMiJWmZkQEbsAU4ATUkr3rPTc5RExZzW35wAK5468C/Ss8dKerP48t+dq2fe9lJKjv5LqW2PKPklqaEXLQKk2kZJXOVDjUDgH5F/AX4GzgKXAbmTT9e4BTk0pXbcBxz8P6AMcBnwOuA84PqV0Ry379gXGAF8l++PxRuDRlNIZheergUrgauA1slUKF6eUlq5vfZLKUyPLvgCak618/xzQEkgppYXr+/6StCYNkIGVQDXZ4n/HAAcDS1NKizescjVVNsBqVCJiS+BiYB8gkU15aUt23se8Gru+mVJap0VcCtfxvQw4HJgP/CqldH6N930e2DGl9O/Cth8CPyL7A/BG4OTlfwRGxJhCTTUdn1Iasy41SRI0nuyLiK2B11c6xJsppa3X8SNJUp0VOQNHAqNX2jw2pTRyfetV02YDLEmSJEkqC54DLEmSJEkqC0VtgCPi6oh4PyKeXc3zEREXR8S0iHg6Inat8dyIiHilcBtRY/tuEfFM4TUXF85XkqTcmHWSyoFZJ6kUFHsEeAzQdw3P9wO2LdxOIjtHiYjoRHYS/J5k1wY7KyI6Fl5zWWHf5a9b0/ElqSGMwayTVPrGYNZJauKK2gCnlB4AZqxhl4HAuJR5GOgQEZsCXwP+mlKaUbiEw1+BvoXn2qWU/pmyk5fHka1qKUm5MesklQOzTlIpyPsc4M3Jrv213PTCtjVtn17LdklqzMw6SeXArJPU6K1ywekGVtt5Hmk9tq964IiTyKbU0Lp1692233779a1RUgl6/PHHP0wpdWmgtzPrJOWmAfPOrJOUm7pmXd4N8HRgixqPuwLvFLZ/ZaXt9xe2d61l/1WklK4ArgDo3bt3mjp1an3VLKkERMSbDfh2Zp2k3DRg3pl1knJT16zLewr0FGB4YdXAvYBZKaV3gTuBgyOiY2GRhIOBOwvPzY6IvQqrBA4Hbs2tekmqG7NOUjkw6yQ1ekUdAY6IiWTf+G0UEdPJVgCsBkgpXQ7cDvQHpgHzgOMLz82IiJ8DjxUOdU5KafmiC98mW4WwJfCXwk2ScmPWSSoHZp2kUhDZonulzakyklYWEY+nlHrnXUd9Musk1abU8s6sk1SbumZd3ucA52bx4sVMnz6dBQsW5F1KSWrRogVdu3aluro671KksmbWFZdZJzUOZl3xmXcqFWXbAE+fPp22bduy9dZbk512ovqSUuKjjz5i+vTpdOvWLe9ypLJm1hWPWSc1HmZdcZl3KiV5L4KVmwULFtC5c2dDsggigs6dO/strNQImHXFY9ZJjYdZV1zmnUpJ2TbAgCFZRP5upcbDf4/F4+9Wajz891hc/n5VKsq6Ac7TzJkzufTSS9f79RdeeCHz5s2rx4okqf6ZdZLKgVknNR02wDlpKkGZUmLZsmVFfx9Jpcmsk1QOzDqp6bABzskZZ5zBq6++Sq9evRg1ahQAv/nNb9h9993ZeeedOeusswCYO3cuhxxyCD179qR79+5cd911XHzxxbzzzjvsv//+7L///rUee8cdd2TnnXfm9NNPB+C9995j0KBB9OzZk549e/LQQw8BcP7559O9e3e6d+/OhRdeCMAbb7zBDjvswHe+8x123XVX3nrrLe666y769OnDrrvuyhFHHMGcOXMa4tckqYkz6ySVA7NOajrKdhXomn72p+d4/p1P6vWYO27WjrMO3Wm1z5933nk8++yzPPnkkwDcddddvPLKKzz66KOklBgwYAAPPPAAH3zwAZttthm33XYbALNmzaJ9+/acf/753HfffWy00UYrHHfGjBncfPPNvPjii0QEM2fOBODUU09lv/324+abb2bp0qXMmTOHxx9/nNGjR/PII4+QUmLPPfdkv/32o2PHjrz00kuMHj2aSy+9lA8//JBzzz2Xu+++m9atW/OrX/2K888/n5/+9Kf1+juTVFxmnVknlQOzzqyT1sQR4Ebirrvu4q677mKXXXZh11135cUXX+SVV16hR48e3H333fzoRz/i73//O+3bt1/jcdq1a0eLFi048cQTuemmm2jVqhUA9957L9/+9rcBqKyspH379jz44IMMGjSI1q1b06ZNGwYPHszf//53ALbaaiv22msvAB5++GGef/559t57b3r16sXYsWN58803i/jbkFSqzDpJ5cCskxovR4Bhjd/oNZSUEmeeeSbf+ta3Vnnu8ccf5/bbb+fMM8/k4IMPXuM3dFVVVTz66KPcc889TJo0iUsuuYR77713te+5Oq1bt15hv4MOOoiJEyeuwyeS1NiYdasy66TSY9atyqyTPuMIcE7atm3L7NmzP338ta99jauvvvrTczDefvtt3n//fd555x1atWrFcccdx+mnn86//vWvWl+/3Jw5c5g1axb9+/fnwgsv/HQqzgEHHMBll10GwNKlS/nkk0/Yd999ueWWW5g3bx5z587l5ptvZp999lnlmHvttRf/+Mc/mDZtGgDz5s3j5Zdfrt9fiKSSZNZJKgdmndR0OAKck86dO7P33nvTvXt3+vXrx29+8xteeOEF+vTpA0CbNm245pprmDZtGqNGjaKiooLq6upPw+6kk06iX79+bLrpptx3332fHnf27NkMHDiQBQsWkFLiggsuAOCiiy7ipJNO4qqrrqKyspLLLruMPn36MHLkSPbYYw8ATjzxRHbZZRfeeOONFWrt0qULY8aM4eijj2bhwoUAnHvuuWy33XbF/jVJauLMOknlwKyTmo5Y03SJUtG7d+80derUFba98MIL7LDDDjlVVB78Hasxi4jHU0q9866jPpl1+fB3rMau1PLOrMuPv2c1ZnXNOqdAS5IkSZLKgg2wJEmSJKks2ABLkiRJksqCDbAkSZIkqSzYAEuSJEmSyoINsCRJkiSpLNgA52TmzJlceuml6/Xa/v37M3PmzHquSJLqn1knqRyYdVLTYQOckzUF5dKlS9f42ttvv50OHToUo6w6WVt9krScWSepHJh1UtNhA5yTM844g1dffZVevXoxatQo7r//fvbff3+OOeYYevToAcBhhx3Gbrvtxk477cQVV1zx6Wu33nprPvzwQ9544w122GEHvvnNb7LTTjtx8MEHM3/+/FXea/LkyXTv3p2ePXuy7777AlnYnX766fTo0YOdd96Z3//+9wDcc8897LLLLvTo0YMTTjiBhQsXfvqe55xzDl/+8peZPHkyr776Kn379mW33XZjn3324cUXXyz2r0xSE2TWSSoHZp3UdFTlXUCj8Jcz4D/P1O8xN+kB/c5b7dPnnXcezz77LE8++SQA999/P48++ijPPvss3bp1A+Dqq6+mU6dOzJ8/n913350hQ4bQuXPnFY7zyiuvMHHiRK688kqGDh3KjTfeyHHHHbfCPueccw533nknm2+++adTbK644gpef/11nnjiCaqqqpgxYwYLFixg5MiR3HPPPWy33XYMHz6cyy67jNNOOw2AFi1a8OCDDwJwwAEHcPnll7PtttvyyCOP8J3vfId77723fn53korDrDPrpHJg1pl10ho4AtyI7LHHHp+GJMDFF19Mz5492WuvvXjrrbd45ZVXVnlNt27d6NWrFwC77bYbb7zxxir77L333owcOZIrr7zy02kud999NyeffDJVVdl3IJ06deKll16iW7dubLfddgCMGDGCBx544NPjHHnkkQDMmTOHhx56iCOOOIJevXrxrW99i3fffbd+fgmSSp5ZJ6kcmHVS4+QIMKzxG72G1Lp160/v33///dx9993885//pFWrVnzlK19hwYIFq7ymefPmn96vrKysdarM5ZdfziOPPMJtt91Gr169ePLJJ0kpEREr7JdSqlN9y5Yto0OHDp9+yympiTDrALNOKnlmHWDWSavjCHBO2rZty+zZs1f7/KxZs+jYsSOtWrXixRdf5OGHH17v93r11VfZc889Oeecc9hoo4146623OPjgg7n88stZsmQJADNmzGD77bfnjTfeYNq0aQCMHz+e/fbbb5XjtWvXjm7dujF58mQgC9innnpqveuTVLrMOknlwKyTmg4b4Jx07tyZvffem+7duzNq1KhVnu/bty9Llixh55135ic/+Ql77bXXer/XqFGj6NGjB927d2ffffelZ8+enHjiiWy55ZbsvPPO9OzZkwkTJtCiRQtGjx7NEUccQY8ePaioqODkk0+u9ZjXXnstV111FT179mSnnXbi1ltvXe/6JJUus05SOTDrpKYj1jY9ohT07t07TZ06dYVtL7zwAjvssENOFZUHf8dqzCLi8ZRS77zrqE9mXT78HauxK7W8M+vy4+9ZjVlds84RYEmSJElSWbABliRJkiSVBRtgSZIkSVJZKOsGuBzOf86Lv1up8fDfY/H4u5UaD/89Fpe/X5WKojbAEdE3Il6KiGkRcUYtz28VEfdExNMRcX9EdC1s3z8inqxxWxARhxWeGxMRr9d4rtf61NaiRQs++ugj/zEXQUqJjz76iBYtWuRditQgzLryZNap3Jh15cu8UympKtaBI6IS+ANwEDAdeCwipqSUnq+x22+BcSmlsRHxVeCXwLCU0n1Ar8JxOgHTgLtqvG5USumGDamva9euTJ8+nQ8++GBDDqPVaNGiBV27ds27DKnozLryZtapXJh1Mu9UKorWAAN7ANNSSq8BRMQkYCBQMyh3BH5QuH8fcEstxzkc+EtKaV59FlddXU23bt3q85CSypNZJ6kcmHWSSkIxp0BvDrxV4/H0wraangKGFO4PAtpGROeV9jkKmLjStl8UptdcEBHN66tgSVoPZp2kcmDWSSoJxWyAo5ZtK5+YcTqwX0Q8AewHvA0s+fQAEZsCPYA7a7zmTGB7YHegE/CjWt884qSImBoRU50OI6mIzDpJ5cCsk1QSitkATwe2qPG4K/BOzR1SSu+klAanlHYBflzYNqvGLkOBm1NKi2u85t2UWQiMJpuSs4qU0hUppd4ppd5dunSpn08kSasy6ySVA7NOUkkoZgP8GLBtRHSLiGZkU16m1NwhIjaKiOU1nAlcvdIxjmalaTKFbw+JiAAOA54tQu2SVFdmnaRyYNZJKglFa4BTSkuAU8imubwAXJ9Sei4izomIAYXdvgK8FBEvA58DfrH89RGxNdk3jX9b6dDXRsQzwDPARsC5xfoMkrQ2Zp2kcmDWSSoVUQ7XS+vdu3eaOnVq3mVIakQi4vGUUu+866hPZp2k2pRa3pl1kmpT16wr5hRoSZIkSZIaDRtgSZIkSVJZsAGWJEmSJJUFG2BJkiRJUlmwAZYkSZIklQUbYEmSJElSWbABliRJkiSVBRtgSZIkSVJZsAGWJEmSJJUFG2BJkiRJUlmwAZYkSZIklQUbYEmSJElSWbABliRJkiSVBRtgSZIkSVJZsAGWJEmSJJUFG2BJkiRJUlmwAZYkSZIklQUbYEmSJElSWbABliRJkiSVBRtgSZIkSVJZsAGWJEmSJJUFG2BJkiRJUlmwAZYkSZIklQUbYEmSJElSWbABliRJkiSVBRtgSZIkSVJZsAGWJEmSJJUFG2BJkiRJUlmwAZYkSZIklQUbYEmSJElSWbABliRJkiSVhaq8C2h03n8RqltAdWto1gqqWkKF3xNIkiRJUlNX1AY4IvoCFwGVwB9TSuet9PxWwNVAF2AGcFxKaXrhuaXAM4Vd/51SGlDY3g2YBHQC/gUMSyktqpeCU4LL+kBatuL2qpZZM7y8Ka5uBc1aF34WtlevbZ/VvK6qWb2ULik/TS7rJGk9mHWSSkHRGuCIqAT+ABwETAcei4gpKaXna+z2W2BcSmlsRHwV+CUwrPDc/JRSr1oO/SvggpTSpIi4HPgGcFm9FJ0SDLkKFs+HxfNg0dyVfs6DxXMLP+fDnP8U7tfYZ+k6ZnZFVY2mueUGNNmF52tuq27l6LVUZE0y6yRpHZl1kkpFMUeA9wCmpZReA4iIScBAoGZQ7gj8oHD/PuCWNR0wIgL4KnBMYdNY4GzqKygrKqD74A07xtIlNZrkebU0zjWa5U+fq6XJXjgb5ry36j6kdatnldHrdW2ya9m3umV2v7IZRGzY70tq+ppe1knSujPrJJWEYjbAmwNv1Xg8HdhzpX2eAoaQTacZBLSNiM4ppY+AFhExFVgCnJdSugXoDMxMKS2pcczNa3vziDgJOAlgyy23rJ9PVBeVVVDZHlq0r/9jpwRLFqzUUM/NRqNX12QvmlvLiPa8rLleYd/5sHThutUTlas20HVpsleYEr66JtvRazUZ5Zl1ksqNWSepJBSzAa5taHDl4cvTgUsiYiTwAPA2WTACbJlSeiciPg/cGxHPAJ/U4ZjZxpSuAK4A6N279zoOmzZSEYXmsiXZfzPq2dIlNUamVxqRXqXJrm1Ee352f9FcmPPBqvus8+h1izo20OvQZLfe2POuVd/MOknlwKyTVBKK2QBPB7ao8bgr8E7NHVJK7wCDASKiDTAkpTSrxnOklF6LiPuBXYAbgQ4RUVX4tnCVY2oDVFZBZTto0a7+j7189Hp5k7zac6vXMm180TyY8/6q+9R19LrDVjDiT9Bxq/r/jCpXZp2kcmDWSSoJxWyAHwO2Lazu9zZwFJ+d4wFARGwEzEgpLQPOJFs5kIjoCMxLKS0s7LM38OuUUoqI+4DDyVYMHAHcWsTPoPpSc/S6Vaf6P/4qo9e1TPteMBPu/TmMPRSOvx3ad63/OlSOzDpJ5cCsk1QSitYAp5SWRMQpwJ1ky+VfnVJ6LiLOAaamlKYAXwF+GRGJbKrMdwsv3wH4v4hYBlSQnSuyfJGFHwGTIuJc4AngqmJ9BjUhdR293mxXGDcQxg7ImuC2mzRMfSpZZp2kcmDWSSoVkVLpn0bRu3fvNHXq1LzLUGPx70dg/KBsBHjkbdCmS94VKQcR8XhKqXfeddQns05SbUot78w6SbWpa9a5zK7Kz5Z7wrHXw8x/Z6PB82bkXZEkSZKkBmADrPK09Zfh6Inw0bSsCZ7/cd4VSZIkSSoyG2CVr232hyOvgfdfgGuGwILarsYgSZIkqVTYAKu8bXcwDB0L7z4F1x4OC+fkXZEkSZKkIrEBlrY/BIb8EaY/BhOPyi6ZJEmSJKnk2ABLADsNgkFXwBsPwqRjYPGCvCuSJEmSVM9sgKXldj4CBl4Cr90H1w+DJQvzrkiSJElSPbIBlmra5Tj4+gXwyl0w+XhYujjviiRJkiTVExtgaWW9T4B+v4aXboMbT4SlS/KuSKp/frkjSZLKUFXeBUiN0p7fyqZA//UnUNkMBl0OFZV5VyXVn4t3hdnvQrPW0KxN4WfrWh6v5bnmbVbcXtUCIvL+dJIkSbWyAZZWZ+9TYelCuPfcrAke8HuocNKESsReJ8PcD2DR3MJtzmf3P5leY3vhubqKirU0zWtqsNvU/rO6lf/2JElSvbABltZk31GwZBE88GuoagaHnO/olkpDn+/Wfd9ly2DJ/FUb5RXuFx4vnFP7c3PeX2nf2ZCW1b2G6vVpqGs8bt5mxeeqW0Ol/wmUJKnc+F9/aW32/59sJPgfF2UjwX3PswlWeamo+KxxZOP6OWZK2WkGa2yqa7tf4+eCmfDJ2ytuX7qo7jVUtVi/ad+f3m+76n5Vzern9yNJkorCBlhamwg48GfZSPAjl2VN8EHn2ARLGyICqltkt9ad6++4SxbB4rmraaLr0GAvnANzPljxuSXz6/7+FdUbdl5187arbve8akmS6o0NsFQXEdD3l9no0kMXQ1Vz+Or/5l2VpJVVNctuLTvW3zGXLV23JrrW86rfXnUbqW7vvz7nVXfZHj6/X/39DiRJKhE2wFJdRUD/32ZN8AO/gcrmsN+ovKuSVGwVldCiXXarL+tyXvXqnpv7AXz8xornX6eln73HQefA3t+vv5olSSoBNsDSuqiogEMvyq6het+52UiTf2BKWldFPa96Dtw+Cv76U1g8H/b7kVOoJUkqsAGW1lVFJQz8Q7Yw1l9/mp0TvNe3865KUrmreV71kD9ml4+6/5fZCLHrFkiSBNgAS+unsgoGX5mNBN9xRtYE7/6NvKuSpExFZXbt8uqW2boFi+dDv197PWVJUtmzAZbWV2U1HD4arjsObvth1gTvOizvqiQpU1EB/X+zYhM84OKsOZYkqUzZAEsboqoZDB0Hk46GKd/LmuCeR+ZdlSRlIrLpz81aZ9Ohl8yHQf+XfYEnSVIZsgGWNlR1CzjyWpgwFG45OfvDsvvgvKuSpEwEfOWMbCT4rz+FxQvgiNHZ5dwkSSozngwk1YdmreDoSdB1D7jxRHjhz3lXJEkr2vv72aXcXroNJh4Fi+blXZEkSQ3OBliqL83bwLGTYbNdYPJIePnOvCuSpBXt8c1sFfvX7odrD4eFs/OuSJKkBmUDLNWnFu3guBvhczvCdcNg2j15VyRJK9rluGwV+38/DOMOg/kf512RJEkNxgZYqm8tO8CwW2CjbWHSMfD63/OuSJJW1ONwOHI8/OdpGHsozP0w74okSWoQNsBSMbTqlDXBHbeGCUdmIy2S1JhsfwgcPRE+fAVG94dP3s27IkmSis4GWCqWNl1g+K3QdhO45nCYPjXviiRpRV84MDtt45O3YXQ/mPnvvCuSJKmobIClYmq7CYz4E7TuDOMHwztP5l2RJK1o6y9nX9bNnwFX94OPXs27IkmSisYGWCq29ptnTXCLdjD+MPjPs3lXJEkr6tobRvwZlszPpkO//2LeFUmSVBQ2wFJD6LAljJgCVS1h3ED/uJTU+Gy6M4y8Pbs/pj+8+1S+9UiSVAQ2wFJD6fT5rAmOChg3AD6clndFkrSijbeH42+H6lYw5lB467G8K5IkqV4VtQGOiL4R8VJETIuIM2p5fquIuCcino6I+yOia2F7r4j4Z0Q8V3juyBqvGRMRr0fEk4Vbr2J+BqlebbRt1gQvW5JdemTG63lXpHpg1qmkdN4ma4JbdcpO23jjwbwrUiNh1kkqBUVrgCOiEvgD0A/YETg6InZcabffAuNSSjsD5wC/LGyfBwxPKe0E9AUujIgONV43KqXUq3BzVSE1LRvvkC04s3gejB3gqqtNnFmnktRhSzjhDmjfFa4ZAtPuzrsi5cysk1QqijkCvAcwLaX0WkppETAJGLjSPjsC9xTu37f8+ZTSyymlVwr33wHeB7oUsVapYW3SA4bfAgtmZSPBn7yTd0Vaf2adSlPbTWDkbdnMlYlHw4u35V2R8mXWSSoJxWyANwfeqvF4emFbTU8BQwr3BwFtI6JzzR0iYg+gGVDzugy/KEyhuSAimtf25hFxUkRMjYipH3zwwYZ8Dqk4Ntslu/7m3A+zJnj2e3lXpPVj1ql0td4oW8V+k53humHwzA15V6T8mHWSSkIxG+CoZVta6fHpwH4R8QSwH/A2sOTTA0RsCowHjk8pLStsPhPYHtgd6AT8qLY3TyldkVLqnVLq3aWLXzKqkdpidzj2hmwEeNyArBlWU2PWqbS17JjNWNmyD9x4IvxrfN4VKR9mnaSSUMwGeDqwRY3HXYEV5nmmlN5JKQ1OKe0C/LiwbRZARLQDbgP+N6X0cI3XvJsyC4HRZFNypKZrqz5w9CT4+A0YdxjMm5F3RVo3Zp1KX/O2cOxk2GZ/mHIKPHJF3hWp4Zl1kkpCMRvgx4BtI6JbRDQDjgKm1NwhIjaKiOU1nAlcXdjeDLiZbCGFySu9ZtPCzwAOA54t4meQGsbn94OjroUPX4Lxg2D+zLwrUt2ZdSoPzVplX9Z98RD4yyh48MK8K1LDMusklYSiNcAppSXAKcCdwAvA9Sml5yLinIgYUNjtK8BLEfEy8DngF4XtQ4F9gZG1LIt/bUQ8AzwDbAScW6zPIDWoLxwIQ8fDe8/BtYfDwtl5V6Q6MOtUVqqaw9Cx0H0I3H0W3PdLSCvPglUpMusklYpIZfAfrt69e6epU6fmXYZUNy/8Ca4fAVvskS2S1ax13hWVpIh4PKXUO+866pNZpwazbCn86VR44hr40vfgoJ9D1HaKqBqDUss7s05SbeqadcWcAi1pfexwKAy5Et56BCYeBYvn512RJK2oohIO/T3scRI89Hu47b8RAVs5AAAgAElEQVRg2bK1v06SpJxV5V2ApFp0HwJLF8PNJ8OkY+GoCVDdIu+qJOkzFRXQ79dQ3RL+cREsWQADfp81x5IkNVI2wFJj1fMoWLoIpnwPJo/Izg+uapZ3VZL0mQg48GdQ3Rru/3/ZjJXBV0Bldd6VSZJUqzpNgY6IlhHxxWIXI2kluw6H/r+Fl++AG0/IRoVVNGadtB4i4Cs/ys4Dfu4muH44LF6Qd1VaA7NOUjlbawMcEYcCTwJ3FB73iogpa36VpHqzxzfha7/MFse6+VvZ4jOqd2adtIH2PjX7wu6l27P1CxbNy7si1cKsk1Tu6jICfDbZRclnAqSUngS2Ll5JklbR5ztw4Nnw7I1w63ddbKY4zsaskzbMHt+EgZfC63+Da4bAgk/yrkirOhuzTlIZq8s5wEtSSrPCyxtI+fryD2DJouw8u8pq+PpF2SI0qi9mnVQfdjk2W7TvppNg/GFw7A3QqlPeVekzZp2kslaXBvjZiDgGqIyIbYFTgYeKW5akWu3337B0Ifz9d1DZLJtu6B8x9cWsk+pL9yFQ1TJbwG/sABh2M7TpkndVyph1kspaXYaPvgfsBCwEJgCzgNOKWZSk1YiAr/4E+pwCj/0R7vwxpJR3VaXCrJPq0/b94ehJ8NE0GNMfPnk374qUMesklbU1jgBHRCXws5TSKODHDVOSpDWKgIPPzVaEfvgP2aWRDjjLkeANYNZJRfKFA+C4G2HCUBjdF4ZPgY5b5V1V2TLrJGktI8AppaXAbg1Ui6S6ioB+v4LdRsKDF8D95+VdUZNm1klFtPXeWeM7/2MY3R8+ejXvisqWWSdJdTsH+InC8viTgbnLN6aUbipaVZLWLgIOuSAbCf7bedlI8D7/lXdVTZlZJxVL191gxJ+zRbFG94Pht8LGO+RdVbky6ySVtbo0wJ2Aj4Cv1tiWAINSyltFBQz4PSxdBPecA5XN4Uun5F1VU2XWScW06c5w/F+yRbFG988WxtqsV95VlSOzTlJZW2sDnFI6viEKkbSeKirhsMuzJviuH2erQ+95Ut5VNTlmndQAunwRjr8dxg3MGuHjboAt9si7qrJi1kkqd2tdBToiukbEzRHxfkS8FxE3RkTXhihOUh1VVsGQq+CL/eEvo2Dq6LwranLMOqmBdN4mGwlu3RnGHQav/z3visqKWSep3NXlMkijgSnAZsDmwJ8K2yQ1JpXVcMQY+MJB8OcfwJMT8q6oqTHrpIbSYYusCe6wBVx7OLxyd94VlROzTlJZq0sD3CWlNDqltKRwGwN4NXupMapqDkeOh8/vB7d+F565Ie+KmhKzTmpIbTeBkbfDRtvBxKPghT/lXVG5MOsklbW6NMAfRsRxEVFZuB1HtniCpMaouiUcNRG27AM3nQTP35p3RU2FWSc1tNadYcSfssWwrh8BT0/Ou6JyYNZJKmt1aYBPAIYC/wHeBQ4vbJPUWDVrBcdcB5vvBjecAC/enndFTYFZJ+WhZYdsReitvgQ3fRP+NS7vikqdWSeprNVlFeh/AwMaoBZJ9al522yF1XGHweQR2ajwtgfmXVWjZdZJOWreFo65Hq47DqZ8DxbPhz2/lXdVJcmsk1Tu6rIK9NiI6FDjcceIuLq4ZUmqFy3aw7CbskuPTDoGXrs/74oaLbNOylmzVnD0RNj+6/CX/4YHL8i7opJk1kkqd3WZAr1zSmnm8gcppY+BXYpXkqR61bIjDLs1u/TIhKPgjX/kXVFjZdZJeatqnq1m3/1wuPtsuPcXkFLeVZUas05SWatLA1wRER2XP4iITtRh6rSkRqR1Zxh+a3bJkQlD4a1H866oMTLrpMagshoGXwG7DIMHfg13/a9NcP0y6ySVtboE3u+AhyJi+fVUjgB+UbySJBVFm41h+BQY0x+uGQLDb8kWydJyZp3UWFRUwqEXQ3Ur+OclsHge9P8dVNTle3uthVknqayt9b8kKaVxwBDgPeB9YHBKaXyxC5NUBO02zS450rIDjB8E7z6dd0WNhlknNTIVFdDvV7D3aTD16uza5kuX5F1Vk2fWSSp3dVkEaxvg1ZTSJcAzwIE1F0+Q1MS075o1wc3awriB8N7zeVfUKJh1UiMUAQeeDfv/GJ6aADd+A5YsyruqJs2sk1Tu6jKX6EZgaUR8Afgj0A2YUNSqJBVXx61hxBSobAbjBsAHL+ddUWNg1kmNUQTs999w8Lnw/C1w/TBYvCDvqpoys05SWatLA7wspbQEGAxclFL6AbBpccuSVHSdt8lGggHGHgofvZpvPfkz66TG7Evfg0N+By/fAROPhEVz866oqTLrJJW1ujTAiyPiaGA48OfCturilSSpwXTZLlsYa+kiGDsAPn4z74ryZNZJjd3uJ8Jhl8HrD2SL+S34JO+KmiKzTlJZq0sDfDzQB/hFSun1iOgGXFPcsiQ1mM/tmK0IvWg2jP06zJqed0V5MeukpqDXMTDkKpj+WLaOwbwZeVfU1Jh1kspaXVaBfj6ldGpKaWJE7JpSej2ldF5DFCepgWzaE4bdDPNnZtOhP3k374oanFknNSHdB8OR18B7z2aZNeeDvCtqMsw6SeVuXS+o98d12Tki+kbESxExLSLOqOX5rSLinoh4OiLuj4iuNZ4bERGvFG4jamzfLSKeKRzz4oiIdfwMkmqz+W5w7A0w+71sYaw57+ddUZ7MOqmx+2I/OOa6bP2C0f3gk3fyrqgpMusklZ11bYDrHEoRUQn8AegH7AgcHRE7rrTbb4FxKaWdgXOAXxZe2wk4C9gT2AM4KyI6Fl5zGXASsG3h1ncdP4Ok1dlyTzh2Msx8K5taOPejvCvKi1knNQXbfBWG3QSz/5M1weW9jsH6MOsklZ11bYB/tg777gFMSym9llJaBEwCBq60z47APYX799V4/mvAX1NKM1JKHwN/BfpGxKZAu5TSP1NKCRgHHLaOn0HSmmy9Nxw9MRtVGT8Q5n+cd0V5MOukpmKrL8HwW7OsGt0PPpyWd0VNiVknqeystgGOiFNq3N8JIKV0yzoce3PgrRqPpxe21fQUMKRwfxDQNiI6r+G1mxfur+mYkjbUNvvDURPgg5dg/GBYMCvviorGrJNKQNfdYORtsGRh1gS/93zeFTU6Zp0kZdY0AnxCjfvj1+PYtU2rSSs9Ph3YLyKeAPYD3gaWrOG1dTlm9uYRJ0XE1IiY+sEHLo4hrbNtD4Sh4+A/T8O1R8DC2XlXVCxmnVQKNukBx98OFZUw5hB454m8K2pszDpJou5ToNdnQYLpwBY1HncFVlihIqX0TkppcEppF+DHhW2z1vDa6YX7qz1mjWNfkVLqnVLq3aVLl/UoXxJf7AeHXw3Tp8KEo2DRvLwrKjazTmrKunwxa4Kbtcmubf7vR/KuqLEy6ySVrTU1wB0iYlBEDAHaRcTgmrc6HPsxYNuI6BYRzYCjgCk1d4iIjSJieQ1nAlcX7t8JHBwRHQuLJBwM3JlSeheYHRF7FVYJHA7cWudPK2nd7TgQBl8B/34IJh0NixfkXVF9M+ukUtLp81kT3LoLjB8Er/0t74oaC7NOkoCqNTz3N2BA4f4DwKE1nkvATWs6cEppSeF8kzuBSuDqlNJzEXEOMDWlNAX4CvDLiEiF9/hu4bUzIuLnZGELcE5KafmV7r8NjAFaAn8p3CQVU4/DYekiuOU7cN1xcNS1UNU876rqi1knlZoOW8Dxf8lWs58wNLtm8LYH5V1V3sw6SQIiW3SvtPXu3TtNnTo17zKkpu/xMfCn78MXD4GhY6GyOu+K1ltEPJ5S6p13HfXJrJNWMvcjuGZQtijW4VfDjgPW/poSVGp5Z9ZJqk1ds25dL4MkqZztNhL6/xZeug1u/AYsXZJ3RZK0eq07w/ApsNkuMHkkPH193hVJknJmAyxp3ezxTTj4F/D8rXDLybBsad4VSdLqtewAw27Orhd800nZTBZJUtla0znAklS7L50CSxfCPedAZTMYcAlU+H2apEaqeRs4djJcNyw7jWPxfNjr23lXJUnKQZ0a4Ij4ErB1zf1TSuOKVJOkpmCf/4Ili+Bv52VN8NcvgFifK2s0HmadVMKqW2YL+N1wAtxxBiyel+VYGTLrJJWztTbAETEe2AZ4Elg+1zEBBqVU7r5yRjYS/OAFWRPc71dNtgk266QyUNUcjhibnb5xzznZSPD+P26yubU+zDpJ5a4uI8C9gR1TOSwXLWndRMABZ2UjwQ//AaqawUE/b6p/TJp1UjmorIJB/5eNCD/wG1g0D772i6aaW+vDrJNU1urSAD8LbAK8W+RaJDVFEdkfj0sXwUO/h8rmcMBP8q5qfZh1UrmoqISvXwRVLbMv7xbPg0POL5e1DMw6SWWtLg3wRsDzEfEosHD5xpRSeV5MT9KqIqDfr7Pp0H//bTbNcL//zruqdWXWSeWkoiI7baNZq+w0jsXzYeAfshHi0mbWSSprdUn5s4tdhKQSUFGRjagsXQz3/SI7J/jLp+Vd1bo4O+8CJDWw5adxVLeG+86FJfNh8B+z0zlK19l5FyBJeVprA5xS+ltDFCKpBFRUZCMoSxfB3WdlTXCf7+RdVZ2YdVKZioD9RmUjwXf+DyxeAEPHQXWLvCsrCrNOUrlb68kuEbFXRDwWEXMiYlFELI2ITxqiOElNUEVltsDMDofCnWfCY3/Mu6I6MeukMtfnu9l5wK/cCROPhEVz866oKMw6SeWuLqs9XAIcDbwCtAROLGyTpNpVVsOQq2G7fnDbf8G/msTVNcw6qdzt/g047HJ4/QEYPxgWzMq7omIw6ySVtTotd5hSmgZUppSWppRGA18palWSmr6qZjB0LGxzAEw5FZ6alHdFa2XWSaLX0XD41fD2VBg3EObNyLuiemfWSSpndVkEa15ENAOejIhfky2b37q4ZUkqCVXN4ahrYcJQuOXb2chw9yF5V7U6Zp2kzE6DoKoFXD8cxnwdht8CbTbOu6r6YtZJKmt1GQEeVtjvFGAusAXQaP+CldTIVLeEoyfBFnvBjd+EF/6Ud0WrY9ZJ+swX+8Ex18PHr8Po/jDr7bwrqi9mnaSyttYGOKX0JhDApimln6WUfliYOiNJddOsNRx7PWy+K0w+Hl66I++KVmHWSVrFNvvDcTfB7P/A6H7w8Rt5V7TBzDpJ5a4uq0AfCjwJ3FF43CsiphS7MEklpnlbOPYG+NxOcP0wmHZP3hWtwKyTVKut+sCIW7MFsUb3hw9fybuiDWLWSSp3dZkCfTawBzATIKX0JLB18UqSVLJadoBhN8NGX4RJx2QrrTYeZ2PWSarN5rvByNtgycJsJPi95/KuaEOcjVknqYzVpQFeklIqyesASMpBq07ZgjIdu8GEI+HNf+Zd0XJmnaTV26Q7HP8XqKiCMYfA2//Ku6L1ZdZJKmt1aYCfjYhjgMqI2DYifg88VOS6JJWy1hvB8Fuh3WZw7REwfWreFYFZJ2ltumyXNcHN22aXSPr3w3lXtD7MOkllrS4N8PeAnYCFwETgE+C0YhYlqQy0/RyM+FPWDI8fDO88kXdFZp2ktevULWuC22wM4wfBa3/Lu6J1ZdZJKmt1WQV6Xkrpxyml3VNKvQv3FzREcZJKXLvNsia4RfvsD8n/PJNbKWadpDpr3xVG3g4dt85msbx8Z94V1ZlZJ6ncVa3uibWtCJhSGlD/5UgqOx22gBFTsnPqxg3M/qjcePsGe3uzTtJ6afu5bGGs8YfBpGPh8Ktgx4F5V7VaZp0kZVbbAAN9gLfIpsc8QnbNOEmqf526ZSPBo/vBuAFZE7zRFxrq3c06SeunVScYPiUbBZ48Eg67HHoemXdVq2PWSRJrngK9CfA/QHfgIuAg4MOU0t9SSk3uhBdJjVznbbI/JJcthbGHwozXGuqdzTpJ62/55d222htu/hZMHZ13Ratj1kkSa2iAU0pLU0p3pJRGAHsB04D7I+J7DVadpPKy8fbZ6tBL5sPYATDz30V/S7NO0gZr3gaOnQzbHgR/Pg0evizvilZh1klSZo2LYEVE84gYDFwDfBe4GLipIQqTVKY26Q7DboGFn2QjwbPeLvpbmnWSNlh1SzjyWtjhULjjDHjgt3lXtAqzTpLWvAjWWLJpMn8BfpZSerbBqpJU3jbrBcfdnC2KNW5AttBM202K8lZmnaR6U9UMDh8Dt3wb7v05LJ4HX/0JRP6n25Zt1v3p+/DxG9CyE7TsmJ23vbr7LdpDRWXeFUsqsjUtgjUMmAtsB5wan4V3ACml1K7ItUkqZ113g+NuyK4RPG5g1gS33qgY72TWSao/lVUw6HKobgF//x0smgd9f9kYmuDyzLrKZrBoLsyaDvNmwIKZkJatZufImuA1Nck1f7bslN1v1qYx/O8rqY5W2wCnlNZ6jWBJKqot94JjrstWWB03MFspulWnen0Ls05SvauohEMvhupW8Mhl2boGh1wAFfnFTdlmXf/frPh42TJYOCtrhud/nN3mzYD5M1a9P/cD+PAlmD8zOy1ndSqqa2mMO67YJNd2v7pFcT+7pFqtaQRYkvLXbR84egJMOCq73ubwKdmqq5LUmEVA3/OyJvjB82HxfBh4aTZCrPxUVBQa0I7r9rqli9fcMNdsqD9+A955Itu+ZMHqj1ndat0a5ladoEUH/z8kbSD/BUlq/Lb5Khx5DUw6Bq4Zkl1ypEVpztaTVEIi4MCzoFkruPfcrAkeclV2rrCalspqaLNxdlsXi+YVGuOaTfKMWkagP4b3X/jsflq6+mM2b1/HhrnjZ9O3W7R3mrZUUNQGOCL6kl1rrhL4Y0rpvJWe3xIYC3Qo7HNGSun2iDgWGFVj152BXVNKT0bE/cCmwPzCcwenlN4v5ueQ1AhsdzAcMQYmj4AJQ+HYG7JLjzQCZp2kNdp3FFS3hjvPhOuOhaHjslWjmxizbj00a5Xd2m9e99eklE25XqFh/nj1jfSM17L7C2at/phR+dnId23nNa9ugbDqljbOKjlFa4AjohL4A9mF1qcDj0XElJTS8zV2+1/g+pTSZRGxI3A7sHVK6Vrg2sJxegC3ppSerPG6Y1NKU4tVu6RGaoevw5A/wg0nwMSj4Jjrsz8scmTWSaqTPt/Jzvn88w+zL/GOmthovsSrC7OuAUVhMa4W7YFudX/d0iVZE7za0eYa9z+ZDv95Jru/eO7qj1nZfKXGuOOqTfLK91t2dJaDGrVijgDvAUxLKb0GEBGTgIFAzaBMwPJ5jO2Bd2o5ztHAxCLWKakp2WlQdi7WTSdlIylHTcx7IRGzTlLd9D4hO+/zlm/DNYPh2MmFJqdJMOsau8oqaN05u62LxQs+m469tqnaH77y2f1li1d/zGZt69Yw1xyBbtEh14XiVD6K2QBvDrxV4/F0YM+V9jkbuCsivge0Bg6s5ThHkgVsTaMjYilwI3BuSinVS8WSmoadh8KShTDlFLh+eHZ+cH7fNpt1kuqu51FQ1RxuPBHGDsjWNKjn1e2LxKwrVdUtoHpTaLdp3V+TEiyas5pFwWpZLGzmvwv3Z5J9T1KbyBa5XO15zbVcgqp1lyZ5OoHyVcwGuLYTBlb+f/zRwJiU0u8iog8wPiK6p5RdoC0i9gTmrXSx9mNTSm9HRFuyoBwGjFvlzSNOAk4C2HLLLTf800hqXHYdBksXwW0/hBuOz84PrqzOoxKzTtK62WkQVLXMvsAbcwgMuwXafi7vqtbGrNNnIqB52+zWYR3+91i2LLsW89ouQTVvBsx5Dz54MTv/edHs2o9X2Rx2HQ57fx86bFE/n00lr5gN8HSg5v8Tu7LqVJhvAH0BUkr/jIgWwEbA8sUPjmKlaTIppbcLP2dHxASyKTmrBGVK6QrgCoDevXv7TaJUinb/RjYd+o4fZVOiB1+Zx+UhzDpJ6+6LfeHY62Hi0TCmf3aJt3VZKKnhmXXacBUV2cjtus56WLIoa5xXbpLfehgeHwOPj85mV3z5h9B5m6KUrtJRzIn2jwHbRkS3iGhGFnpTVtrn38ABABGxA9AC+KDwuAI4Api0fOeIqIqIjQr3q4GvA88iqXztdTIcdA48dxO8fn8eFZh1ktbP578Cx90Es9+D0X1hxut5V7QmZp3yU9UsuwTVxtvDVl+C7Q/JZoIN/AN8/0no/Q145ga4pHe2UOZ7z+VdsRqxojXAKaUlwCnAncALZKsCPhcR50TEgMJu/wV8MyKeIvtGcGSN8z72BaYvX2yhoDlwZ0Q8DTwJvA1cWazPIKmJ2Pv7cNLf4Au1nW5WXGadpA2yVR8YcSss+ARG94cPXs67olqZdWq02neF/r+G056BL30PXr4TLvsSTDwG3n487+rUCEU5rDPQu3fvNHWqq+tL+kxEPJ5S6p13HfXJrJOasP88C+MPy+4PuwU26V5vhy61vDPrtEbzZsAj/wePXJZdFmqbr8I+p8PWe+ddmYqsrlnnWuOSJEl526Q7jLwdKqqyhbEcuZLWT6tOsP+ZcNqzcODZ2fWOx/SHq/vBtLuzFaxV1myAJUmSGoMu28Hxf4EW7WDsQHjzn3lXJDVdLdrBl38A338a+v4KZr4J1wyBK/eHF/6crUitsmQDLEmS1Fh06gbH35FdFumawfDqfXlXJDVtzVplC2ae+gQcelG2gvR1x8Lle2cLZy1bmneFamA2wJIkSY1J+82zkeCOW8OEI7NFfSRtmKrmsNtIOOXx7LKJy5bCjd+AS3aHf43PLrWksmADLEmS1Ni02RhG3gYb7wCTjoHnbsm7Iqk0VFbBzkPhOw/D0PHQrDVMOQV+vys8eiUsnp93hSoyG2BJkqTGqFUnGDEFNt8Nbjgenpq09tdIqpuKCthxAHzrATj2Bmi3Gdx+OlzUE/5xMSyck3eFKhIbYEmSpMaqRXs47ibY+stw88kw9eq8K5JKSwRsexCccCeM+HM26+KvP4ELu8Pffg3zZ+ZdoeqZDbAkSVJj1rwNHHN99kf6n38AT0/OuyKp9ERAt31g+K1w4j2wxV5w3y/ggu5w989g7od5V6h6YgMsSZLU2FW3hCOvzS7rsu2BeVcjlbauveGYSXDyg9m/twcvyBrhO86ET97JuzptIBtgSZKkpqCqGRx4NrTsmHclUnnYpAccMQa++yjsNAge+b/sHOE/fR9mvJ53dVpPNsCSJEmStDpdtoNBl8Gp/4JdjoMnJ8Dvd4ObvgUfvJR3dVpHNsCSJEmStDYdt4avXwDffwr2PBlemAJ/2BOuHw7vPp13daojG2BJkiRJqqt2m0Hf/wenPQP7/BBevQ/+bx+4dii89Wje1WktbIAlSZIkaV213ggO+GnWCO//vzD9MbjqIBh7KLz2N0gp7wpVCxtgSZIkSVpfLTvAfqOyRvjgc7PzgscNgKsOhpfvtBFuZGyAJUmSJGlDNW8DX/oefP9pOOR3MPtdmDA0mx793C2wbFneFQobYEmSJEmqP9UtYPcT4dQnYOClsGgeTB4Bl+4FT02CpUvyrrCs2QBLkiRJUn2rrIZdjoVTHoPDr4aKKrj5W3DJbjB1NCxZmHeFZckGWJIkSZKKpaISug+Bkx+EoyZCq87w59Pgol7w8GXZCLEajA2wJEmSJBVbRQVs3x9OvAeG3QydPg93nAEX9oC/nw8LPsm7wrJgAyxJkiRJDSUCtvkqHH8bHH8HbNYL7vkZXNgd7v0FzJuRd4UlzQZYkiRJkvKwVR847kb45n2w9T7wwK/hgu5w1//C7Pfyrq4k2QBLkiRJUp423xWOuha+/c9smvQ//5BNjb7tdJj5Vt7VlRQbYEmSJElqDD63Iwz5I5wyFXYeCo+PgYt7wa3fhY9ezbu6kmADLEmSJEmNSedtYOAl2bWEe58Az9wAl/SGG74B7z2fd3VNmg2wJEmSJDVGHbaA/r+B7z8NfU6Bl++Ay/rApGPh7X/lXV2TZAMsSZIkSY1Z28/BwT+H056B/X4Eb/wdrtwfxg+GNx/Ku7omxQZYkiRJkpqCVp1g//+B056FA8+Gd5+C0f3g6n4w7R5IKe8KGz0bYEmSJElqSlq0gy//IBsR7vsr+PgNuOb/t3fvUZKX9Z3H39+q6uv0ZZgrAwOCgiIXFZ3gJWrOqiHIUfG2Bry7HAnZxXjflVXPMSa6ZnezAke8gJpRVyWs0YgJBtZINCYojKIIurgDnoVhYKYHmO659L2e/aOqu6uqq29M11R3/d6vc/pU1+/3/J56qh76O3zq96unXl06K/x//h6KxWaPcMUyAEuSJEnSatTeDc+5FN75c3j5lTD8GFz3evjs80sLZxUnmz3CFccALEmSJEmrWaEDnvVWuOyn8KproDgBf3MxfOp34I7/CZPjzR7himEAliRJkqRWkC/A0/8Q/v2P4XVfhvY1pe8QvupsuO1aGB9p9gibrqEBOCLOi4h7ImJnRHygzv4TI+KWiLgjIu6MiPPL20+KiOGI+Hn557MVxzwrIn5Z7vOqiIhGPgdJWoi1TlIWWOukVSSXg9MvgD/6Ibz+f0HvFrjxfXDl0+BfroLRg80eYdM0LABHRB64GngpcDpwUUScXtPsQ8D1KaWzgQuBT1fsuzel9Izyz6UV2z8DXAKcWv45r1HPQZIWYq2TlAXWOmmVioAnnwsX3wxv+Q5sPA3+94fhijPhB/8Vhvc3e4RHXSPPAJ8D7Ewp3ZdSGgOuAy6oaZOAvvLv/cDu+TqMiC1AX0rp1pRSAr4MvHJ5hy1JS2Ktk5QF1jppNYuAk18Ib7kBLv4enPBsuOVjcMVZ8L0/hUP7mj3Co6aRAfh44IGK+7vK2yp9BHhjROwCbgTeUbHv5PIlND+IiBdU9LlrgT4BiIhLImJHROwYGBg4gqchSfOy1knKAmud1CpO+B14/V/DH/0zPOlF8KNPwifPhH+4HIbmfd+qJTQyANf7DEftNzNfBGxPKW0Fzge+EhE54CHgxPIlNO8BvhYRfYvss7QxpWtSSttSSts2btz4uJ+EJC3AWicpC6x1UqvZ8jR43ZfgP9wGZ7wSfvI5uLHXP6gAABkSSURBVPLp8J13lb5XuEU1MgDvAk6ouL+V2ZfCXAxcD5BSuhXoBDaklEZTSo+Ut/8UuBd4crnPrQv0KUlHk7VOUhZY66RWtfHJ8KrPwp/8DJ7xBvj5V+GqZ8K3LoWB3zR7dMuukQH4duDUiDg5ItopLYZwQ02b+4EXA0TEUykVyoGI2FhebIGIeCKlRRHuSyk9BByIiOeUVwl8M/DtBj4HSVqItU5SFljrpFZ3zEnw8ivgnb+AZ18Kd/8tXH0OXP8WeOjOZo9u2TQsAKeUJoDLgJuAX1NaFfDuiPhoRLyi3Oy9wNsj4hfA14G3lhdBeCFwZ3n7N4BLU0qPlo/5Y+DzwE5K7yB+t1HPQZIWYq2TlAXWOilD+o6D8z4O774LXvAeuPf78LkXwFdfBw/c1uzRHbEo1aXWtm3btrRjx45mD0PSChIRP00pbWv2OJaTtU5SPa1W76x10lE2vB9uuxZ+fDUMP1ZaTfqF74eTXlBaXXqFWGyta+Ql0JIkSZKk1axrLfze++Fdd8G5fw4D98CXXg5f/AP4zc2wyk6oGoAlSZIkSfPr6IHnvQPeeSec/99LX5n0tX8Ln3sh/OrbUCw2e4SLYgCWJEmSJC1OWyec83b4kzvggqth7BBc/2b49HPgF38NkxPNHuG8DMCSJEmSpKXJt8HZb4TLbofXfAFyBfjWJfCpZ8GOv4KJ0WaPsC4DsCRJkiTp8cnl4azXwqU/ggu/Dl3r4O/eBVc+A378GRg73OwRVjEAS5IkSZKOTC4Hp50Pb/8+vPGbsO5k+IcPwBVnwT//DxgZavYIAQOwJEmSJGm5RMApL4a33Qhv+y5seTr845/CFWfCLR+Hw48u3EcDGYAlSZIkScvvCc+DN30T3n5L6XuDf/AX8Mkz4eYPw4E9TRmSAViSJEmS1DjHPxMu/Cr88a3wlJfCrZ+CK58GN74f9j9wVIdSOKqPJklaEe4dOMj4ZJGUSt9fn0jT32NfeT8BKaXyLZS2ULGven+itCPN0w/lbqb3zfE4aZ5+pruZpx9mHVd9f6qD+s9j5nFmP37168XUY871ODWvT227mddj7n6mH2OOfqY6mXod2grBxp5ONvd1sKmvg029nWzq7WB9Twf5XCz1PxdJkpbH5tPhtV+Af/Of4UefhB1fLP08/UJ4/ntg/ZMaPgQDsCRl0Bs//xMeGhxp9jAyIwICiIjyLQSljVHVJma1pfJ+nX4oH1NuSgSMThTZf3h81jhyAet7OkrBuByKN/V2sLGv9Pvm8u2Gng7aC14kJklqkPVPggs+Bb/3n+Bfr4Kffgl+/jU449XwgveWgnKDGIAlKYM+9qozGRkvzgpRs0JaTVCbDmXMDmzUC3DBnI9RPqSqn1n9zuqzuh9qx1vbT519tWOdGUedcDpHcK0d68zvta9f8862jk0UGTg4yt6hEfYeqLwdZe+BEfYMjfDLBwfZd3C06mz2lHVr2kvhuLcUlkuhuYNNfVPBuZNNfR10tuWP/pOTJLWGtSfA+f8NXvC+0mXRt38B7voGnPayUjje8rRlf0gDsDIhpcTIeJHxYpHutjyFvGc2lG0vOm1zs4egBmsv5Dh+bRfHr+2at93EZJFHDo1NB+PqkDzKwIERdu49yMCBUSaKs5NyX2ehIhRXBOSabT0d/i+HJGkOvZvh3D+D578bfvLZ0s8pLzEASxOTRQaHx9k/PM7g8DiDh0u3+w+PMTg8wf7hMQaHxxkaHmf/1L5y27GJ4nQ/HYUcazoKdLfnWdNeoLujfNuen9m+xP3t+VxTz/ZI0uNRyOfY3NfJ5r5OoH/OdsVi4rHDY+w9MMqe8tnkgcozywdG2fH/HmPvgdGqejuluz1f9wzypqozzJ30dRWspZKUVd3rSp8Pfu5lUOhoyEMYgHXUpZQ4MDoxHV4Hq8LqWE2wnWkzODzOwdGJefvu6SjQ39U2/XPKpp7S792l++35HIfHJjk0NsHh0ZrbsUn2HRzl8Ngkh8cmODQ6yfD45KKfVyEX8wTkAmva83S3F1jTUXPbnp9zf1db3v8RlLQi5HLB+p7SQlpP3dI3Z7uUEkPDEzNnk8tnkivPMN+9e4hbhvZyaGx2jW0v5GbOHJdD8ua+zvKl2DPb1nW3k3NBL0lqTZ1z/ztzpAzAetxGxidnhdTSmdh6wXbqrOwYQyMTTNa5jG5Kez5Hf3cba8sh9ri1nZy2pZe1Xe30d7Wxtnsm4E4F27VdbfR1tdG2zJc2TxYTw+OTHB6d4NDYJIdGJ+oE6NK+qdB8eKx8v7x9z4ERDu+rPmaep18lArrb5g7ICwXo0v6KM9gdeS8Bl9RQEVGqzd1tnLq5d962B0cnqs4g7x0aKZ1VLp9l3jlwkH+9dx9DI7Pf/CzkYjoUb5zjbPKmvg7Wr2m35kmSphmAM25issjQyMT84XU64I5V3R+tc4nblAimQ+pUOD1xXTf9XYXpIFsZXkuBt7S9s23lXEqczwU9HYVl/exaSonRiWJVmJ4OznUCdL2Avf/wGA/urw7m9T6bNxcvAZe0EvR0FOjZ2MMTN/bM225kfLIcjKfOJleE5gOj7HrsMD+7/zEePTQ269hcwLo1HeVVrqsvvd5Y9XsHHQUX9JKkVmcAbgEpJQ6OTkyH06Hh6s++zhVgBw+Pc2CBS4rXtOfLYbWd/q4CT9zQM30Wtq/ibGzl2dm+rjZ6OwpemjaHiKCzLU9nW571y9jv2ERxUeF5JmRXX/59aHSCfQdHq7aPjM/9JkctLwGX1CidbXlOWNfNCeu65203NlFk38HRmlWvK8PyCHfvHmLfwdG6V+Ic0902HZA31jmbPHWGuavdoCxJq5UBeAUZnZic+exrOaDOLPY0NivUDlVcWjzf2b+2fNDfVQqwa7vb2dzXyVM291YH2OnLiiuCbGeb3wO5irQXcrQX2lk7//8fLslkMXG4IiAfnusy8Hn2Pzw0Mmt7va9cqWepl4C/+plbOW6BFW8lta72Qo7j1nYtWAcmi4lHDlV8LnloJiDvHRplz4FR7t17kIGDo4xPzi5YvR2FciCuDsa123o6XNBLklYaA/AymywmDozMXoF4OsDO2j6z+NN8Z9sioK+zOqxuPaar6v7arvaqUDu1z7NoerzyuaC3s43ezrZl63PqK6mqA/QCl4HX7J/rEvDfPWWDAVjSgvK5KAXV3oVXvt4/PD4TjCtXvy5vu+P+/ewZGqn7saCutvxMQK5c/bq3YyYs93awtrvNf6cl6SgxANeRUuLw2GTFWdixul+rM1hn5eJ6C3VU6p66pLj884T13RVnYcsBtqttVrDt7fSSYrWGiKCrPV+6hHD+j/0tydhEkbx/I5KWUS4XrFvTzro17Zx27NztUkoMjUwwUHM2ec/QzOXYv949xA8OjNb9NoP2Qo6NPR3VZ5N7y6tfV2xbv8aVryXpSBmAK0wWE8/++PfYf3j+S4oLuZj5DGxXGxt7Ojh1U29VsJ3r0mIvKZYaw78tSc0SERVfvzf/yteHxyaqziZPheWBcli+b+AQP77vUQaHx2cdm88FG3ra+dgrz+Ilp29u1NORpJZmAK6QzwUve9pxdLXnq87C9k0t8lT+ap7udi8pliRJS9fdXuCkDQVO2rBm3nYzK1+PMjB9Nrl0hnlzX+dRGq0ktR4DcI2PvOKMZg9BkiRl3GJXvpYkLY3XDEqSJEmSMsEALEmSJEnKBAOwJEmSJCkTDMCSJEmSpEwwAEuSJEmSMsFVoCVJktSSPvitX7Jz70HyuSCfCwrl25mfHIVckIvyvnyQj5i7fZTaVB1T2U/5mFzlsTHT7+zHn3qcHPkcs/uZZ1x+Jaf0+BiAJUmS1JJy5ZA4NlFkopgopsTEZPm2mJis+SltK85sSzPbU2ryk6mRC6qDeS4o5HM1wTxmB+p89TGLfXMgl5vd79F+c6D2MTracvR2FHwzQEtiAJYkSVJL+rNXnrlsfRUrAvF0OJ5MM8G6mChWhWiYKBYplm+rwvZUAJ+s6bNY009KTE4W53iMeuF9ZlyV4b10TGlMk8XqNwPGJ4sMj8/ub7L2cab6myxSTNXPrdjENwfWtOc5tr+T49Z2saW/ky39XRy3tvp2TYeRRzMa+l9DRJwHXAnkgc+nlD5Rs/9E4EvA2nKbD6SUboyI3wc+AbQDY8D7U0rfLx/zT8AWYLjczbkppb2NfB6SNB9rnaQsyHqty+WCHEFbvtkjWXlSZdheINzPDtnF6TA+35sB020q3kAYGZvkocERHhocZvfgCPc8PMDAwdFZZ+v7OgszAXltF8eVg/KWtZ0c19/Fsf2ddDqxmdGwABwReeBq4PeBXcDtEXFDSulXFc0+BFyfUvpMRJwO3AicBOwDXp5S2h0RZwI3AcdXHPeGlNKORo1dkhbLWicpC6x1mk+UL60urIAMOTZRZM/QyEww3l99+4tdgzx6aGzWcevXtLNl6sxxOShvqTizvLmvk7a86we3gkaeAT4H2JlSug8gIq4DLgAqC2UC+sq/9wO7AVJKd1S0uRvojIiOlNJoA8crSY+HtU5SFljrtCq0F3KcsK6bE9Z1z9lmZLx85nh/6czx9O3gMPc/cpgf3/cIB0Ymqo6JgE29HVWXVm+pOZO8sbeDfM7PI690jQzAxwMPVNzfBTy7ps1HgJsj4h3AGuAldfp5DXBHTZH8q4iYBP4G+POUZi9LEBGXAJcAnHjiiY/3OUjSQqx1krLAWqeW0dmW5+QNazh5w5o52xwcneDhOmeQHxoc4Z6HD/BP9wxweGyy6phCLtjc11lzqfXU76WgvH5Nu4t2NVkjA3C9ma0taBcB21NKfxkRzwW+EhFnppSKABFxBvAXwLkVx7whpfRgRPRSKpRvAr4864FSuga4BmDbtm0rbN0+SS3EWicpC6x1ypSejgKnbOrllE29dfenlBganmD34HB1QN4/wu7BYe7ctZ+b7h5hbKJYdVx7IVc+c9w5HYqrFu7q76Kvy5WtG6mRAXgXcELF/a2UL4WpcDFwHkBK6daI6AQ2AHsjYivwLeDNKaV7pw5IKT1Yvj0QEV+jdEnOrEIpSUeJtU5SFljrpAoRQX93G/3dbTx1S1/dNiklHjk0Nh2KH9pfOoM8ddn1T377KA8PjTBZs4x2d3u+6vPHsy67XttFjytbP26NfOVuB06NiJOBB4ELgdfXtLkfeDGwPSKeCnQCAxGxFvh74PKU0r9MNY6IArA2pbQvItqAlwHfa+BzkKSFWOskZYG1TlqiiGBDTwcbejo4a2t/3TaTxcTAgdFyQJ65zHrqrPJv9gyw98Dsla17OwvVZ5ArL7suB2dXtq6vYQE4pTQREZdRWukvD3wxpXR3RHwU2JFSugF4L3BtRLyb0mU0b00ppfJxpwAfjogPl7s8FzgE3FQuknlKRfLaRj0HSVqItU5SFljrpMbI54Jj+zs5tr8T5vh4+/jkzMrWu8tnkSsX7vrlrkEeqbOy9bo17XN+N/KW8mNmcWXrqLPOQMvZtm1b2rHD1fUlzYiIn6aUtjV7HMvJWiepnlard9Y6abaR8UkeHhypOpM8dan1VHAeqrOy9caejqrvRp4OyKtwZevF1jovHpckSZKkVayzLc9JG9Zw0jwrWx8anaj73cgPDY7wmz0H+MFvZq9snc8Fm3s7Zn0vcmVY3tCzula2NgBLkiRJUotbs5iVrUcmqlazrry968FBbv7VntkrW+dzHDu1svXarpqvgSoF5f6uthUTkg3AkiRJkpRxEUF/Vxv9XW2cduzcK1s/emis6vPIlZdd3/bbR9kzNMJEzcrWXW356cuqt9Qs2HXcUV7Z2gAsSZIkSVpQRLC+p4P1PR2cefzcK1vvOzg6E5D3V69s/cP/u7iVrV/+9C0870kblv05GIAlSZIkScsinws293Wyua+Ts+doU29l64crfr/rwUHOOK7PACxJkiRJWt3a8jm2HtPN1mO652zTqG8ryt4XP0mSJEmSVrRGLZplAJYkSZIkZYIBWJIkSZKUCQZgSZIkSVImGIAlSZIkSZlgAJYkSZIkZYIBWJIkSZKUCQZgSZIkSVImGIAlSZIkSZlgAJYkSZIkZYIBWJIkSZKUCQZgSZIkSVImGIAlSZIkSZlgAJYkSZIkZYIBWJIkSZKUCQZgSZIkSVImGIAlSZIkSZlgAJYkSZIkZYIBWJIkSZKUCQZgSZIkSVImGIAlSZIkSZlgAJYkSZIkZYIBWJIkSZKUCQZgSZIkSVImGIAlSZIkSZnQ0AAcEedFxD0RsTMiPlBn/4kRcUtE3BERd0bE+RX7Li8fd09E/MFi+5Sko81aJykLrHWSWkHDAnBE5IGrgZcCpwMXRcTpNc0+BFyfUjobuBD4dPnY08v3zwDOAz4dEflF9ilJR421TlIWWOsktYpGngE+B9iZUrovpTQGXAdcUNMmAX3l3/uB3eXfLwCuSymNppR+C+ws97eYPiXpaLLWScoCa52kltDIAHw88EDF/V3lbZU+ArwxInYBNwLvWODYxfQpSUeTtU5SFljrJLWEQgP7jjrbUs39i4DtKaW/jIjnAl+JiDPnObZeYK/ts/TgEZcAl5TvjkTE3XWa9QODdbZvAPbV67dJ5hpns/pcyrGLbbtQu/n2z7Wv3nbndnmPXUz7lTq3T1hC2/lY65bPav57sNbNbzXP7WLbH625hebUu9VQ68C/h0Yfa62b32qe28W2X/21LqXUkB/gucBNFfcvBy6vaXM3cELF/fuATbVtgZvK/S3Y5xxjuWaJ23c06nV5nK9l3XE2q8+lHLvYtgu1m2//UubXuV3eYxfTvtXn1lq3rK/lqv17sNa17twutv3Rmttmze9qqHVz7fPvYfmOtda17twutn0r1LpGXgJ9O3BqRJwcEe2UFj+4oabN/cCLASLiqUAnMFBud2FEdETEycCpwG2L7LOe7yxx+0rTiHEeSZ9LOXaxbRdqN9/+1Ty/q3luF9u+1efWWrd8VvPfg7Vufqt5bhfbvtXndjXUuoX2rRSr+e/BWje/1Ty3i22/6uc2yum6MZ2Xlr+/AsgDX0wpfSwiPkopzd9QXunvWqCH0iUv/zGldHP52A8C/w6YAN6VUvruXH02YNw7UkrblrtfNZ9z27qaObfWOq00zm1ra9b8Wuu00ji3ra1R89vQALxaRcQlKaVrmj0OLT/ntnU5t0vna9a6nNvW5vwuja9X63JuW1uj5tcALEmSJEnKhEZ+BliSJEmSpBXDACxJkiRJygQDsCRJkiQpEwzASxQRr4yIayPi2xFxbrPHo+UTEU+MiC9ExDeaPRYduYhYExFfKv+9vqHZ41ltrHWty1rXWqx1R8Za17qsda1lOWtdpgJwRHwxIvZGxF0128+LiHsiYmdEfGC+PlJKf5tSejvwVuAPGzhcLcEyze19KaWLGztSHYklzvOrgW+U/15fcdQH20TWutZlrcsGa93iWOtal7UuG5pV6zIVgIHtwHmVGyIiD1wNvBQ4HbgoIk6PiLMi4u9qfjZVHPqh8nFaGbazfHOrlWs7i5xnYCvwQLnZ5FEc40qwHWtdq9qOtS4LtmOtW4ztWOta1XasdVmwnSbUusKRHLzapJR+GBEn1Ww+B9iZUroPICKuAy5IKf0X4GW1fUREAJ8AvptS+lljR6zFWo651cq3lHkGdlEqlj8nY2/2Wetal7UuG6x1i2Ota13WumxoVq3LVKGcw/HMvJsApRf3+HnavwN4CfDaiLi0kQPTEVvS3EbE+oj4LHB2RFze6MFp2cw1z98EXhMRnwG+04yBrTDWutZlrcsGa93iWOtal7UuGxpe6zJ1BngOUWdbmqtxSukq4KrGDUfLaKlz+wjgP36rT915TikdAt52tAezglnrWpe1LhusdYtjrWtd1rpsaHit8wxw6V2FEyrubwV2N2ksWl7ObTY4z4vj69S6nNtscJ4Xx9epdTm32dDweTYAw+3AqRFxckS0AxcCNzR5TFoezm02OM+L4+vUupzbbHCeF8fXqXU5t9nQ8HnOVACOiK8DtwJPiYhdEXFxSmkCuAy4Cfg1cH1K6e5mjlNL59xmg/O8OL5Orcu5zQbneXF8nVqXc5sNzZrnSGnOS+clSZIkSWoZmToDLEmSJEnKLgOwJEmSJCkTDMCSJEmSpEwwAEuSJEmSMsEALEmSJEnKBAOwJEmSJCkTDMDKlIh4YUT8LCImIuK1zR6PJDWCtU5SFljr9HgYgJU19wNvBb7W5HFIUiNZ6yRlgbVOS1Zo9gCkRoqINwPvAxJwZ0rpTeXtxaYOTJKWkbVOUhZY67QcDMBqWRFxBvBB4HdTSvsiYl2zxyRJy81aJykLrHVaLl4CrVb2IuAbKaV9ACmlR5s8HklqBGudpCyw1mlZGIDVyoLSJTKS1MqsdZKywFqnZWEAViv7R+B1EbEewEtlJLUoa52kLLDWaVlESr6RotYVEW8B3g9MAncAVwPfAo4BRoCHU0pnNG+EknTkrHWSssBap+VgAJYkSZIkZYKXQEuSJEmSMsEALEmSJEnKBAOwJEmSJCkTDMCSJEmSpEwwAEuSJEmSMsEALEmSJEnKBAOwJEmSJCkTDMCSJEmSpEz4/yrzs9aGsDPkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1152x432 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting CV results\n",
    "# for each value of c2, make a plot of c1 versus train and test f1-score\n",
    "\n",
    "plt.figure(figsize=(16,6))\n",
    "\n",
    "for i, val in enumerate(params_space['c2']):\n",
    "   \n",
    "    # subplot 1/3/i\n",
    "    plt.subplot(1, 3, i+1)\n",
    "    c2_subset = cv_results[cv_results['param_c2']==val]\n",
    "\n",
    "    plt.plot(c2_subset[\"param_c1\"], c2_subset[\"mean_test_score\"])\n",
    "    plt.plot(c2_subset[\"param_c1\"], c2_subset[\"mean_train_score\"])\n",
    "    plt.xlabel('c1')\n",
    "    plt.ylabel('Mean F-score')\n",
    "    plt.title(\"c2={0}\".format(val))\n",
    "    plt.ylim([0.80, 1])\n",
    "    plt.legend(['test score', 'train score'], loc='upper left')\n",
    "    plt.xscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CRF(algorithm='lbfgs', all_possible_states=None, all_possible_transitions=True,\n",
       "    averaging=None, c=None, c1=0.1, c2=0.1, calibration_candidates=None,\n",
       "    calibration_eta=None, calibration_max_trials=None, calibration_rate=None,\n",
       "    calibration_samples=None, delta=None, epsilon=None, error_sensitive=None,\n",
       "    gamma=None, keep_tempfiles=None, linesearch=None, max_iterations=100,\n",
       "    max_linesearch=None, min_freq=None, model_filename=None, num_memories=None,\n",
       "    pa_type=None, period=None, trainer_cls=None, variance=None, verbose=False)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# building a model with optimal hyperparams\n",
    "crf = sklearn_crfsuite.CRF(\n",
    "    algorithm='lbfgs',\n",
    "    c1=0.1,\n",
    "    c2=0.1,\n",
    "    max_iterations=100,\n",
    "    all_possible_transitions=True\n",
    ")\n",
    "crf.fit(X_train_features, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       B-tim      0.934     0.868     0.900      6109\n",
      "       B-org      0.793     0.735     0.763      5896\n",
      "       I-org      0.817     0.800     0.808      4988\n",
      "       B-geo      0.856     0.907     0.881     11488\n",
      "       B-per      0.853     0.825     0.839      5141\n",
      "       I-per      0.852     0.906     0.878      5201\n",
      "       B-gpe      0.975     0.921     0.948      4787\n",
      "       I-tim      0.838     0.736     0.783      2031\n",
      "       I-geo      0.832     0.798     0.815      2236\n",
      "       I-gpe      0.868     0.524     0.653        63\n",
      "       B-eve      0.500     0.300     0.375        90\n",
      "       B-art      0.406     0.104     0.166       125\n",
      "       I-art      0.074     0.022     0.034        90\n",
      "       B-nat      0.667     0.243     0.356        74\n",
      "       I-nat      0.333     0.100     0.154        20\n",
      "       I-eve      0.383     0.228     0.286        79\n",
      "\n",
      "   micro avg      0.861     0.843     0.852     48418\n",
      "   macro avg      0.686     0.564     0.602     48418\n",
      "weighted avg      0.859     0.843     0.849     48418\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# make predictions \n",
    "y_pred = crf.predict(X_test_features)\n",
    "print(metrics.flat_classification_report(\n",
    "    y_test, y_pred, labels=labels, digits=3\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}