Demonstration of PreProcessing System.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demonstration of PreProcessing System"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PreProcessing import PreProcessing\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pyconll\n",
    "import os\n",
    "import Utility\n",
    "PATH = \"DATA\"\n",
    "conllu_files = pd.Series(os.listdir(PATH))\n",
    "conllu_files = conllu_files[conllu_files.str.contains('.conllu')].values.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initialization of PreProcessing class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pp = PreProcessing()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A - Sentence Splitters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading the UD corpuses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19276 sentences are loaded.\n",
      "Wall time: 3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "sentences = []\n",
    "for file in conllu_files: #[0:2]\n",
    "    data = pyconll.load_from_file(PATH + \"/\" + file)\n",
    "    for sentence in data:\n",
    "        s = sentence.text\n",
    "        \n",
    "        # many of the sentences in the corpus are not in fact sentences, ending with comma, or no punctuation at all\n",
    "        # so we are turning them into sentences by adding ellipsis\n",
    "        \n",
    "        if s[-1] not in pp.MLBasedSentenceSplitter.sentence_ending_punctuations:\n",
    "            s += '...'\n",
    "        sentences.append(s)\n",
    "        \n",
    "print(len(sentences), 'sentences are loaded.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test strings for sentence splitters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = 'Bugun canım çok sıkıldı. Yarın da sıkılacak, yarın da... Hep...'\n",
    "string2 = 'Sayın Av. Meryem Beşer dedi ki... Sonra da dedi ki, \"dava düştü konu kapandı\".'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Machine Learning Based Sentence Splitter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training and Testing using our own implementation of Naive Bayes Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 50min 29s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Score</th>\n",
       "      <td>0.875256</td>\n",
       "      <td>0.103761</td>\n",
       "      <td>0.300436</td>\n",
       "      <td>0.15425</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Accuracy    Recall  Precision       F1\n",
       "Score  0.875256  0.103761   0.300436  0.15425"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "pp.MLBasedSentenceSplitter.fit(sentences, model = 'NBC')\n",
    "y = pp.MLBasedSentenceSplitter.create_labels(sentences)\n",
    "y_hat = pp.MLBasedSentenceSplitter.predict(' '.join(sentences), model = 'NBC')\n",
    "pp.all_metrics_together(y, y_hat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training and Testing using scikit-learn's vanilla Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 51min 11s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Score</th>\n",
       "      <td>0.893389</td>\n",
       "      <td>0.061634</td>\n",
       "      <td>0.643902</td>\n",
       "      <td>0.1125</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Accuracy    Recall  Precision      F1\n",
       "Score  0.893389  0.061634   0.643902  0.1125"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "pp.MLBasedSentenceSplitter.fit(sentences, model = 'LogisticRegression')\n",
    "y = pp.MLBasedSentenceSplitter.create_labels(sentences)\n",
    "y_hat = pp.MLBasedSentenceSplitter.predict(' '.join(sentences), model = 'LogisticRegression')\n",
    "pp.all_metrics_together(y, y_hat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Rule Based Sentence Splitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision</th>\n",
       "      <th>F1 Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Score</th>\n",
       "      <td>0.916042</td>\n",
       "      <td>0.886647</td>\n",
       "      <td>0.901104</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Recall  Precision  F1 Score\n",
       "Score  0.916042   0.886647  0.901104"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "splitted_sentences = pp.RuleBasedSentenceSplitter.split_text_into_sentences(' '.join(sentences))\n",
    "list_for_recall = [1 if sentence in sentences else 0 for sentence in splitted_sentences]\n",
    "list_for_precision = [1 if sentence in splitted_sentences else 0 for sentence in sentences]\n",
    "recall = np.mean(list_for_recall)\n",
    "precision = np.mean(list_for_precision)\n",
    "f1_score = 2 / (recall ** -1 + precision ** -1)\n",
    "df_rulebasedresults = pd.DataFrame([recall, precision, f1_score], columns = ['Score'], index = ['Recall', 'Precision', 'F1 Score']).T\n",
    "df_rulebasedresults"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visual Inspection of Sentence Splitting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing on the first string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = 'Bugun canım çok sıkıldı. Yarın da sıkılacak, yarın da... Hep...'\n",
    "string2 = 'Sayın Av. Meryem Beşer dedi ki... Sonra da dedi ki, \"dava düştü konu kapandı\".'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Bayes result: ['Bugun canım çok sıkıldı.', 'Yarın da sıkılacak, yarın da...', 'Hep...']\n",
      "LogisticRegression result: ['Bugun canım çok sıkıldı. Yarın da sıkılacak, yarın da... Hep...']\n",
      "Rule Based result: ['Bugun canım çok sıkıldı.', 'Yarın da sıkılacak, yarın da...', 'Hep...']\n"
     ]
    }
   ],
   "source": [
    "print('Naive Bayes result:', pp.MLBasedSentenceSplitter.split_text_into_sentences(string, model = 'NBC'))\n",
    "print('LogisticRegression result:', pp.MLBasedSentenceSplitter.split_text_into_sentences(string, model = 'LogisticRegression'))\n",
    "print('Rule Based result:', pp.RuleBasedSentenceSplitter.split_text_into_sentences(string))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing on the second string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Bayes result: ['Sayın', 'Av.', 'Meryem', 'Beşer dedi ki...', 'Sonra da dedi ki,', '\"dava düştü konu kapandı\".']\n",
      "LogisticRegression result: ['Sayın Av. Meryem Beşer dedi ki... Sonra da dedi ki, \"dava düştü konu kapandı\".']\n",
      "Rule Based result: ['Sayın Av. Meryem Beşer dedi ki...', 'Sonra da dedi ki, \"dava düştü konu kapandı\".']\n"
     ]
    }
   ],
   "source": [
    "print('Naive Bayes result:', pp.MLBasedSentenceSplitter.split_text_into_sentences(string2, model = 'NBC'))\n",
    "print('LogisticRegression result:', pp.MLBasedSentenceSplitter.split_text_into_sentences(string2, model = 'LogisticRegression'))\n",
    "print('Rule Based result:', pp.RuleBasedSentenceSplitter.split_text_into_sentences(string2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### B - Tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test strings for tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "test_sentence1 = \"merhaba, name@gmail.com <html>!! www.abc.com #hello selam# nasılsınız:  milli eğitim bakanlığı Av. Hatice Hanım 2.01.1997'de 20:02'de aradı\"\n",
    "test_sentence2 = 'boğaziçi üniversitesi öğrencileri dönemi uzaktan geçirdi.'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Rule Based Tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Testing the rule based Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['merhaba', ',', 'name@gmail.com', '<html>', '!!', 'www.abc.com', '#hello', 'selam', '#', 'nasılsınız', ':', 'milli eğitim bakanlığı', 'Av.', 'Hatice', 'Hanım', \"2.01.1997'de\", \"20:02'de\", 'aradı']\n",
      "['boğaziçi üniversitesi', 'öğrencileri', 'dönemi', 'uzaktan', 'geçirdi', '.']\n"
     ]
    }
   ],
   "source": [
    "# Rule Based Tokenizer Test:\n",
    "list_of_tokens = pp.RuleBasedTokenizer.tokenize(test_sentence1)\n",
    "print(list_of_tokens)\n",
    "list_of_tokens2 = pp.RuleBasedTokenizer.tokenize(test_sentence2)\n",
    "print(list_of_tokens2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Machine Learning Based Tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing the ML based Tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training and Testing using our own implementation of Naive Bayes Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 13.2 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Score</th>\n",
       "      <td>0.86805</td>\n",
       "      <td>0.21142</td>\n",
       "      <td>0.80479</td>\n",
       "      <td>0.334869</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Accuracy   Recall  Precision        F1\n",
       "Score   0.86805  0.21142    0.80479  0.334869"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tokenization_corpus = Utility.load_file(PATH + \"/\" + \"handmade_turkish_blog_corpus.txt\")\n",
    "\n",
    "pp.MLBasedTokenizer.fit(tokenization_corpus, model = 'NBC')\n",
    "y = pp.MLBasedTokenizer.create_labels(tokenization_corpus)\n",
    "y_hat = pp.MLBasedTokenizer.predict(tokenization_corpus)\n",
    "pp.all_metrics_together(y, y_hat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training and Testing using scikit-learn's vanilla Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1.02 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Score</th>\n",
       "      <td>0.996471</td>\n",
       "      <td>0.99682</td>\n",
       "      <td>0.981024</td>\n",
       "      <td>0.988859</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Accuracy   Recall  Precision        F1\n",
       "Score  0.996471  0.99682   0.981024  0.988859"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "pp.MLBasedTokenizer.fit(tokenization_corpus, model = 'LogisticRegression')\n",
    "y = pp.MLBasedTokenizer.create_labels(tokenization_corpus)\n",
    "y_hat = pp.MLBasedTokenizer.predict(tokenization_corpus, model = 'LogisticRegression')\n",
    "pp.all_metrics_together(y, y_hat)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original sentence is: merhaba! Nasılsınız? Ben çok iyiyim.\n",
      "Naive Bayes result: ['merhaba', '! Nasılsınız', '? Ben çok iyiyim', '.']\n",
      "LogisticRegression result: ['merhaba', '!', ' Nasılsınız', '?', ' Ben', ' çok', ' iyiyim', '.']\n",
      "Rule Based result: ['merhaba', '!', 'Nasılsınız', '?', 'Ben', 'çok', 'iyiyim', '.']\n",
      "\n",
      "\n",
      "Original sentence is: merhaba. nasılsınız? Ben 22.12.1996 gününde !Eskişehir'den 20:02 treniyle (buraya) geldim.\n",
      "Naive Bayes result: ['merhaba', '.', ' nasılsınız', '? Ben ', '2', '2', '.', '12', '.', '19', '9', '6', 'g', 'ününde ', \"!Eskişehir'den \", '2', '0', ':', '02', 't', 'reniyle ', '(buraya', ') geldim', '.']\n",
      "LogisticRegression result: ['merhaba', '.', ' nasılsınız', '?', ' Ben', ' 22.12.1996', ' gününde', '!', \"Eskişehir'den\", ' 20:02', ' treniyle', '(', 'buraya', ')', ' geldim', '.']\n",
      "Rule Based result: ['merhaba', '.', 'nasılsınız', '?', 'Ben', '22.12.1996', 'gününde', '!', \"Eskişehir'den\", '20:02', 'treniyle', '(', 'buraya', ')', 'geldim', '.']\n"
     ]
    }
   ],
   "source": [
    "tokenizer_test1 = \"merhaba! Nasılsınız? Ben çok iyiyim.\"\n",
    "\n",
    "print('Original sentence is:', tokenizer_test1)\n",
    "print('Naive Bayes result:', pp.MLBasedTokenizer.split_to_tokens(tokenizer_test1, model = 'NBC'))\n",
    "print('LogisticRegression result:', pp.MLBasedTokenizer.split_to_tokens(tokenizer_test1, model = 'LogisticRegression'))\n",
    "print('Rule Based result:', pp.RuleBasedTokenizer.tokenize(tokenizer_test1))\n",
    "print(\"\\n\")\n",
    "tokenizer_test2 = \"merhaba. nasılsınız? Ben 22.12.1996 gününde !Eskişehir'den 20:02 treniyle (buraya) geldim.\"\n",
    "print('Original sentence is:', tokenizer_test2)\n",
    "print('Naive Bayes result:', pp.MLBasedTokenizer.split_to_tokens(tokenizer_test2, model = 'NBC'))\n",
    "print('LogisticRegression result:', pp.MLBasedTokenizer.split_to_tokens(tokenizer_test2, model = 'LogisticRegression'))\n",
    "print('Rule Based result:', pp.RuleBasedTokenizer.tokenize(tokenizer_test2))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### C - Normalizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- removes punctuations\n",
    "- converts letters to lower case\n",
    "- does not alter the token further if it is in MWE lexicon\n",
    "- converts numbers to text form\n",
    "- removes accent marks\n",
    "- corrects typos (using Lexicon + Levenshtein distance (calculated for both consonants and whole word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['benim', 'kedimin', 'suratı', 'gri', 'gözleri', 'ise', 'sarı', 'renkte']\n",
      "['kah', 'güldük', 'kah', 'üç', 'kez', 'yere', 'düştük', 've', 'sırada']\n",
      "türk hava yolları\n"
     ]
    }
   ],
   "source": [
    "test_list_for_normalizer = ['Benim', 'kedımin', 'surati', 'gri', 'gozleri', 'ise', 'sarı', 'renkkte...']\n",
    "test_list2_for_normalizer = ['Kâh', 'gulduk', 'kôh', '3', 'kez', 'yere', 'düştuk...!!!', 've', 'sırada...']\n",
    "test_list3_for_normalizer = ['Türk Hava Yolları!!!']\n",
    "normalized_tokens = pp.Normalizer.normalize(test_list_for_normalizer)\n",
    "normalized_tokens_2 = pp.Normalizer.normalize(test_list2_for_normalizer)\n",
    "normalized_tokens_3 = pp.Normalizer.normalize(test_list3_for_normalizer)\n",
    "print(normalized_tokens)\n",
    "print(normalized_tokens_2)\n",
    "print(normalized_tokens_3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### D - Stemmer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test words for stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "test_word1 = 'kitapçılardan'\n",
    "test_word2 = 'biliyorlardı'\n",
    "test_word3 = 'nasılsınız'\n",
    "test_word4 = 'gözlemek'\n",
    "test_word5 = 'yapamayacaklarmış'\n",
    "test_word6 = 'kalelerimizdekilerden'\n",
    "test_word7 = 'kulaklıklarım'\n",
    "test_word8 = 'plakçıların'\n",
    "test_word9 = 'kodlama'\n",
    "test_word10 = 'mutluluktan'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Stemmer Test:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kitapçılar\n",
      "kitapçı\n",
      "kitap\n",
      "kitapçılarda\n",
      "kitap\n",
      "is noun\n",
      "biliyorlard\n",
      "biliyorlar\n",
      "biliyor\n",
      "bili\n",
      "bil\n",
      "biliyorlar\n",
      "biliyor\n",
      "is verb\n",
      "nasılsı\n",
      "nasılsı\n",
      "nasıl\n",
      "nasılsı\n",
      "is verb\n",
      "gözle\n",
      "göz\n",
      "gözleme\n",
      "is verb\n",
      "yapamayacaklar\n",
      "yapamayacak\n",
      "yapamay\n",
      "yapa\n",
      "yap\n",
      "is verb\n",
      "kalelerimizdekiler\n",
      "kalelerimizdeki\n",
      "kalelerimizde\n",
      "kalelerimiz\n",
      "kaleleri\n",
      "kale\n",
      "kalelerimizdekilerde\n",
      "kale\n",
      "is noun\n",
      "kulaklıkları\n",
      "kulaklık\n",
      "kulak\n",
      "kulaklıklar\n",
      "kulaklıkl\n",
      "kulaklıkları\n",
      "kulak\n",
      "is noun\n",
      "plakçılar\n",
      "plakçı\n",
      "plak\n",
      "plakçıları\n",
      "plak\n",
      "is noun\n",
      "kodla\n",
      "kodla\n",
      "kod\n",
      "is verb\n",
      "mutluluk\n",
      "mutlu\n",
      "mutlulukta\n",
      "mutlu\n",
      "is noun\n",
      "Test word 1 is: kitapçılardan Stem is: kitap\n",
      "Test word 2 is: biliyorlardı Stem is: bil\n",
      "Test word 3 is: nasılsınız Stem is: nasıl\n",
      "Test word 4 is: gözlemek Stem is: göz\n",
      "Test word 5 is: yapamayacaklarmış Stem is: yap\n",
      "Test word 6 is: kalelerimizdekilerden Stem is: kale\n",
      "Test word 7 is: kulaklıklarım Stem is: kulak\n",
      "Test word 8 is: plakçıların Stem is: plak\n",
      "Test word 9 is: kodlama Stem is: kod\n",
      "Test word 10 is: mutluluktan Stem is: mutlu\n"
     ]
    }
   ],
   "source": [
    "# Stemmer test:\n",
    "stem1 = pp.Stemmer.stemmer(test_word1)\n",
    "stem2 = pp.Stemmer.stemmer(test_word2)\n",
    "stem3 = pp.Stemmer.stemmer(test_word3)\n",
    "stem4 = pp.Stemmer.stemmer(test_word4)\n",
    "stem5 = pp.Stemmer.stemmer(test_word5)\n",
    "stem6 = pp.Stemmer.stemmer(test_word6)\n",
    "stem7 = pp.Stemmer.stemmer(test_word7)\n",
    "stem8 = pp.Stemmer.stemmer(test_word8)\n",
    "stem9 = pp.Stemmer.stemmer(test_word9)\n",
    "stem10 = pp.Stemmer.stemmer(test_word10)\n",
    "print('Test word 1 is:', test_word1, 'Stem is:', stem1)\n",
    "print('Test word 2 is:', test_word2, 'Stem is:', stem2)\n",
    "print('Test word 3 is:', test_word3, 'Stem is:', stem3)\n",
    "print('Test word 4 is:', test_word4, 'Stem is:', stem4)\n",
    "print('Test word 5 is:', test_word5, 'Stem is:', stem5)\n",
    "print('Test word 6 is:', test_word6, 'Stem is:', stem6)\n",
    "print('Test word 7 is:', test_word7, 'Stem is:', stem7)\n",
    "print('Test word 8 is:', test_word8, 'Stem is:', stem8)\n",
    "print('Test word 9 is:', test_word9, 'Stem is:', stem9)\n",
    "print('Test word 10 is:', test_word10, 'Stem is:', stem10)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Output of Tokenizer will be fed into stopword eliminator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### E - Stopword Eliminator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test strings for stopword removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "example_sent = \"Deneme deneme ama bu şu o bu bir cümle ve belki bu bir cümle\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Static Stopword Eliminator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Static stopword eliminator removes all stopwords in the sentence based on a predefined list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original sentence:  Deneme deneme ama bu şu o bu bir cümle ve belki bu bir cümle\n",
      "Filtered sentence:  ['Deneme', 'deneme', 'cümle', 'cümle']\n"
     ]
    }
   ],
   "source": [
    "# Static Stopword Test\n",
    "word_tokens, filtered_sentence = pp.StaticStopWordEliminator.remove_stopwords(example_sent)\n",
    "print('Original sentence: ', example_sent)\n",
    "print('Filtered sentence: ', filtered_sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dynamic Stopword Eliminator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dynamic Stopword eliminator takes list of tokens as input, and detects them using Detect() function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of tokenized and normalized tokens: 175953\n"
     ]
    }
   ],
   "source": [
    "# Re reading whole corpus\n",
    "sentences = []\n",
    "for file in conllu_files:\n",
    "    data = pyconll.load_from_file(PATH + \"/\" + file)\n",
    "    for sentence in data:\n",
    "        s = sentence.text\n",
    "        \n",
    "        # many of the sentences in the corpus are not in fact sentences, ending with comma, or no punctuation at all\n",
    "        # so we are turning them into sentences by adding ellipsis\n",
    "        \n",
    "        if s[-1] not in pp.MLBasedSentenceSplitter.sentence_ending_punctuations:\n",
    "            s += '...'\n",
    "        sentences.append(s)\n",
    "\n",
    "normalized_tokens = []\n",
    "for sentence in sentences:\n",
    "    tokenized_sentence = pp.RuleBasedTokenizer.tokenize(sentence)\n",
    "    for token in tokenized_sentence:\n",
    "        normalized_token = pp.Normalizer.remove_punctuations(token)\n",
    "        normalized_token = pp.Normalizer.convert_to_lower_case(normalized_token)\n",
    "        normalized_token = pp.Normalizer.remove_accent_marks(normalized_token)\n",
    "        normalized_tokens.append(normalized_token) if normalized_token != '' else None\n",
    "        \n",
    "print('Number of tokenized and normalized tokens:', len(normalized_tokens))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Given list of tokens, detect stop words function dynamically determines the stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Detected stop words for the given corpus are: ['bir', 've', 'bu', 'de', 'da', 'için', 'o', 'çok', 'daha', 'gibi', 'ne', 'her', 'sonra', 'kadar', 'en', 'ama', 'ile', 'olarak', 'ben', 'olan', 'ki', 'var', 'diye', 'dedi', 'büyük', 'iki', 'yeni', 'ya', 'değil', 'gün', 'zaman', 'şey', 'önce', 'hiç', 'iyi', 'bana', 'yok', 'ise', 'benim', 'içinde', 'olduğunu']\n"
     ]
    }
   ],
   "source": [
    "pp.DynamicStopWordEliminator.detect_stop_words(normalized_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Given list of tokens, drop stop words function drops the stop words and returns the dropped list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['NOT_STOP_WORD']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pp.DynamicStopWordEliminator.drop_stop_words(['NOT_STOP_WORD', 'bir', 've', 'bu', 'de', 'da', 'için', 'o', 'çok', 'daha', 'gibi', 'ne', 'her', 'sonra', 'kadar', 'en', 'ama', 'ile', 'olarak', 'ben', 'olan', 'ki', 'var', 'diye', 'dedi', 'büyük', 'iki', 'yeni', 'ya', 'değil', 'gün', 'zaman', 'şey', 'önce', 'hiç', 'iyi', 'bana', 'yok', 'ise', 'benim', 'içinde', 'olduğunu'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "unq, cnts = np.unique(normalized_tokens, return_counts = True)\n",
    "df_words = pd.DataFrame({'word': unq, 'counts': cnts}).sort_values(by = 'counts', ascending = False).reset_index(drop = True)\n",
    "argmax_second_der = df_words['counts'].pct_change().abs().pct_change().abs().dropna().idxmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bir</td>\n",
       "      <td>4791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ve</td>\n",
       "      <td>3062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bu</td>\n",
       "      <td>2224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>de</td>\n",
       "      <td>1417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>da</td>\n",
       "      <td>1294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>için</td>\n",
       "      <td>981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>o</td>\n",
       "      <td>886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>çok</td>\n",
       "      <td>840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>daha</td>\n",
       "      <td>743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>gibi</td>\n",
       "      <td>677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ne</td>\n",
       "      <td>604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>her</td>\n",
       "      <td>581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>sonra</td>\n",
       "      <td>578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>kadar</td>\n",
       "      <td>546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>en</td>\n",
       "      <td>543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ama</td>\n",
       "      <td>539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ile</td>\n",
       "      <td>512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>olarak</td>\n",
       "      <td>498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ben</td>\n",
       "      <td>467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>olan</td>\n",
       "      <td>452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ki</td>\n",
       "      <td>416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>var</td>\n",
       "      <td>408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>diye</td>\n",
       "      <td>399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>dedi</td>\n",
       "      <td>389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>büyük</td>\n",
       "      <td>353</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      word  counts\n",
       "0      bir    4791\n",
       "1       ve    3062\n",
       "2       bu    2224\n",
       "3       de    1417\n",
       "4       da    1294\n",
       "5     için     981\n",
       "6        o     886\n",
       "7      çok     840\n",
       "8     daha     743\n",
       "9     gibi     677\n",
       "10      ne     604\n",
       "11     her     581\n",
       "12   sonra     578\n",
       "13   kadar     546\n",
       "14      en     543\n",
       "15     ama     539\n",
       "16     ile     512\n",
       "17  olarak     498\n",
       "18     ben     467\n",
       "19    olan     452\n",
       "20      ki     416\n",
       "21     var     408\n",
       "22    diye     399\n",
       "23    dedi     389\n",
       "24   büyük     353"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_words.head(25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(50, 4000, 'Threshold selected according to Elbow Rule(argmax of 2nd derivative)')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+8AAAEICAYAAAAupMU2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABBfElEQVR4nO3dd5xU1f3/8fdnZndpIl1EQDGKGMqyIFJUFBV7QROVGBsaNcbkm5hfYmwx9kQTe43GAmJBg0aNMbFCEFsAxd5QQKqsNOns7nx+f9w7w2yZnVlYd2aY1/ORm5k5995zz71zd+Rz2jV3FwAAAAAAyF2RbBcAAAAAAADUj+AdAAAAAIAcR/AOAAAAAECOI3gHAAAAACDHEbwDAAAAAJDjCN4BAAAAAMhxBO8AAGwhM5tjZiPD9xeb2b2NmPdqM/te+H6smV3diHn/1cwubaz8GnDcn5nZ1+G5dWjq42eDmY0ws/nZLgcAIH8RvAMAMhYGqevCoCu+7JDtcuUSd/+ju5+Zbjszm2xmabdz923c/cstLZeZjTGzqTXyPsfdr9rSvBtYjmJJN0o6ODy3pfVse6qZefJ1MrO2ZjbOzJaEy+V17PcrM5ttZmvM7GMz2y1p3f+F6741s+lmtk8d+5eE+6UMtgnGAQBNjeAdANBQR4VBV3xZmLzSzIqyVbCtyVZ8HTtLai7pw/o2MrN2ki6uY7ubJLWU1EPSYEmnmNnpSfudKeknko6QtI2kIyV9E64bIulaScdJaiPpPkn/MLNojWOcL6m84acGAMB3h+AdALDFwtbRn5vZ55I+D9OONLOZZrbCzF43s9Kk7QeY2dtmtsrMHjOzCfHu4HW1EIf57xq+b2Zm15vZV2HX67+aWYtw3Qgzm29mvwlbZRfVCOxamNkNZjbXzFaa2dQw7V9m9n81jvmemR2b4nxPCfNYamaX1Fh3uZk9FL5vbmYPhdutMLNpZtbZzK6RNFzS7WHvhdvruY6Jcw91NLMXw2v3XzPbKdyuR7htUVJZJpvZmWb2fUl/lTQsPN6KcH21bvhmdpaZzTKzZWb2THKvijDvc8zs8/Bc7jAzS3F9mpnZzWa2MFxuDtN2k/RpuNkKM3ulrv1Df5J0q8LAO8lRkv7s7mvdfY6CAPyM8LgRSZdJ+rW7f+SBL9x9WbhvD0kfuvsMd3dJD0rqKGm7pLLvLOnk8Ph1MrNWkv4taQdL6oGS6rxT5PFLM/vIzLpt4T19eJjPKjNbYGa/reeaAgDyGME7AKCxHCNpiKTeZjZA0v2Sfiqpg6S7JT0TBiklkp6SNF5Se0l/l/TDBhznWkm7SSqTtKukrpL+kLR+ewWtql0VtMDeYUErriRdL2kPSXuFx/6dpJikcQoCNkmSmfUP9/9XzYObWW9Jd0k6RdIO4fl1S1HW08KydA+3O0fSOne/RNKrkn4R9l74RdI+xyi8jinyPEnSVQqCzpmSHk6xXYK7fxwe+43weG3rOK8DFASsJ0jqImmupAk1NjtS0p6SSsPtDklxyEskDVXwHfVX0EL+e3f/TFKfcJu27n5AXTub2WBJgxRUONS5SY33fcP33cKlr5nNs6B7/BVhUC8FAXfUzIZY0Np+hoJruDgpv9sUtPivS3FsufsaSYdJWlijB0qd513H+f1B0hhJ+7n7fG3ZPX2fpJ+6e+vwOtRXIQIAyGME7wCAhnoqbHldYWZPJaX/yd2Xufs6SWdLutvd33L3KncfJ2mDgsBmqKRiSTe7e4W7T5Q0LZMDhy29ZytoWV3m7qsk/VHSj5I2q5B0ZZj3c5JWS+oVBnBnSPqVuy8Iy/W6u2+Q9Iyk3cysZ5jHKZIec/eNdRTjOEnPuvuUcN9LFVQA1KVCQdC+a3i8Ge7+bZrTTL6OdflX0rEvUdCa3j1Nnpk4SdL97v52mPdFYd49kra51t1XuPtXkiYpCDZT5XWluy9x93JJVyi4pmmFQfWdCio26rqu/5F0oZm1DnsknKGgG720qRLlYEn9JO0v6UQFAa8krZL0hKSpCu7HyySdHbbCK+xpEXX3f2RS1jqkO28zsxvD8u3v7uVbck8nrettZtu6+3J3f3szyw4AyHEE7wCAhjrG3duGyzFJ6fOS3u8k6TdJQf4KBa3PO4TLgnjAFJqb4bE7KQjUZiTl+58wPW6pu1cmfV6rYOxzRwVjrb+omam7r5f0mKSTwyD/RAU9A+qyQ/K5hq2wqSZdGy/peUkTwm7Uf7Zgwrb6zMt0vbuvlrQsLNOW2kFJ30OY91IFrb1xyS3U8euaNq/wfaZlPFfSe+7+Zor1v1TQKv65pKclPSopPnFcvMLjz2ElwxwFvT4OD9N/Iul0Ba3/JQp6WzwbdnlvJenPYf6bK915t1UQqP/J3VeGaVtyT0tBr5XDJc0Nh1EM24LyAwByGME7AKCxJAfj8yRdkxTkt3X3lu7+qKRFkrrWGC+9Y9L7NdrUkioz2z5p3TcKArQ+Sfm2cfdUQWSybyStl7RLivXjFLScHihprbu/kWK7RQoqIuLla6mgdb2WsKX0CnfvraCr/pGSTo2vTpF/qvS45GNvo6D7/0IF101KunYKultnmu9CBZUu8bxbKTivBWn2S5uXgu93YYptazpQ0rFmttjMFiu4bjdYOC9A2Dp9krtv7+59FPxb5n/hvp9K2qjq55r8vkxBr4nP3D3m7v9R8H3uJamngjHxr4bHfVJSl7AcPeooZ13XM915L1dwDzxgZnuHaVtyT8vdp7n7KAXj9p+S9Hgm+wEA8g/BOwDgu/A3SeeEY4vNzFqZ2RFm1lrSG5IqJf3SzIrN7AcKxgbHvSupj5mVmVlzSZfHV4TdqP8m6SYz206SzKyrmaUae50Q7nu/pBvDltaomQ2LTygWBusxSTcodau7JE2UdKSZ7ROO379SKf57amb7m1m/sCv4twq6OMe7gn8t6Xvpyl2Hw5OOfZWkN919XthNe4GC3gNRMztD1SsqvpbULdyvLo9KOj287s0UdN1+K2y9bqhHJf3ezDqZWUcF47cfynDfMZK+ryDQLpM0XUH380skycx2MbMO4TkepqAl+2pJcve1CnpQ/C7sVt8tXP9smPc0SUeY2ffC+/IgBWPNPwiX7knHPVPBNStT3b0hvpbUwczaNOS83X2ygkqiJ81s8Jbc0xY80u4kM2vj7hUK7rFUQzgAAHmO4B0A0OjcfbqksyTdrqC1cZaCoEzhOPIfhJ+XSRqtoJUzvu9nCgLilxR0ja4287ykC8L83jSzb8Pteikzv5X0voIgbpmk61T9v4UPKhgrnTLQdPcPJf1c0iMKWm2Xa1O37Zq2VxDsfyvpY0n/1aaKgVskHWdmy83s1gzLr/C4l4Xl30NJE+0puObnK+ju3kfS60nrXlHw2LXFZlZzBne5+0sKxu8/EZ7XLqo+7rohrlYQdL+n4Hq/HaalFXZ3XxxfFLSkf5vUzXyPMM9VCibYOyn8TuJ+oWBM+EIFFUWPKKi0kYLvd4KkyQq+k1sVTPb2ibtX1jjuMkmx8HNVHeX8REGw/mXY3X2HTM/b3V9UMFb/n2Y2UFt2T58iaU643zkKKgYAAFshqz7kEACApmdmYyXNd/daM3M3cTlOVTCB2T7ZLAcAAEBNtLwDAKDE2PVzJd2T7bIAAADURPAOACh44fjicgXjmB/JcnEAAABqyajbvJnNUTC2rEpSpbsPMrP2CiaF6SFpjqQT3H15OHvwLQoeW7JW0pj4M0fN7DRJ8S6RV4fP/QUAAAAAAPVoSMv7/u5e5u6Dws8XSnrZ3XtKejn8LEmHKXjcSk8FM7zeJUlhsH+ZpCEKZhW+zMzabfkpAAAAAACwdSvagn1HSRoRvh+nYObWC8L0Bz1o0n/TzNqaWZdw2xfdfZkkmdmLkg5VMFNrnTp27Og9evTYgiLmsE8/DV57ZTqZLAAAAAAgn8yYMeMbd+/UGHllGry7pBfMzCXd7e73SOrs7ovC9YsldQ7fd1X156HOD9NSpafUo0cPTZ8+PcMi5pkRI4LXyZOzWQoAAAAAwHfEzOY2Vl6ZBu/7uPsCM9tO0otm9knySnf3MLDfYmZ2toLu9tpxxx0bI0sAAAAAAPJaRsG7uy8IX5eY2T8UjFn/2sy6uPuisFv8knDzBZK6J+3eLUxboE3d7OPpk+s41j0KH9MzaNCgrfch9L/P6qOMAQAAAAB5JO2EdWbWysxax99LOljSB5KekXRauNlpkp4O3z8j6VQLDJW0Muxe/7ykg82sXThR3cFhWmEaOTJYAAAAAABII5OW986S/hE8AU5Fkh5x9/+Y2TRJj5vZTyTNlXRCuP1zCh4TN0vBo+JOlyR3X2ZmV0maFm53ZXzyuoI0c2bwWlaWzVIAAAAAAPJARs95z5ZBgwY5E9YBAAAAAPKRmc1Ietz6FmnIc94BAAAAAEAWELwDAAAAAJDjCiJ4H/f6HD3z7sJsFwMAAAAAgM1SEMH7I299pefeW5TtYgAAAAAAsFkyes57votETJWxHJuY749/zHYJAAAAAAB5oiCC92hEiuXarPp77ZXtEgAAAAAA8kRBdJuPRiKqyrWW99dfDxYAAAAAANIojJZ3U+4F7xdfHLzynHcAAAAAQBoF0vJuuRe8AwAAAACQocIJ3nNtzDsAAAAAABkqnOCdlncAAAAAQJ4qiOA9YgTvAAAAAID8VRAT1hVFLPceFXfzzdkuAQAAAAAgTxRE8B6NmCqrcix4LyvLdgkAAAAAAHmiYLrN51zL+0svBQsAAAAAAGkURMt7UTQHx7xffXXwOnJkdssBAAAAAMh5BdPynnPBOwAAAAAAGSqI4J3nvAMAAAAA8lnhBO+0vAMAAAAA8lRhBO90mwcAAAAA5LGCmLAuJ1ve77472yUAAAAAAOSJggnec+5Rcb16ZbsEAAAAAIA8URjd5iOmylxref/nP4MFAAAAAIA0CqLlPScfFXfDDcHrUUdltxwAAAAAgJxXEC3vRRFTLNeCdwAAAAAAMlQQwXtOdpsHAAAAACBDBRG8R3JxwjoAAAAAADJUEMF7US4+Kg4AAAAAgAwVzIR1MZfcXWaW7eIExo/PdgkAAAAAAHmiIFreo5EgYM+p1vfu3YMljaVLl6qsrExlZWXafvvt1bVrV5WVlalt27bq3bt3oxfr8ssv1/XXX9+gfbbZZps608eMGaOJEyduUXkmT56sI488crP2vfnmm7V27domO953Jfk6nnnmmfroo4+2OM8VK1bozjvvbPB+0Wg0cT+WlZXp2muvlSSNGDFC06dPl5T6fmgMY8eOVadOnVRWVqbdd99dN910U9p9Mr0PzzvvPE2ZMqUxiplV5eXlGjJkiAYMGKBXX3212rqTTjpJvXr1Ut++fXXGGWeooqKiQXln+vuwOffA4YcfrhUrVjR4v5r38sKFC3Xcccc1OB9Jev/99zVmzJjN2hcAAOC7VljBey6Ne3/ssWBJo0OHDpo5c6Zmzpypc845R7/+9a8TnyOR9F9fZWVlY5Q2L21O8J5t6b6ve++9t1EqbTY3eG/RokXi/ps5c6YuvPDCLS5LQ40ePVozZ87Ua6+9pmuuuUbz5s3b4jyXLl2qN998U/vuu2/G++Tq39bLL7+sfv366Z133tHw4cOrrTvppJP0ySef6P3339e6det07733ZqmUm7i7YrGYnnvuObVt27bB+9e8l3fYYYfNrjTs16+f5s+fr6+++mqz9gcAAPguFVbwnkst73fdFSxboKqqSmeddZb69Omjgw8+WOvWrZMUtIKed955GjRokG655RbNmDFD++23n/bYYw8dcsghWrRokSTp1ltvVe/evVVaWqof/ehHiXw/+ugjjRgxQt/73vd06623JtJvvPFG9e3bV3379tXNN99cqzzurl/84hfq1auXRo4cqSVLltRZ7rqOu2bNGp1xxhkaPHiwBgwYoKeffrrWfqm2qaqq0m9/+1v17dtXpaWluu2223Trrbdq4cKF2n///bX//vtLkl544QUNGzZMAwcO1PHHH6/Vq1dLkv7zn/9o991318CBA/Xkk0/WWeY5c+Zo+PDhGjhwoAYOHKjXX389se66665Tv3791L9//0QwO2vWLI0cOVL9+/fXwIED9cUXX8jddf7556tv377q16+fHgsrbyZPnqzhw4fr6KOPVu/eveu9jjVbuC+55BL1799fQ4cO1ddffy1J+uKLLzR06FD169dPv//97+tsBb3wwgv1xRdfqKysTOeff37Ksm2uX//61+rTp48OPPBAlZeXS5JmzpypoUOHqrS0VMcee6yWL1+uJUuWaI899pAkvfvuuzKzROC0yy671Fv50qFDB+26665atGiR5syZo759+ybWXX/99br88str7ZPqb+GJJ57QoYcemtjuyiuv1J577qm+ffvq7LPPlocVfzX/tqZNm6bS0tLEdYyXYezYsTrmmGN00EEHqUePHrr99tt14403asCAARo6dKiWLVsmSfrb3/6mPffcU/3799cPf/jDxPmOGjVKDz74oCTp7rvv1kknnVTrXObMmaMDDjhApaWlOvDAA/XVV19p5syZ+t3vfqenn35aZWVlid+EuMMPP1xmJjPT4MGDNX/+fElBi/oZZ5xR59/9Nddco91220377LOPPv300zq/i9mzZ2vYsGGJey7ZX/7yF+25554qLS3VZZddlih7r169dOqpp6pv376aN2+eevTooW+++UYXXnih7rjjjsT+8db+1atX68ADD9TAgQPVr1+/xN9/zXs5+V4YOnSoPvzww0Re8b+f+n5vjjrqKE2YMKHO8wQAAMgqd8/ZZY899vDGcM9/v/CdLnjWv123sVHyaxT77RcsDXDZZZf5X/7yF3d3nz17tkejUX/nnXfc3f3444/38ePHh1nv5z/72c/c3X3jxo0+bNgwX7Jkibu7T5gwwU8//XR3d+/SpYuvX7/e3d2XL1+eOMawYcN8/fr1Xl5e7u3bt/eNGzf69OnTvW/fvr569WpftWqV9+7d299++213d2/VqpW7uz/xxBM+cuRIr6ys9AULFnibNm3873//e63zqOu4F110UaL8y5cv9549e/rq1at90qRJfsQRR9S7zZ133uk//OEPvaKiwt3dly5d6u7uO+20k5eXl7u7e3l5uQ8fPtxXr17t7u7XXnutX3HFFb5u3Trv1q2bf/bZZx6Lxfz4449PHC/ZmjVrfN26de7u/tlnn3n83nzuued82LBhvmbNmmrHHjx4sD/55JPu7r5u3Tpfs2aNT5w4MXF9Fi9e7N27d/eFCxf6pEmTvGXLlv7ll1+mvY777befT5s2zd3dJfkzzzzj7u7nn3++X3XVVe7ufsQRR/gjjzzi7u533XVX4vtJNnv2bO/Tp0/ic6qy1RSJRLx///6JZcKECXWW66GHHnJ39yuuuMJ//vOfu7t7v379fPLkye7ufumll/qvfvUrd3fv3bu3r1y50m+77TYfNGiQP/TQQz5nzhwfOnRoreM/8MADifzmzp3r/fv393Xr1tU6n7/85S9+2WWXubv7aaed5n//+9/r/Vs49dRTE9cy+Xt0dz/55JMT65L/ttzd+/Tp46+//rq7u19wwQWJMjzwwAO+yy67+LfffutLlizxbbfd1u+66y53dz/vvPP8pptucnf3b775JpHXJZdc4rfeequ7uy9evNh32WUXnzJlivfs2bNaeeKOPPJIHzt2rLu733fffT5q1Kha1yiVjRs3+oABA3zKlCnunv7vfs2aNb5y5UrfZZddEr9ByY466igfN26cu7vffvvtiXvu+eef97POOstjsZhXVVX5EUcc4f/973999uzZbmb+xhtvJPKI/72+/fbbvu+++ybSv//97/tXX33lFRUVvnLlSncP/p532WUXj8Vitb775M833nij/+EPf3B394ULF/puu+3m7ql/S9zdp06d6kceeWS91w8AACBTkqZ7I8XHGbe8m1nUzN4xs2fDzzub2VtmNsvMHjOzkjC9Wfh5Vri+R1IeF4Xpn5rZIY1bDZFaTra8N4Kdd95ZZWVlkqQ99thDc+bMSawbPXq0JOnTTz/VBx98oIMOOkhlZWW6+uqrE61tpaWlOumkk/TQQw+pqGjT3IVHHHGEmjVrpo4dO2q77bbT119/ralTp+rYY49Vq1attM022+gHP/hBrfG0U6ZM0YknnqhoNKoddthBBxxwQJ3lruu4L7zwgq699lqVlZVpxIgRWr9+fa2uq6m2eemll/TTn/40kVf79u1rHfPNN9/URx99pL333ltlZWUaN26c5s6dq08++UQ777yzevbsKTPTySefXGeZKyoqdNZZZ6lfv346/vjjE+POX3rpJZ1++ulq2bJl4tirVq3SggULdOyxx0qSmjdvrpYtW2rq1KmJ69O5c2ftt99+mjZtmiRp8ODB2nnnnRt0HUtKShLj85O//zfeeEPHH3+8JOnHP/5xnfvWVF/ZktXsNh+/z5JFIpFE+sknn6ypU6dq5cqVWrFihfbbbz9J0mmnnZYYX77XXnvptdde05QpU3TxxRdrypQpevXVV2t1+Y577LHHVFpaql133VXnnnuumjdvntE51ve3sGjRInXq1Cmx7aRJkzRkyBD169dPr7zySrXW2/i5rVixQqtWrdKwYcMk1b7W+++/v1q3bq1OnTqpTZs2OuqooyQFXbPj39UHH3yg4cOHq1+/fnr44YcTx+ncubOuvPJK7b///rrhhhvqvKffeOONxDFPOeUUTZ06NaPrIEnnnnuu9t1332rXuK6/+1dffVXHHnusWrZsqW233VZHH310nfm99tprOvHEExNliXvhhRf0wgsvaMCAARo4cKA++eQTff7555KknXbaSUOHDq2V14ABA7RkyRItXLhQ7777rtq1a6fu3bvL3XXxxRertLRUI0eO1IIFCxK9TVI54YQTEl3oH3/88cRY+Pp+b7bbbjstXLgw00sJAADQZBoy2/yvJH0sadvw83WSbnL3CWb2V0k/kXRX+Lrc3Xc1sx+F2402s96SfiSpj6QdJL1kZru5e1UjnUtKW2vw3qxZs8T7aDRarYtsq1atJAU9K/r06aM33nij1v7/+te/NGXKFP3zn//UNddco/fff7/OfBt7bG9dx3V3PfHEE+rVq1e1bZP/cZ5qm0y4uw466CA9+uij1dJnzpyZ0f433XSTOnfurHfffVexWCzjgDFT8e+rIYqLixNPT/guvqfGku4JD/vuu69effVVzZ07V6NGjdJ1110nM9MRRxxR5/ajR4/W7bffrunTp+vggw/W0UcfraKiIsViscQ269evr7VffX8LLVq0SOyzfv16nXvuuZo+fbq6d++uyy+/vFp+mX5XyX9HkUgk8TkSiSS+qzFjxuipp55S//79NXbsWE2ePDmxz/vvv68OHTo0eiB5xRVXqLy8XHfffXfK8m7O/VTX9+zuuuiii/TTn/60WvqcOXPqvY7HH3+8Jk6cqMWLFycqSx5++GGVl5drxowZKi4uVo8ePer8npN17dpVHTp00HvvvafHHntMf/3rXxPlSvVbsn79erVo0SLt+QIAADS1jFrezaybpCMk3Rt+NkkHSIrPCjRO0jHh+1HhZ4XrDwy3HyVpgrtvcPfZkmZJGtwI55BWJBcnrGsivXr1Unl5eSJgqaio0IcffqhYLKZ58+Zp//3313XXXaeVK1cmxoDXZfjw4Xrqqae0du1arVmzRv/4xz9qtYzuu+++euyxx1RVVaVFixZp0qRJtfJJddxDDjlEt912W2Js8TvvvFNr31TbHHTQQbr77rsTwUZ8PHHr1q21atUqScHY19dee02zZs2SFIyf/+yzz7T77rtrzpw5+uKLLySpVnAft3LlSnXp0kWRSETjx49XVVVV4tgPPPBAYqzysmXL1Lp1a3Xr1k1PPfWUJGnDhg1au3athg8fnrg+5eXlmjJligYPrv0nkMl1rM/QoUP1xBNPSFLKsbvJ10ZSxmXLRCwWS7R2PvLII9pnn33Upk0btWvXLtFbY/z48YlW+OHDh+uhhx5Sz549FYlE1L59ez333HPaZ5996j3OoEGDdMopp+iWW25R586dtWTJEi1dulQbNmzQs88+W2v7VH8LkvT9738/cW/EA8KOHTtq9erVKSc/a9u2rVq3bq233npLUuprXZ9Vq1apS5cuqqio0MMPP5xI/9///qd///vfeuedd3T99ddr9uzZtfbda6+9Esd8+OGHU/ZUSHbvvffq+eef16OPPprRhJf77ruvnnrqKa1bt06rVq3SP//5zzq323vvvauVJe6QQw7R/fffn/htWbBgQcq5MJKNHj1aEyZM0MSJExO9SFauXKnttttOxcXFmjRpkubOnSup9r1cV15//vOftXLlSpWWlibKler35rPPPqs2fwIAAECuyLTb/M2Sficp3rTVQdIKd483zcyX1DV831XSPEkK168Mt0+k17FPgpmdbWbTzWx6fKKrLVWUiy3vEycGy3espKREEydO1AUXXKD+/furrKxMr7/+uqqqqnTyySerX79+GjBggH75y1/WO9PzwIEDNWbMGA0ePFhDhgzRmWeeqQEDBlTb5thjj1XPnj3Vu3dvnXrqqYnuxMlSHffSSy9VRUWFSktL1adPH1166aW19k21zZlnnqkdd9xRpaWl6t+/vx555BFJ0tlnn61DDz1U+++/vzp16qSxY8fqxBNPVGlpqYYNG6ZPPvlEzZs31z333KMjjjhCAwcO1HbbbVfn+Z977rkaN26c+vfvr08++STRanjooYfq6KOP1qBBg1RWVpZ4jNb48eN16623qrS0VHvttZcWL16sY489NlHGAw44QH/+85+1/fbb1zpWJtexPjfffLNuvPFGlZaWatasWWrTpk2tbTp06KC9995bffv21fnnn59x2datW1ftUXF1zTbfqlUr/e9//1Pfvn31yiuv6A9/+IMkady4cTr//PNVWlqqmTNnJtJ79Oghd0/M9L7PPvuobdu2ateuXdpzveCCC/TAAw9o/fr1+sMf/qDBgwfroIMO0u67715r21R/C1LQZTze6t22bVudddZZ6tu3rw455BDtueeeKY9/33336ayzzlJZWZnWrFlT57Wuz1VXXaUhQ4Zo7733TpR5w4YNOuuss3T//fdrhx120A033KAzzjgjEWjG3XbbbXrggQdUWlqq8ePH65Zbbkl7vHPOOUdff/21hg0bprKyMl155ZX1bj9w4ECNHj1a/fv312GHHZbyWtxyyy2644471K9fPy1YsCCRfvDBB+vHP/5xYjK74447rt5AO65Pnz5atWqVunbtqi5dukgKZsqfPn26+vXrpwcffDBxvWreyzUdd9xxmjBhgk444YREWn2/N5MmTUrZ6wMAACCbrOY/CGttYHakpMPd/VwzGyHpt5LGSHrT3XcNt+ku6d/u3tfMPpB0qLvPD9d9IWmIpMvDfR4K0+8L90kZwQ4aNMjjM2tvicenzdPvnnhPUy/YX93atdzi/IBctnbtWrVo0UJmpgkTJujRRx+tc/Z+VLfPPvvo2WefbdDjylavXp2Yzf/aa6/VokWLMgqikZs2bNig/fbbT1OnTq02DwgAAMDmMrMZ7j6oMfLK5F8ne0s62swOl9RcwZj3WyS1NbOisHW9m6R4c8sCSd0lzTezIkltJC1NSo9L3uc7Fe82nzQkNvvGjg1ex4zJZimwFZoxY4Z+8YtfyN3Vtm1b3X///dkuUl644YYb9NVXXzUoeP/Xv/6lP/3pT6qsrNROO+2ksfG/a+Slr776Stdeey2BOwAAyElpW96rbRy2vLv7kWb2d0lPJE1Y956732lmP5fUz93PCSes+4G7n2BmfSQ9omCc+w6SXpbUs74J6xqr5f2pdxbovMdm6pXf7Kfvdar9zOusGDEieE2aoAoAAAAAsPVo6pb3VC6QNMHMrpb0jqT7wvT7JI03s1mSlimYYV7u/qGZPS7pI0mVkn7eFDPNS0kt7wU4YR0AAAAAIP81KHh398mSJofvv1Qds8W7+3pJx6fY/xpJ1zS0kFtq04R1TX1kAAAAAAC2XKazzee1SPj84cqcGvQOAAAAAEBmCiJ4j+bihHUAAAAAAGSoIKbUTXSbz6Ux7889l+0SAAAAAADyREEE75HEmPccanpvyfPmAQAAAACZKYxu85aDE9bdeWewAAAAAACQRmEE74mW9xzqNv/448ECAAAAAEAaBO8AAAAAAOS4Agneg9ecmrAOAAAAAIAMFUjwHpxmjJZ3AAAAAEAeKozgPZywrpLgHQAAAACQhwrkUXHBa06NeZ88OdslAAAAAADkiYJoeS+Kd5tnzDsAAAAAIA8VRPAen7Aup7rNX399sAAAAAAAkEZBBO+RcMx7Tk1Y9+yzwQIAAAAAQBoFEbzHu83n1Jh3AAAAAAAyVBDBe05OWAcAAAAAQIYKIniPRoJu81VMWAcAAAAAyEMF8ai4RPCeSy3vLVpkuwQAAAAAgDxRGMG75WDw/u9/Z7sEAAAAAIA8UVjd5nMpeAcAAAAAIEMFFbzHcmnM+1VXBQsAAAAAAGkUVPBemUst7y+/HCwAAAAAAKRREMF7JBfHvAMAAAAAkKGCCN6L4t3mCd4BAAAAAHmoIIL3nOw2DwAAAABAhgriUXFmJrMcm7CuQ4dslwAAAAAAkCcKIniXgq7zOTXm/Yknsl0CAAAAAECeKIhu81IwaV1OBe8AAAAAAGSoYIL3aK61vF90UbAAAAAAAJBGwXSbj0Ystyase+ONbJcAAAAAAJAnCqrlPacmrAMAAAAAIEOFE7wz5h0AAAAAkKfSBu9m1tzM/mdm75rZh2Z2RZi+s5m9ZWazzOwxMysJ05uFn2eF63sk5XVRmP6pmR3ynZ1VHXJuzDsAAAAAABnKpOV9g6QD3L2/pDJJh5rZUEnXSbrJ3XeVtFzST8LtfyJpeZh+U7idzKy3pB9J6iPpUEl3mlm0Ec+lXsXRiDZWxZrqcOl16xYsAAAAAACkkTZ498Dq8GNxuLikAyRNDNPHSTomfD8q/Kxw/YFmZmH6BHff4O6zJc2SNLgxTiITzYoi2liZQ8H7Qw8FCwAAAAAAaWQ05t3MomY2U9ISSS9K+kLSCnevDDeZL6lr+L6rpHmSFK5fKalDcnod+yQf62wzm25m08vLyxt8QqmU5FrwDgAAAABAhjIK3t29yt3LJHVT0Fq++3dVIHe/x90HufugTp06NVq+JUU51m3+vPOCBQAAAACANBr0nHd3X2FmkyQNk9TWzIrC1vVukhaEmy2Q1F3SfDMrktRG0tKk9Ljkfb5zJdGINlTkUPA+c2a2SwAAAAAAyBOZzDbfyczahu9bSDpI0seSJkk6LtzsNElPh++fCT8rXP+Ku3uY/qNwNvqdJfWU9L9GOo+0cq7lHQAAAACADGXS8t5F0rhwZviIpMfd/Vkz+0jSBDO7WtI7ku4Lt79P0ngzmyVpmYIZ5uXuH5rZ45I+klQp6efuXtW4p5NaSVFEq9ZXpt8QAAAAAIAckzZ4d/f3JA2oI/1L1TFbvLuvl3R8iryukXRNw4u55UqiTFgHAAAAAMhPDRrzns9yrtv8brtluwQAAAAAgDxRWMF7LrW833NPtksAAAAAAMgTGT0qbmvQrCiiDbkUvAMAAAAAkKGCCd6DMe9NNj9eemefHSwAAAAAAKRRWN3mc2nM+2efZbsEAAAAAIA8UTgt77k25h0AAAAAgAwVTvAejSrmUmUutb4DAAAAAJCBwgnei4JTzamu8wAAAAAAZKCgxrxL0sbKmFqWZLkwklRWlu0SAAAAAADyREEG7znh5puzXQIAAAAAQJ4omG7zzaLBqfKsdwAAAABAvimY4D3nxryffHKwAAAAAACQBt3ms2X+/GyXAAAAAACQJwqn5T2aY8E7AAAAAAAZKpzgPde6zQMAAAAAkKGCCd6b5Vq3eQAAAAAAMsSY92wZNizbJQAAAAAA5ImCC95z5lFxf/pTtksAAAAAAMgThddtnjHvAAAAAIA8UzDBe0k0KimHus3/8IfBAgAAAABAGgXXbT5ngvelS7NdAgAAAABAniiclvdE8F6V5ZIAAAAAANAwhRe8M+YdAAAAAJBnCid4j+ZYt3kAAAAAADJUMGPei6MmKYeC9wMPzHYJAAAAAAB5omCCdzNTSVFEG3Kl2/yll2a7BAAAAACAPFEw3eYlqVk0kjst7wAAAAAAZKiggveSohwK3g87LFgAAAAAAEijYLrNSzkWvK9bl+0SAAAAAADyROG1vOfKmHcAAAAAADJUWME7Y94BAAAAAHmosIL3XOo2DwAAAABAhtIG72bW3cwmmdlHZvahmf0qTG9vZi+a2efha7sw3czsVjObZWbvmdnApLxOC7f/3MxO++5Oq2451W3+yCODBQAAAACANDKZsK5S0m/c/W0zay1phpm9KGmMpJfd/Vozu1DShZIukHSYpJ7hMkTSXZKGmFl7SZdJGiTJw3yecffljX1SqZREI9qQKy3vv/1ttksAAAAAAMgTaVve3X2Ru78dvl8l6WNJXSWNkjQu3GycpGPC96MkPeiBNyW1NbMukg6R9KK7LwsD9hclHdqYJ5MO3eYBAAAAAPmoQWPezayHpAGS3pLU2d0XhasWS+ocvu8qaV7SbvPDtFTpNY9xtplNN7Pp5eXlDSleWs1yKXgfMSJYAAAAAABII+Pg3cy2kfSEpPPc/dvkde7uCrrCbzF3v8fdB7n7oE6dOjVGlgk5NeYdAAAAAIAMZRS8m1mxgsD9YXd/Mkz+OuwOr/B1SZi+QFL3pN27hWmp0psMj4oDAAAAAOSjTGabN0n3SfrY3W9MWvWMpPiM8adJejop/dRw1vmhklaG3eufl3SwmbULZ6Y/OExrMox5BwAAAADko0xmm99b0imS3jezmWHaxZKulfS4mf1E0lxJJ4TrnpN0uKRZktZKOl2S3H2ZmV0laVq43ZXuvqwxTiJTdJsHAAAAAOSjtMG7u0+VZClWH1jH9i7p5ynyul/S/Q0pYGMqiUZzp+X9hBPSbwMAAAAAgDJred9q5FS3+XPPzXYJAAAAAAB5okGPist38W7zQeeALFu7NlgAAAAAAEijoFremxUFdRUbq2JqVhTNbmEOPzx4nTw5q8UAAAAAAOS+wmp5j4bBe650nQcAAAAAIAOFFbwXEbwDAAAAAPJPYQbvPC4OAAAAAJBHCit4p9s8AAAAACAPFdSEdTnVbX7MmGyXAAAAAACQJwoyeN9A8A4AAAAAyCOF1W0+l4L3b74JFgAAAAAA0iiolvdmuTTm/bjjglee8w4AAAAASKMgW96ZbR4AAAAAkE8KM3jPhZZ3AAAAAAAyRPAOAAAAAECOK6zgPT7mvaoqyyUBAAAAACBzBTVhXU61vP/sZ9kuAQAAAAAgTxC8Z8vo0dkuAQAAAAAgTxRUt/lm0aikHHnO+7x5wQIAAAAAQBoF1fLevCSoq1i9oTLLJZF0yinBK895BwAAAACkUVgt70VRdW/fQp9/vTrbRQEAAAAAIGMFFbxL0ve331YfL/o228UAAAAAACBjhRe8d9lWs5eu0dqNOdB1HgAAAACADBRk8O4ufbp4VbaLAgAAAABARgpqwjpJ2n371pKkz79erQE7tsteQX7zm+wdGwAAAACQVwoueN9u22aSpKVrNma3IEcdld3jAwAAAADyRsF1m29RHFVJNKIVa7McvH/6abAAAAAAAJBGwbW8m5natizWirUV2S3IT38avPKcdwAAAABAGgXX8i4pCN7XZbnlHQAAAACADBVo8F6i5dlueQcAAAAAIEOFGby3KNZKgncAAAAAQJ4oyOC9XcsSLc/2hHUAAAAAAGSo4Cask+Jj3ivk7jKz7BTi97/PznEBAAAAAHmnQIP3Em2sjGldRZValmTpEowcmZ3jAgAAAADyTtpu82Z2v5ktMbMPktLam9mLZvZ5+NouTDczu9XMZpnZe2Y2MGmf08LtPzez076b08lM25bFkpTdx8XNnBksAAAAAACkkcmY97GSDq2RdqGkl929p6SXw8+SdJiknuFytqS7pCDYl3SZpCGSBku6LB7wZ0O7XAjezzsvWAAAAAAASCNt8O7uUyQtq5E8StK48P04ScckpT/ogTcltTWzLpIOkfSiuy9z9+WSXlTtCoEm06ZFiSRpBZPWAQAAAADywObONt/Z3ReF7xdL6hy+7yppXtJ288O0VOm1mNnZZjbdzKaXl5dvZvHq13GbIHhf/O367yR/AAAAAAAa0xY/Ks7dXZI3Qlni+d3j7oPcfVCnTp0aK9tqvtdpG23TrEgz5i7/TvIHAAAAAKAxbW7w/nXYHV7h65IwfYGk7knbdQvTUqVnRTRiGrhTO02fQ/AOAAAAAMh9mxu8PyMpPmP8aZKeTko/NZx1fqiklWH3+uclHWxm7cKJ6g4O07Jmz53a6dOvV2Vv3Psf/xgsAAAAAACkkcmj4h6V9IakXmY238x+IulaSQeZ2eeSRoafJek5SV9KmiXpb5LOlSR3XybpKknTwuXKMC1r9tq1gyTp3x8szlIB9goWAAAAAADSsGDIem4aNGiQT58+/TvJ29016o7XtHztRr3ymxEqjm7x8P+Gef314JUAHgAAAAC2SmY2w90HNUZeTRyx5g4z0zn77aJ5y9Zp2pwsdAK4+OJgAQAAAAAgjYIN3iVp7106SpJmMHEdAAAAACCHFXTw3qZlsXbrvI2m8cg4AAAAAEAOK+jgXZIG9Wivd+YuV1Usd8f+AwAAAAAKW8EH70N2bq9VGyo1c96KbBcFAAAAAIA6FWW7ANk2otd2Ko6anv9wsfbYqV3THfjmm5vuWAAAAACAvFbwLe9tWhRr71076t8fLFKTPjavrCxYAAAAAABIo+CDd0k6rO/2mrdsnT5c+G3THfSll4IFAAAAAIA0CN4lHdR7e0Ujpv98sLjpDnr11cECAAAAAEAaBO+S2rcq0ZCd2+vZ9xZqfUVVtosDAAAAAEA1BO+hMXv10Jyla3XMHa/poTfnZrs4AAAAAAAkELyHDu6zvW4eXaaYu37/1Ad6bNpXWrxyfdNOYgcAAAAAQB0K/lFxyY4Z0FWH9t1ex975ui544n1J0qiyHXTTCWWKRCzLpQMAAAAAFCqC9xqaF0f1xM+GaeZXK/TyJ0t039TZcpeuGtVXbVoWN96B7r678fICAAAAAGzVCN7r0LKkSHvt2lHDdumgbZsX67ZXPtfytRs17vTBjdcC36tX4+QDAAAAANjqMea9HmamX43sqStH9dWrn3+jXzz6tlatr2iczP/5z2ABAAAAACANWt4zcOLg7lqxbqNueOEztWnxsf70g9Itz/SGG4LXo47a8rwAAAAAAFs1gvcMmJnOHbGrVqyt0D1TvtTajVU6fo/u2nvXDjJjIjsAAAAAwHeL4L0Bfj1yN22sjOnJt+fr6ZkL1bfrtnrkrKHatnkjTmQHAAAAAEANjHlvgBYlUV1+dB/975KRuu6H/fTRwm/1p+c+yXaxAAAAAABbOYL3zdC8OKrRe+6oM4d/T4/+7yu9PuubbBcJAAAAALAVo9v8Fvh/B+2mFz/6Wmc+OF27b99abVuWaKcOLbVnj/Ya1KOdtmvdPPXO48c3XUEBAAAAAHnN3D3bZUhp0KBBPn369GwXo15flK/WfVNna843a7RibYW+/Ga11lfEJEmH9OmsCw7dXTt3bMXEdgAAAABQYMxshrsPapS8CN4b18bKmD5cuFKvfLJEf3v1S62viKl5cUSH9tleg3fuoOP26KaSooj02GPBDqNHZ7fAAAAAAIDvBMF7nihftUHPvLtQs5as0r/eW6Rv11dq1+22Ue8u2+r//fFsFUUi+u89j6tDqxJ12KaZ+nVto+bF0WwXGwAAAADQCBozeGfM+3eoU+tm+sk+O0uS/nhsP/3ng8W6b+psvTt/hcpXbVBVzHXJPz5IbL9Th5b6wYBuateqWG1aFKtdyxK1a1miNi2K1bw4opKiiJoVRVVSFFE0Qjd8AAAAACgUtLxny4gRirlryTPPa+maDZq7dK2uf/5TffnNmox2L4qYdu7YSvvt1kkjem2nPXdup2ZFtNoDAAAAQK6g5X0rETHT9m2aa/s2zdVnhzY6vF8XVVTFtGJthVas3ajlayu0fO1GfbuuQhsqY9pYGdOGypg2VFZpfUUwtv7BN+bq3qmz1aI4qqHfa6/O2zZXi5KoWpZE1bKkKHwN3ndr10I9O7dWSTSi4qgxiR4AAAAA5AmC9xxTHI2oU+tm6tS6WUbbr91YqTe/XKrJn5brzS+X6sOF32rtxiqt3VipWD2dKsykZkURlUQjala8KcBv17JY7VqVqF3LYrVvWaK2LUvUvlWJmhdHVRQxRaOm5kXB9iVFQVf+kmgkyKsoouLwfVE00khXBAAAAABA8J4tEyc2SjYtS4p0wO6ddcDunaulu7s2VMa0bmOV1mys1NqNVfp08SotXrleGyqrklrxg5b8dRurtHpDpVasrdDHi75NtP7XVwFQn5JoRM2LI2peHFWLkqiaF0UTn4Mlohbh+2ZFEUUjQW+AaMRUFI2oJGqJioHi8DUasU2LmSIRU1HEEhUGyXk3Lw6O2aw4WEcvAwAAAAD5jOA9Wzp2/E6zN7NEMNuuVYkkabfOrRuURyzm+nZ9hZat2aj1FTFVxVwVsZjWV1Rp7YYqbayKqaIqlujSv7EypspYTBsqYlqzsUrrKzYt6yqCrv7rK6q0Ym2Q37pw3YbKIO/KWEyVVa7Kza0xqEc84I+GAX80GgT9zYsjYcXCpqC/WVHy+6A3QVCpYCqORILXcOhBfF3EqlcqREzhsSKJHgrFUVOzokhi24iFS0SKWjCMIV6+eMVFfN94vgAAAAAKE8F7towdG7yOGZPNUtQrEjG1DbvONyV3V0WVq6IqrBQIX2MeBPaxWPBaFS4VVTGtr9g0F8D6iiqtT3q/oTKWtE9MVTGpMhZL7LduY3z7Kq1aX6nyig3aULlp38qqeMWFq7Iqttm9ERrDpgqCYM6EqAUVEUVJPRLMTGYKKweC102fg/dmwT7JFRHRSCSodEhUJATHM9tU8VEcjfeMiFdaxPMPtk+8jx87sqlskaQ0U1CGeFq8TMmfIxZUfsSPYxbsF4lIJlP4v03npPi5Jb2XkipTgvMsSapwqbWfxcsW5iuFx6leBqvz3Dato6cHAAAAGluTB+9mdqikWyRFJd3r7tc2dRlyQh4E79liZiopClqfW2U29L9JxcIeCBVVrorKmKo8qFCo8qAyIRZTkOauyiqvVgGxsSqoSIjFt3Up5ps+uyuotKja1JthY61jhPskHbMqFhwryEtyBXnFP8fc5R6ULeYul1RZFVNlWPmxbmOVqrxKnlyueDk9yCveM6IiUZmiauWOn0c2KzdySXIFQHLFwKZKg7BCJawE2FSpUb3yIV4RUFfFhFWrYKij8sGSKxiql2HTPpvyjJdNSt42Od9NPVeqnVutsiSXsXoFiVKt06YKnKKoVbtOSto2/j48VOKDVUu3atskn1Py95OcX6rjxM85XglUV/4191O1/OooXx3nknLbWudS/TtI7KvkfJLe13Ucqc4P6batef3qO27ymrq3rb/slpQa7/mUfH82NL/kPFPVrdX1XWSSX+101fkh3fapjhmJbOqVlbK8qm9dcnoG50XlIwDkrCYN3s0sKukOSQdJmi9pmpk94+4fNWU5gC0RiZiaRaJqViQpBysXss19UzBfGVYAVFR5tfS6Khhqrq9K7BuTu4IlaR9XkCbF9w/zULBewf+S8oslenRUVrlcQWWGa9N+8TLE844fR/F8Y15tXVVsU2+Q+LE27V+7rF4jn/j2lbFYohIk+TwT+2nT+XhyfkmflbyfVz/vynjeNda7JI9Jrlgiv+RzSJx34lwU9mKJbfrOpFp5q1Y5q5+P5LXKH78+8e++KhZ8R0o613i+4ekmjgWgaTRG5UmtjTLcJzk9mtTzqWZmNeseUlcw1V/pUUcx663YqLVvA46V7ji1jlrvOda/b83rG68cS1TA1qG++pzU+9RzrTbrOA0vRH3VUCkr8erZK5OKv0zz29w6slTXdfPOteHHCYZ+SkVBN8jU+29G3un3a7pj1X9uDb9HGlNTt7wPljTL3b+UJDObIGmUJIJ3YCuR6LYvU1E026VBoUgE9TUC/U3vVa0yQCnSa+7rYU+ayrCmJ7nSIOV+NSoV0h2jepk8Kf/ax4r37qmIxarto1r5qI68N12n6nvVPk7NDdJtW9dxk49X7ZJU27Z2HjXzSvQuisVUF0+TX/Uy1y5bQ8tXO5+Gba86jlvfd5RcCVdX3jVV27+xypyi3JuTV62/j4zuuer5xnud1Zwjp/Z1SX3N6i1Hnesz3zfNx9T3Xprj1ty31voGnlNQCR3cX1Up7qmax8tEvfdn7Tsr/T6NnV+q9Z74v0Y7Tt37pN6pvqudard6v6EUO23eccLepfX8Hqcvz2atSnnNNuc8gv3q+Q42815Mvc9m7FSPpg7eu0qal/R5vqQhyRuY2dmSzpakHXfcselKBgDIW8nDC5JSs1IWAACAOLu48fLKuYdxu/s97j7I3Qd16tQp28UBAAAAACDrmrrlfYGk7kmfu4Vphee557JdAgAAAABAnmjqlvdpknqa2c5mViLpR5KeaeIy5IaWLYMFAAAAAIA0mrTl3d0rzewXkp5X8Ki4+939w6YsQ864887g9dxzs1sOAAAAAEDOa/LnvLv7c5LoM/7448ErwTsAAAAAII2cm7AOAAAAAABUR/AOAAAAAECOI3gHAAAAACDHEbwDAAAAAJDjzN2zXYaUzKxc0txGyq6jpG8aKS8gV3GfoxBwn6MQcJ+jEHCfoxD0cvfWjZFRk8823xDu3qmx8jKz6e4+qLHyA3IR9zkKAfc5CgH3OQoB9zkKgZlNb6y86DYPAAAAAECOI3gHAAAAACDHFVLwfk+2CwA0Ae5zFALucxQC7nMUAu5zFIJGu89zesI6AAAAAABQWC3vAAAAAADkJYJ3AAAAAABy3FYfvJvZoWb2qZnNMrMLs10eYHOZWXczm2RmH5nZh2b2qzC9vZm9aGafh6/twnQzs1vDe/89MxuY3TMAMmdmUTN7x8yeDT/vbGZvhffzY2ZWEqY3Cz/PCtf3yGrBgQyZWVszm2hmn5jZx2Y2jN9zbG3M7Nfhv1k+MLNHzaw5v+fId2Z2v5ktMbMPktIa/PttZqeF239uZqdlcuytOng3s6ikOyQdJqm3pBPNrHd2SwVstkpJv3H33pKGSvp5eD9fKOlld+8p6eXwsxTc9z3D5WxJdzV9kYHN9itJHyd9vk7STe6+q6Tlkn4Spv9E0vIw/aZwOyAf3CLpP+6+u6T+Cu53fs+x1TCzrpJ+KWmQu/eVFJX0I/F7jvw3VtKhNdIa9PttZu0lXSZpiKTBki6LB/z12aqDdwUXYpa7f+nuGyVNkDQqy2UCNou7L3L3t8P3qxT8Q6+rgnt6XLjZOEnHhO9HSXrQA29KamtmXZq21EDDmVk3SUdIujf8bJIOkDQx3KTmfR6//ydKOjDcHshZZtZG0r6S7pMkd9/o7ivE7zm2PkWSWphZkaSWkhaJ33PkOXefImlZjeSG/n4fIulFd1/m7sslvajaFQK1bO3Be1dJ85I+zw/TgLwWdiUbIOktSZ3dfVG4arGkzuF77n/kq5sl/U5SLPzcQdIKd68MPyffy4n7PFy/MtweyGU7SyqX9EA4POReM2slfs+xFXH3BZKul/SVgqB9paQZ4vccW6eG/n5v1u/61h68A1sdM9tG0hOSznP3b5PXefDsR57/iLxlZkdKWuLuM7JdFuA7VCRpoKS73H2ApDXa1MVSEr/nyH9hF+BRCiqrdpDUShm0LAL57rv8/d7ag/cFkronfe4WpgF5ycyKFQTuD7v7k2Hy1/Huk+HrkjCd+x/5aG9JR5vZHAVDnQ5QMDa4bdjtUqp+Lyfu83B9G0lLm7LAwGaYL2m+u78Vfp6oIJjn9xxbk5GSZrt7ubtXSHpSwW88v+fYGjX093uzfte39uB9mqSe4ayWJQomyXgmy2UCNks47us+SR+7+41Jq56RFJ+h8jRJTyelnxrOcjlU0sqk7jxATnL3i9y9m7v3UPCb/Yq7nyRpkqTjws1q3ufx+/+4cHtaK5HT3H2xpHlm1itMOlDSR+L3HFuXryQNNbOW4b9h4vc5v+fYGjX09/t5SQebWbuwl8rBYVq9bGv/mzCzwxWMn4xKut/dr8luiYDNY2b7SHpV0vvaNBb4YgXj3h+XtKOkuZJOcPdl4X8ob1fQRW2tpNPdfXqTFxzYTGY2QtJv3f1IM/uegpb49pLekXSyu28ws+aSxiuYA2KZpB+5+5dZKjKQMTMrUzApY4mkLyWdrqBRhd9zbDXM7ApJoxU8MecdSWcqGNfL7znylpk9KmmEpI6SvlYwa/xTauDvt5mdoeDf8pJ0jbs/kPbYW3vwDgAAAABAvtvau80DAAAAAJD3CN4BAAAAAMhxBO8AAAAAAOQ4gncAAAAAAHIcwTsAAAAAADmO4B0AAAAAgBxH8A4AAAAAQI77//KaFeEc5orRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1224x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "f, ax = plt.subplots(figsize = (17, 4))\n",
    "ax.plot(df_words['counts'])\n",
    "ax.set_title('Frequency distribution of ' + str(len(unq)) + ' tokens')\n",
    "ax.set_xlim(-20, 1000)\n",
    "ax.axvline(argmax_second_der, color = 'red', ls = '--')\n",
    "ax.annotate('Threshold selected according to Elbow Rule(argmax of 2nd derivative)',(argmax_second_der + 10, 4000), size = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}