chapter_6.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fec9c63d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from collections import defaultdict\n",
    "import re\n",
    "from nltk.corpus import *\n",
    "import random\n",
    "from nltk.classify import apply_features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d959091",
   "metadata": {},
   "source": [
    "__☼ Using any of the three classifiers described in this chapter, and any features you can think of, build the best name gender classifier you can. Begin by splitting the Names Corpus into three subsets: 500 words for the test set, 500 words for the dev-test set, and the remaining 6900 words for the training set. Then, starting with the example name gender classifier, make incremental improvements. Use the dev-test set to check your progress. Once you are satisfied with your classifier, check its final performance on the test set. How does the performance on the test set compare to the performance on the dev-test set? Is this what you'd expect?__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "208bc7b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "labeled_names = ([(name, 'male') for name in names.words('male.txt')] +\n",
    "                [(name, 'female') for name in names.words('female.txt')])\n",
    "random.shuffle(labeled_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "894b4ba2",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = labeled_names[:len(labeled_names)-1000]\n",
    "dev_set = labeled_names[-1000:-500]\n",
    "test_set = labeled_names[-500:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7f3c2dde",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set is 6944\n",
      "Dev set is 500\n",
      "Test set is 500\n"
     ]
    }
   ],
   "source": [
    "print(\"Train set is {}\\nDev set is {}\\nTest set is {}\".format(len(train_set), len(dev_set), len(test_set)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "586b20f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def gender_features(name):\n",
    "    features = {}\n",
    "    features[\"first_letter\"] = name[0].lower()\n",
    "    features[\"last_letter\"] = name[-1].lower()\n",
    "    for letter in 'abcdefghijklmnopqrstuvwxyz':\n",
    "        features[\"count({})\".format(letter)] = name.lower().count(letter)\n",
    "        features[\"has({})\".format(letter)] = (letter in name.lower())\n",
    "    features[\"vowels\"] = len(re.findall(r'[aeoui]', name.lower()))  \n",
    "    features[\"consonants\"] = len(re.findall(r'[aeoui]', name.lower())) - len(name)\n",
    "    features[\"last_two\"] = name[-2:].lower()\n",
    "    features[\"last_three\"] = name[-3:].lower()\n",
    "    features[\"length\"] = len(name)\n",
    "    features[\"double\"] = len(re.findall(r'(\\w)\\1', name))\n",
    "    \n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "5d2ca99d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = apply_features(gender_features, train_set)\n",
    "dev_data = apply_features(gender_features, dev_set)\n",
    "test_data = apply_features(gender_features, test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "dfa522bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_gender = nltk.NaiveBayesClassifier.train(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "9cdbbfd0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.78"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# First model from the book had an accuracy of 0.74\n",
    "# After adding features \"vowel\", \"consonant\" the accuracy is 0.748\n",
    "# After adding feature \"last_two\", \"last_three\" the accuracy is 0.774, and 7/10 of most infomrative features is \"last_two\"\n",
    "# Accuracy 0.78 after \"length\" and \"double\" features\n",
    "nltk.classify.accuracy(model_gender, dev_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "f292c56a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "                last_two = 'na'           female : male   =     95.4 : 1.0\n",
      "                last_two = 'la'           female : male   =     70.4 : 1.0\n",
      "                last_two = 'ia'           female : male   =     53.5 : 1.0\n",
      "             last_letter = 'k'              male : female =     43.7 : 1.0\n",
      "             last_letter = 'a'            female : male   =     35.0 : 1.0\n",
      "                last_two = 'sa'           female : male   =     31.1 : 1.0\n",
      "                last_two = 'us'             male : female =     27.3 : 1.0\n",
      "                last_two = 'ch'             male : female =     26.0 : 1.0\n",
      "                last_two = 'do'             male : female =     24.9 : 1.0\n",
      "              last_three = 'ana'          female : male   =     24.3 : 1.0\n"
     ]
    }
   ],
   "source": [
    "model_gender.show_most_informative_features(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "94a20025",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.776"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.classify.accuracy(model_gender, test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07ff7747",
   "metadata": {},
   "source": [
    "__☼ The Senseval 2 Corpus contains data intended to train word-sense disambiguation classifiers. It contains data for four words: hard, interest, line, and serve. Choose one of these four words, and load the corresponding data:__\n",
    "\n",
    " \t\n",
    "> from nltk.corpus import senseval\n",
    "\n",
    "> instances = senseval.instances('hard.pos')\n",
    "\n",
    "> size = int(len(instances) * 0.1)\n",
    "\n",
    "> train_set, test_set = instances[size:], instances[:size]\n",
    "\n",
    "Using this dataset, build a classifier that predicts the correct sense tag for a given instance. See the corpus HOWTO at https://www.nltk.org/_modules/nltk/corpus/reader/senseval.html for information on using the instance objects returned by the Senseval 2 Corpus."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "ec38b45e",
   "metadata": {},
   "outputs": [],
   "source": [
    "instances = senseval.instances('hard.pos')\n",
    "size = int(len(instances) * 0.1)\n",
    "idx = list(range(len(instances)))\n",
    "random.shuffle(idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "df64b260",
   "metadata": {},
   "outputs": [],
   "source": [
    "# random.shuffle does not directly shuffle a senseval instances, so I did it this way. \n",
    "# Maybe, I will come back later and do it in a more elegant manner\n",
    "shuffled = []\n",
    "for id in idx:\n",
    "    shuffled.append(instances[id])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "4c6c43ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_hard, test_hard = shuffled[size:], shuffled[:size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "67068aa6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3900, 433)"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_hard), len(test_hard)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "id": "572d81c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({('HARD1',), ('HARD2',), ('HARD3',)}, {('HARD1',), ('HARD2',), ('HARD3',)})"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#some sanity check\n",
    "test = set()\n",
    "train = set()\n",
    "for ins in train_hard:\n",
    "    train.add(ins.senses)\n",
    "\n",
    "for ins in test_hard:\n",
    "    test.add(ins.senses)\n",
    "\n",
    "test, train    # Yes, each set has all three senses of the word \"hard\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "72c67086",
   "metadata": {},
   "outputs": [],
   "source": [
    "def hard_features(instance):\n",
    "    features = {}\n",
    "    features[\"tag_0\"] = instance.context[instance.position][1] \n",
    "    if instance.position !=0 :\n",
    "        features[\"tag_0-1\"] = instance.context[instance.position-1][1]\n",
    "        features[\"word_0-1\"] = instance.context[instance.position-1][0]\n",
    "    else:\n",
    "        features[\"tag_0-1\"] = \"<START>\"\n",
    "        features[\"word_0-1\"] = \"<START>\"\n",
    "        \n",
    "    if instance.position != len(instance.context) - 1:\n",
    "        features[\"tag_0+1\"] = instance.context[instance.position+1][1]\n",
    "        features[\"word_0+1\"] = instance.context[instance.position+1][0]\n",
    "    else:\n",
    "        features[\"tag_0+1\"] = \"<END>\"\n",
    "        features[\"word_0+1\"] = \"<END>\"\n",
    "    \n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "8526d9a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_hard_data = [(hard_features(ins), ins.senses) for ins in train_hard]\n",
    "test_hard_data = [(hard_features(ins), ins.senses) for ins in test_hard]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "09648d43",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_hard = nltk.NaiveBayesClassifier.train(train_hard_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "id": "bbd0b2bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9006928406466512"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# When trained only on tags 1 step away the target word, the accuracy was around 77%\n",
    "# yet adding words 1 step away the target word increased accuracy up to 90%\n",
    "nltk.classify.accuracy(model_hard, test_hard_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb4c5a1b",
   "metadata": {},
   "source": [
    "__☼ Using the movie review document classifier discussed in this chapter, generate a list of the 30 features that the classifier finds to be most informative. Can you explain why these particular features are informative? Do you find any of them surprising?__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "id": "2cfdaa04",
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = [(list(movie_reviews.words(fileid)), category)\n",
    "             for category in movie_reviews.categories()\n",
    "             for fileid in movie_reviews.fileids(category)]\n",
    "random.shuffle(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "id": "88e9f8ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_words = nltk.FreqDist(w.lower() for w in movie_reviews.words())\n",
    "word_features = list(all_words)[:2000]\n",
    "\n",
    "def document_features(document):\n",
    "    document_words = set(document)\n",
    "    features = {}\n",
    "    for word in word_features:\n",
    "        features['contains({})'.format(word)] = (word in document_words)\n",
    "    for bigram in nltk.bigrams(document):\n",
    "        features['bigrams({})'.format(bigram)] = (bigram in nltk.bigrams(document))\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "0d9c5deb",
   "metadata": {},
   "outputs": [],
   "source": [
    "featuresets = [(document_features(d), c) for (d,c) in documents]\n",
    "train_set_movie, test_set_movie = featuresets[100:], featuresets[:100]\n",
    "classifier = nltk.NaiveBayesClassifier.train(train_set_movie)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "id": "9e0e3bb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "bigrams(('not', 'funny')) = True              neg : pos    =     17.8 : 1.0\n",
      "bigrams(('is', 'terrific')) = True              pos : neg    =     16.9 : 1.0\n",
      "bigrams(('and', 'boring')) = True              neg : pos    =     14.4 : 1.0\n",
      "   contains(outstanding) = True              pos : neg    =     13.8 : 1.0\n",
      "bigrams(('a', 'boring')) = True              neg : pos    =     13.1 : 1.0\n",
      " bigrams(('&', 'robin')) = True              neg : pos    =     12.4 : 1.0\n",
      "bigrams(('batman', '&')) = True              neg : pos    =     12.4 : 1.0\n",
      "bigrams(('quite', 'frankly')) = True              neg : pos    =     12.4 : 1.0\n",
      "bigrams(('.', 'cameron')) = True              pos : neg    =     12.3 : 1.0\n",
      " bigrams(('our', 'own')) = True              pos : neg    =     12.3 : 1.0\n",
      "bigrams(('works', 'well')) = True              pos : neg    =     12.3 : 1.0\n",
      "  bigrams(('-', 'note')) = True              neg : pos    =     11.7 : 1.0\n",
      "  bigrams(('be', 'fun')) = True              neg : pos    =     11.7 : 1.0\n",
      "bigrams(('insult', 'to')) = True              neg : pos    =     11.7 : 1.0\n",
      "bigrams(('brilliant', ',')) = True              pos : neg    =     11.6 : 1.0\n",
      "bigrams(('enjoyable', ',')) = True              pos : neg    =     11.6 : 1.0\n",
      "bigrams(('portrayed', 'by')) = True              pos : neg    =     11.6 : 1.0\n",
      "bigrams(('well', 'worth')) = True              pos : neg    =     11.6 : 1.0\n",
      "bigrams(('amounts', 'to')) = True              neg : pos    =     11.1 : 1.0\n",
      "bigrams(('an', 'outstanding')) = True              pos : neg    =     10.9 : 1.0\n",
      "bigrams(('and', 'believable')) = True              pos : neg    =     10.9 : 1.0\n",
      "bigrams(('and', 'great')) = True              pos : neg    =     10.9 : 1.0\n",
      "bigrams((',', 'stupid')) = True              neg : pos    =     10.4 : 1.0\n",
      "bigrams(('good', 'actors')) = True              neg : pos    =     10.4 : 1.0\n",
      "bigrams(('.', 'jackie')) = True              pos : neg    =     10.3 : 1.0\n",
      "bigrams(('best', 'supporting')) = True              pos : neg    =     10.3 : 1.0\n",
      "bigrams(('fairy', 'tale')) = True              pos : neg    =     10.3 : 1.0\n",
      "bigrams(('is', 'superb')) = True              pos : neg    =     10.3 : 1.0\n",
      "bigrams(('study', 'of')) = True              pos : neg    =     10.3 : 1.0\n",
      "bigrams(('very', 'similar')) = True              pos : neg    =     10.3 : 1.0\n"
     ]
    }
   ],
   "source": [
    "classifier.show_most_informative_features(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "id": "747580a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.78"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# adding bigrams increased accuracy from 75% yo 78%\n",
    "# as there are cases when unigrams do not work well\n",
    "# for example, the first informative feature\n",
    "# I could also try adding more features and maybe working with trigrams\n",
    "# but I want to finish the book asap. Later, I will come back to this question\n",
    "\n",
    "nltk.classify.accuracy(classifier, test_set_movie)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4083d8a6",
   "metadata": {},
   "source": [
    "The features (based on unigrams) do not seem to be surprising as the majority of them are positive or negative words such as \"wonderful\"/\"superb\" or \"poorly\"/\"mess\". The only surprising thing, yet still expected from a model trained with feature \"contain_word\" is that there are some names of actors in the list. \n",
    "\n",
    "The features after adding bigrams are similar to those of unigrams, as both contain names of actors. Yet, I am surprised that \"batman &\" got a negative tag! In addition there are some bigrams that do not really make sense out of context, such as \"our own\" or \"- note\" "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf0b3f0a",
   "metadata": {},
   "source": [
    "__☼ Select one of the classification tasks described in this chapter, such as name gender detection, document classification, part-of-speech tagging, or dialog act classification. Using the same training and test data, and the same feature extractor, build three classifiers for the task: a decision tree, a naive Bayes classifier, and a Maximum Entropy classifier. Compare the performance of the three classifiers on your selected task. How do you think that your results might be different if you used a different feature extractor?__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5c403e9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "posts = nltk.corpus.nps_chat.xml_posts()[:10000]\n",
    "\n",
    "def dialogue_act_features(post):\n",
    "    features = {}\n",
    "    for word in nltk.word_tokenize(post):\n",
    "        features['contains({})'.format(word.lower())] = True\n",
    "    return features\n",
    "\n",
    "featuresets = [(dialogue_act_features(post.text), post.get('class')) for post in posts]\n",
    "size = int(len(featuresets) * 0.1)\n",
    "train_set_dialogue, test_set_dialogue = featuresets[size:], featuresets[:size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "id": "21948959",
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier_bayes = nltk.NaiveBayesClassifier.train(train_set_dialogue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "id": "c653fc58",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  ==> Training (100 iterations)\n",
      "\n",
      "      Iteration    Log Likelihood    Accuracy\n",
      "      ---------------------------------------\n",
      "             1          -2.70805        0.050\n",
      "             2          -1.25297        0.847\n",
      "             3          -0.92154        0.881\n",
      "             4          -0.74995        0.898\n",
      "             5          -0.63694        0.910\n",
      "             6          -0.55420        0.918\n",
      "             7          -0.49058        0.924\n",
      "             8          -0.44065        0.929\n",
      "             9          -0.40112        0.932\n",
      "            10          -0.36951        0.937\n",
      "            11          -0.34385        0.940\n",
      "            12          -0.32262        0.943\n",
      "            13          -0.30470        0.946\n",
      "            14          -0.28932        0.948\n",
      "            15          -0.27594        0.950\n",
      "            16          -0.26415        0.952\n",
      "            17          -0.25366        0.953\n",
      "            18          -0.24425        0.954\n",
      "            19          -0.23575        0.956\n",
      "            20          -0.22801        0.957\n",
      "            21          -0.22095        0.958\n",
      "            22          -0.21445        0.959\n",
      "            23          -0.20846        0.960\n",
      "            24          -0.20291        0.960\n",
      "            25          -0.19775        0.961\n",
      "            26          -0.19293        0.962\n",
      "            27          -0.18843        0.962\n",
      "            28          -0.18420        0.963\n",
      "            29          -0.18023        0.963\n",
      "            30          -0.17648        0.964\n",
      "            31          -0.17294        0.965\n",
      "            32          -0.16960        0.965\n",
      "            33          -0.16642        0.965\n",
      "            34          -0.16341        0.966\n",
      "            35          -0.16054        0.966\n",
      "            36          -0.15780        0.966\n",
      "            37          -0.15520        0.967\n",
      "            38          -0.15270        0.968\n",
      "            39          -0.15032        0.968\n",
      "            40          -0.14804        0.968\n",
      "            41          -0.14585        0.969\n",
      "            42          -0.14375        0.969\n",
      "            43          -0.14173        0.969\n",
      "            44          -0.13979        0.969\n",
      "            45          -0.13792        0.970\n",
      "            46          -0.13612        0.970\n",
      "            47          -0.13438        0.970\n",
      "            48          -0.13271        0.971\n",
      "            49          -0.13109        0.971\n",
      "            50          -0.12953        0.971\n",
      "            51          -0.12801        0.971\n",
      "            52          -0.12655        0.972\n",
      "            53          -0.12513        0.972\n",
      "            54          -0.12376        0.972\n",
      "            55          -0.12243        0.972\n",
      "            56          -0.12114        0.973\n",
      "            57          -0.11988        0.973\n",
      "            58          -0.11866        0.974\n",
      "            59          -0.11748        0.974\n",
      "            60          -0.11633        0.974\n",
      "            61          -0.11521        0.974\n",
      "            62          -0.11412        0.975\n",
      "            63          -0.11306        0.975\n",
      "            64          -0.11203        0.975\n",
      "            65          -0.11103        0.975\n",
      "            66          -0.11004        0.975\n",
      "            67          -0.10909        0.976\n",
      "            68          -0.10816        0.976\n",
      "            69          -0.10725        0.976\n",
      "            70          -0.10636        0.976\n",
      "            71          -0.10549        0.976\n",
      "            72          -0.10464        0.977\n",
      "            73          -0.10381        0.977\n",
      "            74          -0.10300        0.977\n",
      "            75          -0.10221        0.977\n",
      "            76          -0.10144        0.977\n",
      "            77          -0.10068        0.977\n",
      "            78          -0.09994        0.977\n",
      "            79          -0.09921        0.977\n",
      "            80          -0.09850        0.977\n",
      "            81          -0.09781        0.977\n",
      "            82          -0.09712        0.977\n",
      "            83          -0.09645        0.978\n",
      "            84          -0.09580        0.978\n",
      "            85          -0.09516        0.978\n",
      "            86          -0.09453        0.978\n",
      "            87          -0.09391        0.978\n",
      "            88          -0.09330        0.978\n",
      "            89          -0.09271        0.978\n",
      "            90          -0.09212        0.978\n",
      "            91          -0.09155        0.978\n",
      "            92          -0.09099        0.978\n",
      "            93          -0.09043        0.978\n",
      "            94          -0.08989        0.978\n",
      "            95          -0.08935        0.978\n",
      "            96          -0.08883        0.978\n",
      "            97          -0.08831        0.978\n",
      "            98          -0.08781        0.978\n",
      "            99          -0.08731        0.978\n",
      "         Final          -0.08682        0.978\n"
     ]
    }
   ],
   "source": [
    "classifier_maxent = nltk.MaxentClassifier.train(train_set_dialogue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "id": "ca79220c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# classifier_tree = nltk.DecisionTreeClassifier.train(train_set_dialogue)  # takes a very long time to train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "id": "e7c6fdbf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For Naive Bayes the accuracy is 0.667\n",
      "For Maximum Entropy the accuracy is 0.711\n"
     ]
    }
   ],
   "source": [
    "print(\"For Naive Bayes the accuracy is\", nltk.classify.accuracy(classifier_bayes, test_set_dialogue))\n",
    "# print(\"For Decision Tree the accuracy is\", nltk.classify.accuracy(classifier_tree, test_set_dialogue))\n",
    "print(\"For Maximum Entropy the accuracy is\", nltk.classify.accuracy(classifier_maxent, test_set_dialogue))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "873ba35a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Since tree training takes a lot of time in nltk, I will use sklearn's Tree with the same data \n",
    "import numpy as np\n",
    "\n",
    "X = [post.text for post in posts]\n",
    "y = [post.get(\"class\") for post in posts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8e9f4f70",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "X_train, X_test, y_train, y_test = X[:size], X[size:], y[:size], y[size:]\n",
    "cv = CountVectorizer()  # no ngrams ot other params since with nltk I did not use them\n",
    "le = LabelEncoder()\n",
    "\n",
    "X_train_cv = cv.fit_transform(X_train)\n",
    "y_train_le = le.fit_transform(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7835b33f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier()"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tree = DecisionTreeClassifier()\n",
    "tree.fit(X_train_cv, y_train_le)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7220a06a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6365555555555555"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tree.score(cv.transform(X_test), le.transform(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "04d2ca8e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.32      0.20      0.25       295\n",
      "           1       0.51      0.87      0.64       101\n",
      "           2       0.00      0.00      0.00       126\n",
      "           3       0.08      0.09      0.08       120\n",
      "           4       0.92      0.35      0.50      2494\n",
      "           5       0.00      0.00      0.00        29\n",
      "           6       0.81      0.93      0.87      1100\n",
      "           7       0.00      0.00      0.00         0\n",
      "           8       0.02      0.06      0.02        34\n",
      "           9       0.45      0.64      0.52      1922\n",
      "          10       0.91      0.99      0.95      1987\n",
      "          11       0.45      0.33      0.38        91\n",
      "          12       0.75      0.73      0.74       455\n",
      "          13       0.30      0.53      0.38        55\n",
      "          14       0.21      0.50      0.30       191\n",
      "\n",
      "    accuracy                           0.64      9000\n",
      "   macro avg       0.38      0.41      0.38      9000\n",
      "weighted avg       0.72      0.64      0.63      9000\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Raushan\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\Raushan\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "C:\\Users\\Raushan\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1245: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(tree.predict(cv.transform(X_test)), le.transform(y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "2c988b0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7714444444444445"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Just having some fin with diiferent feature extractions and models\n",
    "\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "tfidf = TfidfVectorizer(ngram_range = (1,4), analyzer = \"char_wb\")\n",
    "X_train_tf = tfidf.fit_transform(X_train)\n",
    "\n",
    "my_classifier = OneVsRestClassifier(SGDClassifier(loss = \"log\", penalty = \"l1\", alpha = 0.00001, random_state = 42))\n",
    "my_classifier.fit(X_train_tf, y_train_le)\n",
    "\n",
    "my_classifier.score(tfidf.transform(X_test), le.transform(y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f33c0579",
   "metadata": {},
   "source": [
    "__☼ The synonyms \"strong\" and \"powerful\" pattern differently (try combining them with \"chip\" and \"sales\"). What features are relevant in this distinction? Build a classifier that predicts when each word should be used__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "bbcc487d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(370, 18)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reuter = reuters.sents()\n",
    "len([sent for sent in reuter if \"strong\" in sent]), len([sent for sent in reuter if \"powerful\" in sent])  # not enough data in Reuters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "15cacbd1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(188, 62)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "brown_ = brown.sents()\n",
    "len([sent for sent in brown_ if \"strong\" in sent]), len([sent for sent in brown_ if \"powerful\" in sent])  # not enough data in Brown either"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "1e469532",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(569, 64)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "guten = gutenberg.sents()\n",
    "len([sent for sent in guten if \"strong\" in sent]), len([sent for sent in guten if \"powerful\" in sent])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "7fc6605b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# So, I can combine these three corpuses to have some data to train on\n",
    "strong = [sent for sent in reuter if \"strong\" in sent] + [sent for sent in brown_ if \"strong\" in sent] + [sent for sent in guten if \"strong\" in sent]\n",
    "powerful = [sent for sent in reuter if \"powerful\" in sent] + [sent for sent in brown_ if \"powerful\" in sent] + [sent for sent in guten if \"powerful\" in sent]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "56944083",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1127, 144)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(strong), len(powerful)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "88fcd2ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Not all of these corpuses have tagged sents, so for now I will base my classifier on the words only\n",
    "# Later I will use a trained tagger to tag Reuters and Gutenberg, and train classifier based on the tags of the following word\n",
    "\n",
    "def feats(sent):\n",
    "    features = {}\n",
    "    if \"strong\" in sent:\n",
    "        features[\"next_word\"] = sent[sent.index(\"strong\") + 1]\n",
    "        features[\"prev_word\"] = sent[sent.index(\"strong\") - 1]\n",
    "    else:\n",
    "        features[\"next_word\"] = sent[sent.index(\"powerful\") + 1]\n",
    "        features[\"prev_word\"] = sent[sent.index(\"powerful\") - 1]\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "cb09480b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [(feats(sent), \"strong\") for sent in strong] + [(feats(sent), \"powerful\") for sent in powerful]\n",
    "random.shuffle(data)\n",
    "n = int(len(data) * 0.9)\n",
    "\n",
    "trainData, testData = data[:n], data[n:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "8744b288",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.921875"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_model = nltk.NaiveBayesClassifier.train(trainData)\n",
    "nltk.classify.accuracy(my_model, testData)  # I am quite surprised to see an accuracy of 88% on test data, given the size of my dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "dd6b0d81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "               prev_word = 'more'         powerf : strong =     19.3 : 1.0\n",
      "               prev_word = 'most'         powerf : strong =     19.3 : 1.0\n",
      "               next_word = '.'            strong : powerf =      7.1 : 1.0\n",
      "               next_word = 'central'      powerf : strong =      5.4 : 1.0\n",
      "               next_word = 'influence'    powerf : strong =      5.4 : 1.0\n",
      "               prev_word = 'O'            powerf : strong =      4.2 : 1.0\n",
      "               prev_word = 'sufficiently' powerf : strong =      4.2 : 1.0\n",
      "               prev_word = 'them'         powerf : strong =      4.2 : 1.0\n",
      "               prev_word = 'unusually'    powerf : strong =      4.2 : 1.0\n",
      "               prev_word = 'what'         powerf : strong =      4.2 : 1.0\n",
      "               prev_word = 'this'         powerf : strong =      4.2 : 1.0\n",
      "               next_word = 'as'           strong : powerf =      3.4 : 1.0\n",
      "               prev_word = 'very'         strong : powerf =      3.3 : 1.0\n",
      "               next_word = 'act'          powerf : strong =      3.3 : 1.0\n",
      "               next_word = 'argument'     powerf : strong =      3.3 : 1.0\n",
      "               next_word = 'arms'         powerf : strong =      3.3 : 1.0\n",
      "               next_word = 'consolation'  powerf : strong =      3.3 : 1.0\n",
      "               next_word = 'divine'       powerf : strong =      3.3 : 1.0\n",
      "               next_word = 'incentive'    powerf : strong =      3.3 : 1.0\n",
      "               next_word = 'nation'       powerf : strong =      3.3 : 1.0\n"
     ]
    }
   ],
   "source": [
    "# Let us see the informative features it uses\n",
    "my_model.show_most_informative_features(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2236df75",
   "metadata": {},
   "source": [
    "Okay, now I get why the accuracy is high. The word _powerful_ in comparative needs _more_ before it and _most_ in superlative."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1575276b",
   "metadata": {},
   "source": [
    "__◑ The dialog act classifier assigns labels to individual posts, without considering the context in which the post is found. However, dialog acts are highly dependent on context, and some sequences of dialog act are much more likely than others. For example, a ynQuestion dialog act is much more likely to be answered by a yanswer than by a greeting. Make use of this fact to build a consecutive classifier for labeling dialog acts. Be sure to consider what features might be useful. See the code for the consecutive classifier for part-of-speech tags in 1.7 to get some ideas.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "09afe20f",
   "metadata": {},
   "outputs": [],
   "source": [
    "featureset_consecutive = []\n",
    "\n",
    "for idx, post in enumerate(list(posts)):\n",
    "    feature_consecutive = {}\n",
    "    for word in nltk.word_tokenize(post.text):\n",
    "        feature_consecutive['contains({})'.format(word.lower())] = True\n",
    "    if idx != 0: feature_consecutive[\"prev_post\"] = posts[idx-1].get(\"class\")\n",
    "    featureset_consecutive.append((feature_consecutive, posts[idx].get('class')))\n",
    "    \n",
    "size = int(len(featureset_consecutive) * 0.1)\n",
    "train_set_consecutive, test_set_consecutive = featureset_consecutive[size:], featureset_consecutive[:size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "79f68c0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier_consecutive = nltk.NaiveBayesClassifier.train(train_set_consecutive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "8110103d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.653"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.classify.accuracy(classifier_consecutive, test_set_consecutive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "3e68e3b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "            contains(hi) = True            Greet : System =    408.2 : 1.0\n",
      "             contains(>) = True            Other : System =    384.6 : 1.0\n",
      "         contains(empty) = True            Other : System =    339.4 : 1.0\n",
      "          contains(part) = True           System : Statem =    302.0 : 1.0\n",
      "           contains(brb) = True              Bye : Statem =    300.4 : 1.0\n",
      "            contains(no) = True           nAnswe : System =    262.3 : 1.0\n",
      "             contains(<) = True            Other : Greet  =    249.2 : 1.0\n",
      "           contains(yes) = True           yAnswe : Emotio =    242.5 : 1.0\n",
      "             contains(0) = True            Other : Statem =    199.4 : 1.0\n",
      "           contains(are) = True           whQues : System =    198.6 : 1.0\n",
      "            contains(na) = True           ynQues : System =    167.4 : 1.0\n",
      "            contains(ok) = True           Accept : System =    165.6 : 1.0\n",
      "           contains(lol) = True           Emotio : System =    154.3 : 1.0\n",
      "            contains(tc) = True              Bye : Statem =    152.8 : 1.0\n",
      "          contains(what) = True           whQues : Emotio =    143.0 : 1.0\n",
      "           contains(wan) = True           ynQues : System =    141.9 : 1.0\n",
      "         contains(where) = True           whQues : System =    136.3 : 1.0\n",
      "         contains(right) = True           Accept : System =    127.1 : 1.0\n",
      "           contains(and) = True           Contin : Emotio =    123.2 : 1.0\n",
      "             contains(u) = True           whQues : System =    123.1 : 1.0\n"
     ]
    }
   ],
   "source": [
    "classifier_consecutive.show_most_informative_features(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8141cb77",
   "metadata": {},
   "source": [
    "__◑ Word features can be very useful for performing document classification, since the words that appear in a document give a strong indication about what its semantic content is. However, many words occur very infrequently, and some of the most informative words in a document may never have occurred in our training data. One solution is to make use of a lexicon, which describes how different words relate to one another. Using WordNet lexicon, augment the movie review document classifier presented in this chapter to use features that generalize the words that appear in a document, making it more likely that they will match words found in the training data.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "8c2e7bc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = [(list(movie_reviews.words(fileid)), category)\n",
    "             for category in movie_reviews.categories()\n",
    "             for fileid in movie_reviews.fileids(category)]\n",
    "random.shuffle(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "42f8dcfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_words = nltk.FreqDist(w.lower() for w in movie_reviews.words())\n",
    "word_lemmas = [nltk.stem.WordNetLemmatizer().lemmatize(w) for w in list(all_words)[:2000]]\n",
    "\n",
    "def document_features_lemmas(document):\n",
    "    document_words = set([nltk.stem.WordNetLemmatizer().lemmatize(w) for w in document])\n",
    "    features = {}\n",
    "    for word in word_lemmas:\n",
    "        features['contains({})'.format(word)] = (word in document_words)\n",
    "    for bigram in nltk.bigrams(document):\n",
    "        features['bigrams({})'.format(bigram)] = (bigram in nltk.bigrams(document_words))\n",
    "        \n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "6464ba21",
   "metadata": {},
   "outputs": [],
   "source": [
    "featuresets_lemmas = [(document_features_lemmas(d), c) for (d,c) in documents]\n",
    "train_set_movie_lemmas, test_set_movie_lemmas = featuresets_lemmas[100:], featuresets_lemmas[:100]\n",
    "classifier_lemmas = nltk.NaiveBayesClassifier.train(train_set_movie_lemmas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "8550c240",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.84"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compared to 0.78 when thew ords were not lemmatized. I also want to see waht happens if I use PorterStemmer\n",
    "nltk.classify.accuracy(classifier_lemmas, test_set_movie_lemmas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "37c7cb49",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "bigrams(('is', 'terrific')) = False             pos : neg    =     17.2 : 1.0\n",
      "bigrams(('not', 'funny')) = False             neg : pos    =     16.8 : 1.0\n",
      "bigrams(('the', 'political')) = False             pos : neg    =     16.5 : 1.0\n",
      "bigrams(('and', 'boring')) = False             neg : pos    =     14.2 : 1.0\n",
      " bigrams(('our', 'own')) = False             pos : neg    =     13.1 : 1.0\n",
      "bigrams(('well', 'worth')) = False             pos : neg    =     12.5 : 1.0\n",
      "bigrams(('insult', 'to')) = False             neg : pos    =     12.2 : 1.0\n",
      "bigrams(('fairy', 'tale')) = False             pos : neg    =     11.8 : 1.0\n",
      "bigrams(('quite', 'frankly')) = False             neg : pos    =     11.5 : 1.0\n",
      "bigrams(('.', 'cameron')) = False             pos : neg    =     11.1 : 1.0\n"
     ]
    }
   ],
   "source": [
    "classifier_lemmas.show_most_informative_features()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5873301",
   "metadata": {},
   "source": [
    "__★ The PP Attachment Corpus is a corpus describing prepositional phrase attachment decisions. Each instance in the corpus is encoded as a PPAttachment object__\n",
    "\n",
    "Select only the instances where inst.attachment is N. Using this sub-corpus, build a classifier that attempts to predict which preposition is used to connect a given pair of nouns. For example, given the pair of nouns \"team\" and \"researchers,\" the classifier should predict the preposition \"of\". See the corpus HOWTO at https://www.nltk.org/howto/corpus.html#ppattach or https://www.nltk.org/_modules/nltk/corpus/reader/ppattach.html for more information on using the PP attachment corpus."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "bc946f4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "noun_pp = [pp for pp in ppattach.attachments(\"training\") if pp.attachment == \"N\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "b8ac823d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PPAttachment(sent='1', verb='is', noun1='chairman', prep='of', noun2='N.V.', attachment='N'),\n",
       " PPAttachment(sent='2', verb='named', noun1='director', prep='of', noun2='conglomerate', attachment='N'),\n",
       " PPAttachment(sent='3', verb='caused', noun1='percentage', prep='of', noun2='deaths', attachment='N'),\n",
       " PPAttachment(sent='9', verb='is', noun1='asbestos', prep='in', noun2='products', attachment='N'),\n",
       " PPAttachment(sent='12', verb='led', noun1='team', prep='of', noun2='researchers', attachment='N')]"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "noun_pp[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "874cb6a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pp_features(inst):\n",
    "    features = {}\n",
    "    features[\"N1\"] = nltk.stem.WordNetLemmatizer().lemmatize(inst.noun1)    # the only informative feature available is N1\n",
    "    # features[\"N2\"] = nltk.stem.WordNetLemmatizer().lemmatize(inst.noun2)  # it does not affect the accuracy much\n",
    "    # features[\"V\"] = nltk.stem.WordNetLemmatizer().lemmatize(inst.verb)    # it deacrses accuracy\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "557bcc22",
   "metadata": {},
   "outputs": [],
   "source": [
    "featureset_pp = [(pp_features(inst), inst.prep) for inst in noun_pp]\n",
    "size_pp = int(len(featureset_pp) * 0.9)\n",
    "train_pp, test_pp = featureset_pp[:size], featureset_pp[size:]\n",
    "my_model_pp = nltk.NaiveBayesClassifier.train(train_pp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "d696c8e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5575266092245311"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For higher accuracy I need more information than just two noun words before and after\n",
    "nltk.classify.accuracy(my_model_pp, test_pp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "4ace47fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "                      N1 = 'stake'            in : of     =     11.8 : 1.0\n",
      "                      N1 = 'interest'         in : of     =      6.8 : 1.0\n",
      "                      N1 = 'executive'      with : of     =      6.1 : 1.0\n",
      "                      N1 = '%'                of : to     =      5.7 : 1.0\n",
      "                      N1 = 'million'          in : for    =      5.6 : 1.0\n",
      "                      N1 = 'one'              of : for    =      4.6 : 1.0\n",
      "                      N1 = 'stock'            in : of     =      4.4 : 1.0\n",
      "                      N1 = 'increase'         in : of     =      3.4 : 1.0\n",
      "                      N1 = 'billion'          in : of     =      3.2 : 1.0\n",
      "                      N1 = 'growth'           in : of     =      3.1 : 1.0\n"
     ]
    }
   ],
   "source": [
    "my_model_pp.show_most_informative_features()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbb18b7d",
   "metadata": {},
   "source": [
    "__★ Suppose you wanted to automatically generate a prose description of a scene, and already had a word to uniquely describe each entity, such as _the jar_, and simply wanted to decide whether to use \"in\" or \"on\" in relating various items, e.g. _the book is in the cupboard_ vs _the book is on the shelf_. Explore this issue by looking at corpus data; writing programs as needed.__\n",
    "\n",
    "a.\t\tin the car versus on the train\n",
    "\n",
    "b.\t\tin town versus on campus\n",
    "\n",
    "c.\t\tin the picture versus on the screen\n",
    "\n",
    "d.\t\tin Macbeth versus on Letterman\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "cef62e1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# I did not quite get the question, so I will just make a model to predict \"in\" or \"on\" given a word\n",
    "corpora = brown.words() + reuters.words()\n",
    "in_on_tuples = [(w, corpora[idx + 1], corpora[idx+2]) for idx, w in enumerate(list(corpora)) if (w == \"in\" or w == \"on\") and (corpora[idx+1] in [\"the\", \"a\", \"an\"])]\n",
    "in_on_tuples += [(w, corpora[idx + 1]) for idx, w in enumerate(list(corpora)) if (w == \"in\" or w == \"on\") and (corpora[idx+1] not in [\"the\", \"a\", \"an\"])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "ceb42012",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('in', 'the', 'hard-fought'),\n",
       " ('in', 'the', 'election'),\n",
       " ('on', 'a', 'number'),\n",
       " ('in', 'the', 'Fulton'),\n",
       " ('in', 'the', 'state'),\n",
       " ('in', 'the', 'future'),\n",
       " ('on', 'the', 'Fulton'),\n",
       " ('in', 'the', 'appointment'),\n",
       " ('in', 'a', 'manner'),\n",
       " ('on', 'the', 'petition')]"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "in_on_tuples[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "6b37524f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature_in_on(tuple_):\n",
    "    features = {}\n",
    "    if len(tuple_) == 2:\n",
    "        features[\"word\"] = tuple_[1]\n",
    "    if len(tuple_) == 3:\n",
    "        features[\"word\"] = tuple_[2]\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "id": "4db408b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "featureset_in_on = [(feature_in_on(t), t[0]) for t in in_on_tuples]\n",
    "size_in_on = int(len(featureset_in_on) * 0.9)\n",
    "train_in_on, test_in_on = featureset_in_on[:size], featureset_in_on[size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "eca5e4d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_in_on = nltk.NaiveBayesClassifier.train(train_in_on)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "id": "586cc6ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7435837571916951"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.classify.accuracy(model_in_on, test_in_on)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "34efb4b8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "                    word = 'past'             in : on     =      4.7 : 1.0\n",
      "                    word = 'Congo'            in : on     =      3.9 : 1.0\n",
      "                    word = 'national'         on : in     =      3.8 : 1.0\n",
      "                    word = \"President's\"      on : in     =      3.8 : 1.0\n",
      "                    word = 'subject'          on : in     =      3.8 : 1.0\n",
      "                    word = 'last'             in : on     =      3.5 : 1.0\n",
      "                    word = 'world'            in : on     =      3.5 : 1.0\n",
      "                    word = 'market'           on : in     =      2.9 : 1.0\n",
      "                    word = 'matter'           on : in     =      2.7 : 1.0\n",
      "                    word = 'second'           on : in     =      2.7 : 1.0\n"
     ]
    }
   ],
   "source": [
    "model_in_on.show_most_informative_features()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "id": "c70288ff",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'in'"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_in_on.classify({\"word\": \"car\"})"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}