chapter_7.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d82c71e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from collections import defaultdict\n",
    "import re\n",
    "from nltk.corpus import *\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68b86b33",
   "metadata": {},
   "source": [
    "__☼ Write a tag pattern to match noun phrases containing plural head nouns, e.g. \"many/JJ researchers/NNS\", \"two/CD weeks/NNS\", \"both/DT new/JJ positions/NNS\". Try to do this by generalizing the tag pattern that handled singular noun phrases.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "1895209a",
   "metadata": {},
   "outputs": [],
   "source": [
    "nltk.app.chunkparser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "f0e0c680",
   "metadata": {},
   "outputs": [],
   "source": [
    "grammar = r\"\"\"\n",
    "  NP:   {(<PRP\\$|POS>)<JJS?|VBG|CD|VBN>?<NN(S|P)?>+}\n",
    "        {<RBR>?<DT|PP\\$|PRP\\$>?<RB|JJR>?<JJ|VBG|CD|VBN>*<NN(S|P)?>+}\n",
    "        {<PRP>}\n",
    "\"\"\"\n",
    "cp = nltk.RegexpParser(grammar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5184b288",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(S (NP many/JJ researchers/NNS))\n",
      "(S (NP two/CD weeks/NNS))\n",
      "(S (NP both/DT new/JJ positions/NNS))\n"
     ]
    }
   ],
   "source": [
    "sentences = [[(\"many\", \"JJ\"), (\"researchers\", \"NNS\")], [(\"two\", \"CD\"), (\"weeks\", \"NNS\")], [(\"both\", \"DT\"), (\"new\", \"JJ\"), (\"positions\", \"NNS\")]]\n",
    "for sent in sentences:\n",
    "    print(cp.parse(sent))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a194cb75",
   "metadata": {},
   "source": [
    "__☼ Pick one of the three chunk types in the CoNLL corpus. Inspect the CoNLL corpus and try to observe any patterns in the POS tag sequences that make up this kind of chunk. Develop a simple chunker using the regular expression chunker nltk.RegexpParser. Discuss any tag sequences that are difficult to chunk reliably.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e90ee7b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "conll_train_NP = conll2000.chunked_sents('train.txt', chunk_types=['NP'])\n",
    "conll_test_NP = conll2000.chunked_sents('test.txt', chunk_types=['NP'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "440362c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(NP Confidence/NN)\n",
      "(NP the/DT pound/NN)\n",
      "(NP another/DT sharp/JJ dive/NN)\n",
      "(NP trade/NN figures/NNS)\n",
      "(NP September/NNP)\n",
      "(NP release/NN)\n",
      "(NP tomorrow/NN)\n",
      "(NP a/DT substantial/JJ improvement/NN)\n",
      "(NP July/NNP and/CC August/NNP)\n",
      "(NP 's/POS near-record/JJ deficits/NNS)\n",
      "(NP the/DT Exchequer/NNP)\n",
      "(NP Nigel/NNP Lawson/NNP)\n",
      "(NP 's/POS restated/VBN commitment/NN)\n",
      "(NP a/DT firm/NN monetary/JJ policy/NN)\n",
      "(NP a/DT freefall/NN)\n",
      "(NP sterling/NN)\n",
      "(NP the/DT past/JJ week/NN)\n",
      "(NP analysts/NNS)\n",
      "(NP underlying/VBG support/NN)\n",
      "(NP sterling/NN)\n",
      "(NP the/DT chancellor/NN)\n",
      "(NP 's/POS failure/NN)\n",
      "(NP any/DT new/JJ policy/NN measures/NNS)\n",
      "(NP his/PRP$ Mansion/NNP House/NNP speech/NN)\n",
      "(NP last/JJ Thursday/NNP)\n",
      "(NP This/DT)\n",
      "(NP the/DT risk/NN)\n",
      "(NP the/DT government/NN)\n",
      "(NP base/NN rates/NNS)\n",
      "(NP 16/CD %/NN)\n",
      "(NP their/PRP$ current/JJ 15/CD %/NN level/NN)\n",
      "(NP the/DT pound/NN)\n",
      "(NP economists/NNS)\n",
      "(NP foreign/JJ exchange/NN market/NN analysts/NNS)\n",
      "(NP The/DT risks/NNS)\n",
      "(NP sterling/NN)\n",
      "(NP a/DT bad/JJ trade/NN figure/NN)\n",
      "(NP the/DT down/JJ side/NN)\n",
      "(NP Chris/NNP Dillow/NNP)\n",
      "(NP senior/JJ U.K./NNP economist/NN)\n",
      "(NP Nomura/NNP Research/NNP Institute/NNP)\n",
      "(NP there/EX)\n",
      "(NP another/DT bad/JJ trade/NN number/NN)\n",
      "(NP there/EX)\n",
      "(NP an/DT awful/JJ lot/NN)\n",
      "(NP pressure/NN)\n",
      "(NP Simon/NNP Briscoe/NNP)\n",
      "(NP U.K./NNP economist/NN)\n",
      "(NP Midland/NNP Montagu/NNP)\n",
      "(NP a/DT unit/NN)\n",
      "(NP Midland/NNP Bank/NNP PLC/NNP)\n",
      "(NP Forecasts/NNS)\n",
      "(NP the/DT trade/NN figures/NNS)\n",
      "(NP few/JJ economists/NNS)\n",
      "(NP the/DT data/NNS)\n",
      "(NP a/DT very/RB marked/VBN improvement/NN)\n",
      "(NP deficit/NN)\n",
      "(NP the/DT current/JJ account/NN)\n",
      "(NP August/NNP)\n",
      "(NP The/DT August/NNP deficit/NN)\n",
      "(NP the/DT #/# 2.2/CD billion/CD gap/NN)\n",
      "(NP July/NNP)\n",
      "(NP the/DT #/# 2.3/CD billion/CD deficit/NN)\n",
      "(NP October/NNP 1988/CD)\n",
      "(NP Sanjay/NNP Joshi/NNP)\n",
      "(NP European/JJ economist/NN)\n",
      "(NP Baring/NNP Brothers/NNPS &/CC Co./NNP)\n",
      "(NP there/EX)\n",
      "(NP no/DT sign/NN)\n",
      "(NP Britain/NNP)\n",
      "(NP 's/POS manufacturing/NN industry/NN)\n",
      "(NP itself/PRP)\n",
      "(NP exports/NNS)\n",
      "(NP the/DT same/JJ time/NN)\n",
      "(NP he/PRP)\n",
      "(NP the/DT outlook/NN)\n",
      "(NP imports/NNS)\n",
      "(NP\n",
      "  continued/VBD\n",
      "  high/JJ\n",
      "  consumer/NN\n",
      "  and/CC\n",
      "  capital/NN\n",
      "  goods/NNS\n",
      "  inflows/NNS)\n",
      "(NP He/PRP)\n",
      "(NP the/DT current/JJ account/NN deficit/NN)\n",
      "(NP only/RB #/# 1.8/CD billion/CD)\n",
      "(NP September/NNP)\n",
      "(NP Mr./NNP Dillow/NNP)\n",
      "(NP he/PRP)\n",
      "(NP a/DT reduction/NN)\n",
      "(NP raw/JJ material/NN stockbuilding/VBG)\n",
      "(NP industry/NN)\n",
      "(NP a/DT sharp/JJ drop/NN)\n",
      "(NP imports/NNS)\n",
      "(NP some/DT rebound/NN)\n",
      "(NP exports/NNS)\n",
      "(NP August/NNP)\n",
      "(NP 's/POS unexpected/JJ decline/NN)\n",
      "(NP the/DT deficit/NN)\n",
      "(NP as/RB little/JJ as/IN #/# 1.3/CD billion/CD)\n",
      "(NP Mr./NNP Briscoe/NNP)\n",
      "(NP who/WP)\n",
      "(NP a/DT #/# 1.3/CD billion/CD current/JJ account/NN gap/NN)\n",
      "(NP the/DT trade/NN figures/NNS)\n",
      "(NP sterling/NN)\n",
      "(NP the/DT currency/NN)\n",
      "(NP much/JJ)\n",
      "(NP investors/NNS)\n",
      "(NP further/JJ evidence/NN)\n",
      "(NP the/DT turnaround/NN)\n",
      "(NP positions/NNS)\n",
      "(NP he/PRP)\n",
      "(NP No/DT one/PRP)\n",
      "(NP the/DT trade/NN figures/NNS)\n",
      "(NP a/DT flat/JJ position/NN)\n",
      "(NP the/DT pound/NN)\n",
      "(NP overall/JJ evidence/NN)\n",
      "(NP the/DT economy/NN)\n",
      "(NP his/PRP$ Mansion/NNP House/NNP speech/NN)\n",
      "(NP Mr./NNP Lawson/NNP)\n",
      "(NP a/DT further/JJ slowdown/NN)\n",
      "(NP the/DT impact/NN)\n",
      "(NP the/DT last/JJ rise/NN)\n",
      "(NP interest/NN rates/NNS)\n",
      "(NP earlier/RBR this/DT month/NN)\n",
      "(NP effect/NN)\n",
      "(NP U.K./JJ base/NN rates/NNS)\n",
      "(NP their/PRP$ highest/JJS level/NN)\n",
      "(NP eight/CD years/NNS)\n",
      "(NP consumer/NN expenditure/NN data/NNS)\n",
      "(NP Friday/NNP)\n",
      "(NP the/DT U.K./NNP economy/NN)\n",
      "(NP The/DT figures/NNS)\n",
      "(NP spending/NN)\n",
      "(NP 0.1/CD %/NN)\n",
      "(NP the/DT third/JJ quarter/NN)\n",
      "(NP the/DT second/JJ quarter/NN)\n",
      "(NP 3.8/CD %/NN)\n",
      "(NP a/DT year/NN)\n",
      "(NP This/DT)\n",
      "(NP a/DT 1.6/CD %/NN rise/NN)\n",
      "(NP the/DT second/NN)\n",
      "(NP the/DT first/JJ quarter/NN)\n",
      "(NP a/DT 5.4/CD %/NN increase/NN)\n",
      "(NP the/DT second/JJ quarter/NN)\n",
      "(NP 1988/CD)\n",
      "(NP Mr./NNP Dillow/NNP)\n",
      "(NP the/DT data/NNS)\n",
      "(NP the/DT economy/NN)\n",
      "(NP suggestions/NNS)\n",
      "(NP much/NN)\n",
      "(NP the/DT spending/NN)\n",
      "(NP services/NNS)\n",
      "(NP consumer/NN goods/NNS)\n",
      "(NP fears/NNS)\n",
      "(NP more/JJR import/NN rises/NNS)\n",
      "(NP the/DT chancellor/NN)\n",
      "(NP it/PRP)\n",
      "(NP he/PRP)\n",
      "(NP interest/NN rates/NNS)\n",
      "(NP a/DT substantial/JJ slowdown/NN)\n",
      "(NP place/NN)\n",
      "(NP sterling/NN)\n",
      "(NP Thursday/NNP)\n",
      "(NP he/PRP)\n",
      "(NP his/PRP$ audience/NN)\n",
      "(NP the/DT government/NN)\n",
      "(NP the/DT necessary/JJ rigor/NN)\n",
      "(NP monetary/JJ policy/NN)\n",
      "(NP exchange/NN rate/NN weakness/NN)\n",
      "(NP Analysts/NNS)\n",
      "(NP there/EX)\n",
      "(NP little/JJ)\n",
      "(NP sterling/NN)\n",
      "(NP the/DT moment/NN)\n",
      "(NP Mr./NNP Lawson/NNP)\n",
      "(NP 's/POS promise/NN)\n",
      "(NP rates/NNS)\n",
      "(NP they/PRP)\n",
      "(NP any/DT further/JJ drop/NN)\n",
      "(NP the/DT government/NN)\n",
      "(NP 's/POS popularity/NN)\n",
      "(NP this/DT promise/NN)\n",
      "(NP Sterling/NNP)\n",
      "(NP some/DT signs/NNS)\n",
      "(NP a/DT lack/NN)\n",
      "(NP confidence/NN)\n",
      "(NP Mr./NNP Lawson/NNP)\n",
      "(NP 's/POS promise/NN)\n",
      "(NP Friday/NNP)\n",
      "(NP European/JJ trading/NN)\n",
      "(NP it/PRP)\n",
      "(NP $/$ 1.5890/CD)\n",
      "(NP 2.9495/CD marks/NNS)\n",
      "(NP $/$ 1.5940/CD)\n",
      "(NP 2.9429/CD marks/NNS)\n",
      "(NP late/JJ Thursday/NNP)\n",
      "(NP Economists/NNS)\n",
      "(NP the/DT pound/NN)\n",
      "(NP much/JJ)\n",
      "(NP 2.90/CD marks/NNS)\n",
      "(NP the/DT government/NN)\n",
      "(NP rates/NNS)\n",
      "(NP 16/CD %/NN)\n",
      "(NP any/DT further/JJ decline/NN)\n",
      "(NP the/DT balance/NN)\n",
      "(NP monetary/JJ policy/NN)\n",
      "(NP Friday/NNP)\n"
     ]
    }
   ],
   "source": [
    "# let us take a look at all NP in first 30 tree\n",
    "for tree in conll_train_NP[:30]:\n",
    "    for num in range(len(tree) - 1):\n",
    "        try:\n",
    "            tree[num].label()\n",
    "        except AttributeError:\n",
    "            pass\n",
    "        else:\n",
    "            print(tree[num])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "1cb17c8c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ChunkParse score:\n",
      "    IOB Accuracy:  88.3%%\n",
      "    Precision:     82.8%%\n",
      "    Recall:        76.9%%\n",
      "    F-Measure:     79.7%%\n"
     ]
    }
   ],
   "source": [
    "print(cp.evaluate(conll_test_NP))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2376c460",
   "metadata": {},
   "source": [
    "__☼ An early definition of chunk was the material that occurs between chinks. Develop a chunker that starts by putting the whole sentence in a single chunk, and then does the rest of its work solely by chinking. Determine which tags (or tag sequences) are most likely to make up chinks with the help of your own utility program. Compare the performance and simplicity of this approach relative to a chunker based entirely on chunk rules.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "91c85910",
   "metadata": {},
   "outputs": [],
   "source": [
    "grammar_chink = r\"\"\"\n",
    "  NP:\n",
    "    {<.*>+}          \n",
    "    }<VBG><VBN>{\n",
    "    }<RB>+<VB(P|Z|D|G|N)?|IN|TO|MD>{\n",
    "    }<VB(P|Z|D|G|N)?><JJ>{\n",
    "    }<IN|TO|VB(P|Z|D)?|MD|CC|.|,|WDT|``|''|WRB>+{\n",
    "    <NN(P|S)>}{<POS>\n",
    "    <CD>}{<DT>\n",
    "    \n",
    "  \"\"\"\n",
    "cp_chink = nltk.RegexpParser(grammar_chink)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "459b17a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ChunkParse score:\n",
      "    IOB Accuracy:  86.4%%\n",
      "    Precision:     68.2%%\n",
      "    Recall:        72.9%%\n",
      "    F-Measure:     70.5%%\n"
     ]
    }
   ],
   "source": [
    "print(cp_chink.evaluate(conll_test_NP))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d71ef3fd",
   "metadata": {},
   "source": [
    "__◑ Write a tag pattern to cover noun phrases that contain gerunds, e.g. \"the/DT receiving/VBG end/NN\", \"assistant/NN managing/VBG editor/NN\". Add these patterns to the grammar, one per line. Test your work using some tagged sentences of your own devising.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "e28f73e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# already done, my RegexP chunker above accounts for possible VBGs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c18bd43c",
   "metadata": {},
   "source": [
    "__◑ Write one or more tag patterns to handle coordinated noun phrases, e.g. \"July/NNP and/CC August/NNP\", \"all/DT your/PRP$ managers/NNS and/CC supervisors/NNS\", \"company/NN courts/NNS and/CC adjudicators/NNS\".__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "2052543c",
   "metadata": {},
   "outputs": [],
   "source": [
    "grammar_2 = r\"\"\"\n",
    "  NP:   {(<PRP\\$|POS>)<JJS?|VBG|CD|VBN>?<NN(S|P)?>+}\n",
    "        {<RBR>?<DT|PP\\$|PRP\\$>?<RB|JJR>?<JJ|VBG|CD|VBN>*<NN(S|P)?>+}\n",
    "        <NN(P|S)?>{}<CC><NN(P|S)?>\n",
    "\"\"\"\n",
    "cp_2 = nltk.RegexpParser(grammar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "f1a62694",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ChunkParse score:\n",
      "    IOB Accuracy:  86.6%%\n",
      "    Precision:     81.6%%\n",
      "    Recall:        70.4%%\n",
      "    F-Measure:     75.6%%\n"
     ]
    }
   ],
   "source": [
    "print(cp_2.evaluate(conll_test_NP))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67353fc2",
   "metadata": {},
   "source": [
    "__◑ Carry out the following evaluation tasks for any of the chunkers you have developed earlier. (Note that most chunking corpora contain some internal inconsistencies, such that any reasonable rule-based approach will produce errors.)__\n",
    "\n",
    "Evaluate your chunker on 100 sentences from a chunked corpus, and report the precision, recall and F-measure.\n",
    "\n",
    "Use the chunkscore.missed() and chunkscore.incorrect() methods to identify the errors made by your chunker. Discuss.\n",
    "\n",
    "Compare the performance of your chunker to the baseline chunker discussed in the evaluation section of this chapter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "17a8f768",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number 1 was already done above\n",
    "chunkscore = nltk.ChunkScore()\n",
    "unchunked = conll2000.chunked_sents('test.txt', chunk_types=[''])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "7ae93845",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall 0.2823061630218688\n",
      "Precision 0.2823061630218688\n",
      "F-score 0.2823061630218688\n"
     ]
    }
   ],
   "source": [
    "chunkscore.score(conll_test_NP, ([cp.parse(sent) for sent in unchunked]))\n",
    "print(\"Recall\", chunkscore.recall())\n",
    "print(\"Precision\", chunkscore.precision())\n",
    "print(\"F-score\", chunkscore.f_measure())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "184e5a6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1444, 1444)"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(chunkscore.incorrect()), len(chunkscore.missed())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "42337e65",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(S\n",
      "  In/IN\n",
      "  (NP the/DT 1988/CD third/JJ quarter/NN)\n",
      "  ,/,\n",
      "  (NP the/DT forest-products/NNS company/NN)\n",
      "  reported/VBD\n",
      "  (NP profit/NN)\n",
      "  of/IN\n",
      "  $/$\n",
      "  144.9/CD\n",
      "  million/CD\n",
      "  ,/,\n",
      "  or/CC\n",
      "  (NP 69/CD cents/NNS)\n",
      "  (NP a/DT share/NN)\n",
      "  ./.)\n"
     ]
    }
   ],
   "source": [
    "print(cp.parse(chunkscore.incorrect()[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "f222e8f9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(S\n",
      "  Without/IN\n",
      "  (NP the/DT gain/NN)\n",
      "  ,/,\n",
      "  (NP operating/VBG profit/NN)\n",
      "  was/VBD\n",
      "  (NP $/$ 64/CD million/CD)\n",
      "  ,/,\n",
      "  or/CC\n",
      "  (NP 71/CD cents/NNS)\n",
      "  (NP a/DT share/NN)\n",
      "  ./.)\n"
     ]
    }
   ],
   "source": [
    "print(cp.parse(chunkscore.missed()[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "258ba2ad",
   "metadata": {},
   "source": [
    "__◑ Develop a chunker for one of the chunk types in the CoNLL corpus using a regular-expression based chunk grammar RegexpChunk. Use any combination of rules for chunking, chinking, merging or splitting.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "eb9836fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "grammar = r\"\"\"\n",
    "  NP:   {(<PRP\\$|POS>)<JJS?|VBG|CD|VBN>?<NN(S|P)?>+}\n",
    "        {<RBR>?<DT|PP\\$|PRP\\$>?<RB|JJR>?<JJ|VBG|CD|VBN>*<NN(S|P)?>+}\n",
    "        {<PRP>}\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6c48262",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "56fd76cb",
   "metadata": {},
   "source": [
    "__◑ Sometimes a word is incorrectly tagged, e.g. the head noun in \"12/CD or/CC so/RB cases/VBZ\". Instead of requiring manual correction of tagger output, good chunkers are able to work with the erroneous output of taggers. Look for other examples of correctly chunked noun phrases with incorrect tags.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "1f3018da",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(NP a/DT couple/NN of/IN dozen/NN cases/VBZ)\n",
      "(NP only/RB about/IN 20/CD cases/VBZ)\n",
      "(NP 50/CD to/TO 75/CD cases/VBZ)\n",
      "(NP only/RB 12/CD or/CC so/RB cases/VBZ)\n",
      "(NP closes/VBZ to/TO $/$ 15/CD billion/CD)\n",
      "(NP weaker/JJR housing/NN starts/VBZ)\n",
      "(NP estimates/VBZ)\n",
      "(NP what/WP changes/VBZ)\n",
      "(NP operating/VBG results/VBZ)\n",
      "(NP offers/VBZ)\n",
      "(NP 's/VBZ)\n",
      "(NP the/DT production/NN runs/VBZ)\n",
      "(NP Newsprint/NN and/CC postage/NN prices/VBZ)\n",
      "(NP other/JJ commodity/NN markets/VBZ)\n",
      "(NP employees/VBZ)\n",
      "(NP Sell/VB stops/VBZ)\n",
      "(NP buy/VB stops/VBZ)\n",
      "(NP commissions/VBZ and/CC fees/NNS)\n",
      "(NP back/RB taxes/VBZ)\n",
      "(NP leads/VBZ)\n",
      "(NP admits/VBZ)\n",
      "(NP admits/VBZ)\n",
      "(NP numbers/VBZ)\n",
      "(NP nursing-home/NN stays/VBZ)\n",
      "(NP longer/RB hours/VBZ)\n",
      "(NP base/NN rates/VBZ)\n",
      "(NP prices/VBZ)\n",
      "(NP seven/CD or/CC eight/CD times/VBZ)\n",
      "(NP those/DT ``/`` plain/JJ vanilla/NN ''/'' funds/VBZ)\n",
      "(NP stocks/VBZ)\n",
      "(NP 2,387,226/CD shares/VBZ)\n",
      "(NP 1,000/CD bicycle/NN and/CC motorbike/NN tires/VBZ)\n",
      "(NP calls/VBZ)\n",
      "(NP rivets/VBZ)\n",
      "(NP back/RB taxes/VBZ)\n",
      "(NP increases/VBZ)\n",
      "(NP '/POS hopes/VBZ)\n",
      "(NP turns/VBZ)\n",
      "(NP 's/POS faces/VBZ)\n",
      "(NP prices/VBZ)\n",
      "(NP markets/VBZ)\n",
      "(NP Frequent/JJ trading/NN runs/VBZ)\n",
      "(NP stocks/VBZ)\n",
      "(NP stocks/VBZ)\n",
      "(NP times/VBZ)\n",
      "(NP hopes/VBZ)\n",
      "(NP rumors/VBZ)\n",
      "(NP banks/VBZ)\n",
      "(NP 24/CD hours/VBZ)\n",
      "(NP the/DT 50/CD or/CC so/RB projects/VBZ)\n",
      "(NP all/PDT those/DT offers/VBZ)\n",
      "(NP higher/JJR spreads/VBZ)\n",
      "(NP stocks/VBZ)\n",
      "(NP three/CD times/VBZ)\n",
      "(NP banks/VBZ)\n",
      "(NP rate/NN rises/VBZ)\n",
      "(NP telephone/NN calls/VBZ)\n",
      "(NP toll/NN calls/VBZ)\n",
      "(NP doubts/VBZ)\n",
      "(NP all/DT stores/VBZ)\n",
      "(NP 's/POS totals/VBZ)\n",
      "(NP no/DT regrets/VBZ)\n",
      "(NP 's/POS likes/VBZ and/CC dislikes/VBZ)\n",
      "(NP 's/POS likes/VBZ and/CC dislikes/VBZ)\n",
      "(NP the/DT Fox/NNP shows/VBZ)\n",
      "(NP shows/VBZ)\n",
      "(NP polls/VBZ)\n",
      "(NP telephone/NN calls/VBZ)\n",
      "(NP back/RB bonuses/VBZ)\n",
      "(NP back/RB bonuses/VBZ)\n",
      "(NP bonuses/VBZ)\n",
      "(NP calls/VBZ)\n",
      "(NP Even/RB claims/VBZ)\n",
      "(NP flashlights/NNS and/CC fire/NN alarms/VBZ)\n",
      "(NP hopes/VBZ)\n",
      "(NP points/VBZ)\n",
      "(NP rates/VBZ)\n",
      "(NP any/DT means/VBZ)\n",
      "(NP three/CD times/VBZ the/DT current/JJ minimum/NN)\n",
      "(NP 24/CD hours/VBZ)\n",
      "(NP three/CD tracks/VBZ)\n",
      "(NP prices/VBZ)\n",
      "(NP telephone/NN calls/VBZ)\n",
      "(NP the/DT stiffest/JJS sanctions/VBZ)\n",
      "(NP Puts/VBZ)\n",
      "(NP almost/RB 15/CD times/VBZ the/DT $/$ 557,000/CD)\n",
      "(NP Canadian/NNP Pacific/NNP and/CC Soo/NNP Line/NNP tracks/VBZ)\n",
      "(NP harder/JJR ``/`` leads/VBZ)\n",
      "(NP loan/NN sales/VBZ)\n",
      "(NP some/DT sounds/VBZ)\n",
      "(NP supports/VBZ)\n",
      "(NP housing/NN starts/VBZ)\n",
      "(NP rates/VBZ)\n",
      "(NP building/NN permits/VBZ)\n",
      "(NP The/DT housing/NN starts/VBZ numbers/NNS)\n",
      "(NP the/DT home/NN runs/VBZ)\n",
      "(NP two/CD occasions/VBZ)\n",
      "(NP means/VBZ)\n",
      "(NP enough/RB votes/VBZ)\n",
      "(NP bulls/NNS and/CC bears/VBZ)\n",
      "(NP Bridge/NNP and/CC highway/NN collapses/VBZ)\n",
      "(NP some/DT runs/VBZ)\n",
      "(NP five/CD times/VBZ the/DT size/NN)\n",
      "(NP 40/CD times/VBZ the/DT size/NN)\n",
      "(NP damages/VBZ)\n",
      "(NP 'S/VBZ poor/JJ)\n",
      "(NP matching/VBG funds/VBZ)\n",
      "(NP calls/VBZ)\n",
      "(NP Attacks/VBZ)\n",
      "(NP what/WP weights/VBZ)\n",
      "(NP calls/VBZ)\n",
      "(NP IBM/NNP shares/VBZ)\n",
      "(NP shares/VBZ)\n",
      "(NP fits/VBZ)\n",
      "(NP the/DT British/JJ luxury-car/NN maker/NN 's/VBZ)\n",
      "(NP 'S/VBZ)\n",
      "(NP calls/VBZ)\n",
      "(NP all/DT calls/VBZ)\n",
      "(NP the/DT damage/NN totals/VBZ)\n",
      "(NP loans/VBZ)\n",
      "(NP growing/VBG calls/VBZ)\n",
      "(NP roughly/RB 3.7/CD million/CD USAir/NNP shares/VBZ)\n",
      "(NP even/RB results/VBZ)\n",
      "(NP damage/NN claims/VBZ)\n",
      "(NP the/DT disaster-contingency/NN plans/VBZ)\n"
     ]
    }
   ],
   "source": [
    "for tree in conll_train_NP:\n",
    "    for num in range(len(tree) - 1):\n",
    "        try:\n",
    "            tree[num].label()\n",
    "        except AttributeError:\n",
    "            pass\n",
    "        else:\n",
    "            for (w, tag) in (tree[num]):\n",
    "                if tag == \"VBZ\":\n",
    "                    print(tree[num])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4abba46d",
   "metadata": {},
   "source": [
    "__◑ The bigram chunker scores about 90% accuracy. Study its errors and try to work out why it doesn't get 100% accuracy. Experiment with trigram chunking. Are you able to improve the performance any more?__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "d574dfbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BigramChunker(nltk.ChunkParserI):\n",
    "    def __init__(self, train_sents):\n",
    "        train_data = [[(t,c) for w,t,c in nltk.chunk.tree2conlltags(sent)]\n",
    "                      for sent in train_sents]\n",
    "        self.tagger = nltk.BigramTagger(train_data)\n",
    "\n",
    "    def parse(self, sentence):\n",
    "        pos_tags = [pos for (word,pos) in sentence]\n",
    "        tagged_pos_tags = self.tagger.tag(pos_tags)\n",
    "        chunktags = [chunktag for (pos, chunktag) in tagged_pos_tags]\n",
    "        conlltags = [(word, pos, chunktag) for ((word,pos),chunktag)\n",
    "                     in zip(sentence, chunktags)]\n",
    "        return nltk.chunk.conlltags2tree(conlltags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "f1ed751e",
   "metadata": {},
   "outputs": [],
   "source": [
    "conll_train = conll2000.chunked_sents('train.txt')\n",
    "conll_test = conll2000.chunked_sents('test.txt')\n",
    "conll_test_unchunked = conll2000.chunked_sents('test.txt', chunk_types = [\"\"])\n",
    "\n",
    "bigram_chunker = BigramChunker(conll_train)\n",
    "chunkscore_2 = nltk.ChunkScore()\n",
    "chunkscore_2.score(conll_test, ([bigram_chunker.parse(s) for s in conll_test_unchunked]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "3646c3b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.21520874751491054"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chunkscore_2.precision()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "56ecdf45",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(S\n",
      "  (NP Scott/NNP Paper/NNP Co./NNP)\n",
      "  (VP said/VBD)\n",
      "  (NP it/PRP)\n",
      "  (VP is/VBZ abandoning/VBG)\n",
      "  (NP\n",
      "    a/DT\n",
      "    proposed/VBN\n",
      "    $/$\n",
      "    650/CD\n",
      "    million/CD\n",
      "    tree-farming/JJ\n",
      "    project/NN)\n",
      "  (PP in/IN)\n",
      "  (NP Indonesia/NNP)\n",
      "  (PP because/IN)\n",
      "  (NP it/PRP)\n",
      "  no/RB\n",
      "  longer/RB\n",
      "  (VP expects/VBZ to/TO use/VB)\n",
      "  (PP as/IN)\n",
      "  (NP much/JJ eucalyptus/NN pulp/NN)\n",
      "  (PP as/IN)\n",
      "  (NP previously/RB anticipated/VBN)\n",
      "  ./.)\n"
     ]
    }
   ],
   "source": [
    "print(chunkscore_2.incorrect()[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aced89e0",
   "metadata": {},
   "source": [
    "__★ We saw in 5. that it is possible to establish an upper limit to tagging performance by looking for ambiguous n-grams, n-grams that are tagged in more than one possible way in the training data. Apply the same method to determine an upper bound on the performance of an n-gram chunker.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d10424d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cff3b8d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "2444d1a3",
   "metadata": {},
   "source": [
    "__★ Pick one of the three chunk types in the CoNLL corpus. Write functions to do the following tasks for your chosen type:__\n",
    "\n",
    "List all the tag sequences that occur with each instance of this chunk type.\n",
    "\n",
    "Count the frequency of each tag sequence, and produce a ranked list in order of decreasing frequency; each line should consist of an integer (the frequency) and the tag sequence.\n",
    "\n",
    "Inspect the high-frequency tag sequences. Use these as the basis for developing a better chunker."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "id": "60cd1e7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# A dictionary where key is the index of a tree on corpus, \n",
    "# the value is another dictionary where its key is the index of NP in a tree\n",
    "# and its value is the list of tags of that particular NP\n",
    "tags = defaultdict(lambda : defaultdict(lambda : list()))\n",
    "\n",
    "for idx_tree, tree in enumerate(list(conll_train_NP)):\n",
    "    for num in range(len(tree) - 1):\n",
    "        try:\n",
    "            tree[num].label()\n",
    "        except AttributeError:\n",
    "            pass\n",
    "        else:\n",
    "            for (w, tag) in tree[num]:\n",
    "                tags[idx_tree][num].append(tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "id": "f0e0a875",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['DT', 'NN']\n",
      "(NP the/DT pound/NN)\n"
     ]
    }
   ],
   "source": [
    "print(tags[0][2])\n",
    "print(conll_train_NP[0][2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "id": "c1c8982d",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_tagset = []\n",
    "for tree_num, dict_second in tags.items():\n",
    "    for tuple_num, tagset in dict_second.items():\n",
    "        final_tagset.append(tuple(tagset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "a772a6a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_NP_freq = nltk.FreqDist(final_tagset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "a51afbb9",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('DT', 'NN'), 7222),\n",
       " (('PRP',), 3802),\n",
       " (('NNS',), 3278),\n",
       " (('NN',), 3242),\n",
       " (('NNP',), 3242),\n",
       " (('NNP', 'NNP'), 2639),\n",
       " (('DT', 'JJ', 'NN'), 2119),\n",
       " (('JJ', 'NNS'), 1717),\n",
       " (('DT', 'NNS'), 1173),\n",
       " (('JJ', 'NN'), 1143),\n",
       " (('NN', 'NNS'), 1012),\n",
       " (('WDT',), 930),\n",
       " (('DT', 'NN', 'NN'), 921),\n",
       " (('CD',), 866),\n",
       " (('CD', 'NN'), 827),\n",
       " (('$', 'CD', 'CD'), 823),\n",
       " (('CD', 'NNS'), 688),\n",
       " (('NNP', 'NNP', 'NNP'), 671),\n",
       " (('PRP$', 'NN'), 624),\n",
       " (('POS', 'NN'), 550)]"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_NP_freq.most_common(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "id": "5a9cd34e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Regex based only on 20 most_common tag sequences\n",
    "grammar_on_freq = r\"\"\"\n",
    "  NP:   {<DT|POS|PRP\\$|CD>?<JJ>*<NN(S|P)?>+}\n",
    "        {<PRP>}\n",
    "        {<$>?<CD>+}\n",
    "        {<WDT>}\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "cp_on_freq = nltk.RegexpParser(grammar_on_freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "id": "84a6ce17",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ChunkParse score:\n",
      "    IOB Accuracy:  88.6%%\n",
      "    Precision:     77.9%%\n",
      "    Recall:        79.2%%\n",
      "    F-Measure:     78.6%%\n"
     ]
    }
   ],
   "source": [
    "# Regex based only on 20 most freq tag sequences, scores quite good. \n",
    "# It has higher recall score, than the first parser I wrote above, which is not surprising\n",
    "# This could be extended to 30 or 40 most_freq tag sequences and evaluated on conll_test_NP\n",
    "print(cp_on_freq.evaluate(conll_test_NP))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "69d52003",
   "metadata": {},
   "source": [
    "__★ The baseline chunker presented in the evaluation section tends to create larger chunks than it should. For example, the phrase: [every/DT time/NN] [she/PRP] sees/VBZ [a/DT newspaper/NN] contains two consecutive chunks, and our baseline chunker will incorrectly combine the first two: [every/DT time/NN she/PRP]. Write a program that finds which of these chunk-internal tags typically occur at the start of a chunk, then devise one or more rules that will split up these chunks. Combine these with the existing baseline chunker and re-evaluate it, to see if you have discovered an improved baseline.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "id": "ae664405",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(NP release/NN) (NP tomorrow/NN)\n",
      "(NP July/NNP and/CC August/NNP) (NP 's/POS near-record/JJ deficits/NNS)\n",
      "(NP the/DT Exchequer/NNP) (NP Nigel/NNP Lawson/NNP)\n",
      "(NP Nigel/NNP Lawson/NNP) (NP 's/POS restated/VBN commitment/NN)\n",
      "(NP the/DT chancellor/NN) (NP 's/POS failure/NN)\n",
      "(NP his/PRP$ Mansion/NNP House/NNP speech/NN) (NP last/JJ Thursday/NNP)\n",
      "(NP Britain/NNP) (NP 's/POS manufacturing/NN industry/NN)\n",
      "(NP August/NNP) (NP 's/POS unexpected/JJ decline/NN)\n",
      "(NP interest/NN rates/NNS) (NP earlier/RBR this/DT month/NN)\n",
      "(NP Mr./NNP Lawson/NNP) (NP 's/POS promise/NN)\n",
      "(NP the/DT government/NN) (NP 's/POS popularity/NN)\n",
      "(NP Mr./NNP Lawson/NNP) (NP 's/POS promise/NN)\n",
      "(NP 's/POS promise/NN) (NP Friday/NNP)\n",
      "(NP European/JJ trading/NN) (NP it/PRP)\n",
      "(NP 2.9429/CD marks/NNS) (NP late/JJ Thursday/NNP)\n",
      "(NP Friday/NNP) (NP 's/POS Market/NNP Activity/NN)\n",
      "(NP the/DT coming/VBG week/NN) (NP the/DT foreign/JJ exchange/NN market/NN)\n",
      "(NP 1.8470/CD marks/NNS) (NP late/JJ Thursday/NNP)\n",
      "(NP New/NNP York/NNP) (NP late/JJ Thursday/NNP)\n",
      "(NP Friday/NNP) (NP 's/POS Tokyo/NNP close/NN)\n",
      "(NP $/$ 367.30/CD) (NP an/DT ounce/NN)\n",
      "(NP Hong/NNP Kong/NNP) (NP Monday/NNP)\n",
      "(NP $/$ 366.50/CD) (NP an/DT ounce/NN)\n",
      "(NP $/$ 40/CD) (NP a/DT share/NN)\n",
      "(NP A.P./NNP Green/NNP) (NP 's/POS board/NN)\n",
      "(NP $/$ 35.50/CD to/TO $/$ 36.50/CD) (NP a/DT share/NN)\n",
      "(NP Both/DT) (NP Westwood/NNP Brick/NNP)\n",
      "(NP about/IN 4/CD %/NN) (NP this/DT year/NN)\n",
      "(NP 30/CD %/NN) (NP who/WP)\n",
      "(NP railroads/NNS and/CC trucking/NN companies/NNS) (NP that/WDT)\n",
      "(NP overcapacity/NN) (NP they/PRP)\n",
      "(NP the/DT price/NN cutting/VBG) (NP that/WDT)\n",
      "(NP its/PRP$ rates/NNS) (NP 5.3/CD %/NN)\n",
      "(NP 5.3/CD %/NN) (NP late/JJ this/DT year/NN)\n",
      "(NP another/DT) (NP as/RB much/JJ)\n",
      "(NP about/IN 20/CD %/NN) (NP earlier/RBR this/DT month/NN)\n",
      "(NP its/PRP$ rates/NNS) (NP a/DT further/JJ 25/CD %/NN)\n",
      "(NP Salomon/NNP) (NP 's/POS Mr./NNP Lloyd/NNP)\n",
      "(NP rates/NNS) (NP more/JJR than/IN 10/CD %/NN)\n",
      "(NP Europe/NNP) (NP last/JJ September/NNP)\n",
      "(NP Asia/NNP) (NP about/IN 10/CD %/NN)\n",
      "(NP us/PRP) (NP what/WP)\n",
      "(NP Mr./NNP Stone/NNP) (NP 's/POS case/NN)\n",
      "(NP the/DT court/NN) (NP the/DT full/JJ picture/NN)\n"
     ]
    }
   ],
   "source": [
    "for tree in conll_train_NP[:100]:\n",
    "    for num in range(len(tree) - 1):\n",
    "        try:\n",
    "            tree[num].label()\n",
    "            tree[num + 1].label()\n",
    "        except AttributeError:\n",
    "            pass\n",
    "        else:\n",
    "            print(tree[num], tree[num+1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "id": "b816b6f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# I will continue improving my regex based on freq of tag sequences\n",
    "\n",
    "grammar_on_freq_split = r\"\"\"\n",
    "  NP:   {<DT|POS|PRP\\$|CD>?<JJ>*<NN(S|P)?>+}\n",
    "        {<PRP>}\n",
    "        {<$>?<CD>+}\n",
    "        {<WDT>}\n",
    "        <NN(P|S)?>}{<POS>\n",
    "        <NN(P|S)?>}{<PRP|JJ>\n",
    "        <CD|NN(P|S)?>}{<DT><JJ>*<NN(P|S)?>\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "cp_on_freq_split = nltk.RegexpParser(grammar_on_freq_split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "id": "4e61db44",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ChunkParse score:\n",
      "    IOB Accuracy:  88.6%%\n",
      "    Precision:     77.9%%\n",
      "    Recall:        79.2%%\n",
      "    F-Measure:     78.6%%\n"
     ]
    }
   ],
   "source": [
    "print(cp_on_freq_split.evaluate(conll_test_NP))     # no change in metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a9b1020",
   "metadata": {},
   "source": [
    "__★ Develop an NP chunker that converts POS-tagged text into a list of tuples, where each tuple consists of a verb followed by a sequence of noun phrases and prepositions, e.g. the little cat sat on the mat becomes ('sat', 'on', 'NP')...__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e27944f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eef834cd",
   "metadata": {},
   "source": [
    "__★ The Penn Treebank contains a section of tagged Wall Street Journal text that has been chunked into noun phrases. The format uses square brackets, and we have encountered it several times during this chapter. The Treebank corpus can be accessed using: for sent in nltk.corpus.treebank_chunk.chunked_sents(fileid). These are flat trees, just as we got using nltk.corpus.conll2000.chunked_sents().__\n",
    "\n",
    "The functions nltk.tree.pprint() and nltk.chunk.tree2conllstr() can be used to create Treebank and IOB strings from a tree. Write functions chunk2brackets() and chunk2iob() that take a single chunk tree as their sole argument, and return the required multi-line string representation.\n",
    "\n",
    "Write command-line conversion utilities bracket2iob.py and iob2bracket.py that take a file in Treebank or CoNLL format (resp) and convert it to the other format. (Obtain some raw Treebank or CoNLL data from the NLTK Corpora, save it to a file, and then use for line in open(filename) to access it from Python.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "id": "a6aeadb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "treebank = nltk.corpus.treebank_chunk.chunked_sents()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "id": "8f3cb2a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# I saved the code as my_conll_print.py\n",
    "import my_conll_print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "id": "6576fcb8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(S\n",
      "  ''/''\n",
      "  (NP Neither/DT Lorillard/NNP)\n",
      "  nor/CC\n",
      "  (NP the/DT researchers/NNS)\n",
      "  (NP who/WP)\n",
      "  studied/VBD\n",
      "  (NP the/DT workers/NNS)\n",
      "  were/VBD\n",
      "  aware/JJ\n",
      "  of/IN\n",
      "  (NP any/DT research/NN)\n",
      "  on/IN\n",
      "  (NP smokers/NNS)\n",
      "  of/IN\n",
      "  (NP the/DT)\n",
      "  Kent/NNP\n",
      "  (NP cigarettes/NNS)\n",
      "  ./.)\n"
     ]
    }
   ],
   "source": [
    "my_conll_print.pprint_tree(treebank[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "id": "b4b5985c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Pierre NNP B-NP\n",
      "Vinken NNP I-NP\n",
      ", , O\n",
      "61 CD B-NP\n",
      "years NNS I-NP\n",
      "old JJ O\n",
      ", , O\n",
      "will MD O\n",
      "join VB O\n",
      "the DT B-NP\n",
      "board NN I-NP\n",
      "as IN O\n",
      "a DT B-NP\n",
      "nonexecutive JJ I-NP\n",
      "director NN I-NP\n",
      "Nov. NNP I-NP\n",
      "29 CD I-NP\n",
      ". . O\n"
     ]
    }
   ],
   "source": [
    "my_conll_print.pprint_iob(treebank[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a3f6953",
   "metadata": {},
   "source": [
    "__★ An n-gram chunker can use information other than the current part-of-speech tag and the n-1 previous chunk tags. Investigate other models of the context, such as the n-1 previous part-of-speech tags, or some combination of previous chunk tags along with previous and following part-of-speech tags.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "id": "00182763",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConsecutiveNPChunkTagger(nltk.TaggerI):\n",
    "\n",
    "    def __init__(self, train_sents):\n",
    "        train_set = []\n",
    "        for tagged_sent in train_sents:\n",
    "            untagged_sent = nltk.tag.untag(tagged_sent)\n",
    "            history = []\n",
    "            for i, (word, tag) in enumerate(tagged_sent):\n",
    "                featureset = npchunk_features(untagged_sent, i, history)\n",
    "                train_set.append( (featureset, tag) )\n",
    "                history.append(tag)\n",
    "        self.classifier = nltk.MaxentClassifier.train(\n",
    "            train_set, trace=0)\n",
    "\n",
    "    def tag(self, sentence):\n",
    "        history = []\n",
    "        for i, word in enumerate(sentence):\n",
    "            featureset = npchunk_features(sentence, i, history)\n",
    "            tag = self.classifier.classify(featureset)\n",
    "            history.append(tag)\n",
    "        return zip(sentence, history)\n",
    "\n",
    "class ConsecutiveNPChunker(nltk.ChunkParserI):\n",
    "    def __init__(self, train_sents):\n",
    "        tagged_sents = [[((w,t),c) for (w,t,c) in\n",
    "                         nltk.chunk.tree2conlltags(sent)]\n",
    "                        for sent in train_sents]\n",
    "        self.tagger = ConsecutiveNPChunkTagger(tagged_sents)\n",
    "\n",
    "    def parse(self, sentence):\n",
    "        tagged_sents = self.tagger.tag(sentence)\n",
    "        conlltags = [(w,t,c) for ((w,t),c) in tagged_sents]\n",
    "        return nltk.chunk.conlltags2tree(conlltags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b71b38af",
   "metadata": {},
   "outputs": [],
   "source": [
    "def npchunk_features(sentence, i, history):\n",
    "    word, pos = sentence[i]\n",
    "    return {\"pos\": pos}\n",
    "\n",
    "chunker = ConsecutiveNPChunker(conll_train)\n",
    "print(chunker.evaluate(conll_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2aaf3ed2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be511a72",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}