Movie_reviews_detection(NLTK).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NLTK(Natural language tool kit) documentation\n",
    "\n",
    "#### Proccession Human readable content into computer understandable format"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Installation setps\n",
    "- !pip install nltk\n",
    "- import nltk\n",
    "- nltk.download('all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenizers\n",
    "     Two types\n",
    "     1) word_tokenizer - divide text into words\n",
    "     2) sentence_tokenizer- divide text into sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing nltk package\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize,sent_tokenize\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hi', ',', 'this', 'is', 'Srinivasa', 'Rao', 'Thammaneni', '.', 'Studing', 'NLTK', 'package', 'to', 'process', 'Text', 'to', 'mechine', 'understandable', 'format', '.', 'And', 'also', 'working', 'as', 'software', 'engineer', 'in', 'Tata', 'consultancy', 'services', '.', 'yoooyoooo', '.']\n"
     ]
    }
   ],
   "source": [
    "example_text=\"Hi, this is Srinivasa Rao Thammaneni. Studing NLTK package to process Text to mechine understandable format. And also working as software engineer in Tata consultancy services. yoooyoooo.\"\n",
    "print(word_tokenize(example_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hi, this is Srinivasa Rao Thammaneni.', 'Studing NLTK package to process Text to mechine understandable format.', 'And also working as software engineer in Tata consultancy services.', 'yoooyoooo.']\n"
     ]
    }
   ],
   "source": [
    "print(sent_tokenize(example_text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'should', 'both', 'their', 's', \"mightn't\", \"you'd\", 'm', 'nor', 'more', 'aren', 'an', 'weren', 'only', 'between', 'couldn', 'so', 'on', 'with', \"wouldn't\", 'while', 'he', 'my', 'any', 'been', 'until', 'because', \"aren't\", 'but', \"needn't\", 'itself', 'after', 'to', \"shan't\", 'shouldn', 'myself', 'just', 'ourselves', 'her', \"couldn't\", 'during', 'has', 'about', 'i', \"you've\", \"she's\", 'below', \"you're\", 'me', 'off', 'when', \"won't\", 'it', 'its', 'you', 'from', 'yourself', 'than', 'y', 'each', 've', 'mustn', 'once', 'under', 'wouldn', \"weren't\", 'yours', \"that'll\", 'am', 'out', 'such', \"doesn't\", 'have', 'there', 're', 'ma', 'other', 'is', 'here', 'this', 'don', 'down', 'them', \"isn't\", 'or', 'needn', 'his', 'shan', 'what', 'doing', 'by', \"don't\", \"haven't\", 'over', 'hers', 'hasn', 'who', 'won', 'some', 'too', 'can', 'why', 'most', 'she', 'up', \"you'll\", 'no', 'now', 'those', 'was', 'that', 'had', 'then', 'themselves', 'mightn', 'will', 'doesn', 'herself', 'didn', 'himself', 'where', 'the', 'these', 'same', 'wasn', \"it's\", 'haven', 'do', 'against', 'for', 'll', 'your', \"should've\", 'does', 'him', 'were', 'did', 'how', 'and', \"wasn't\", 'isn', 'if', 'at', 'hadn', \"didn't\", 't', 'a', 'd', 'o', 'before', 'own', \"shouldn't\", 'yourselves', 'few', 'be', 'again', \"hadn't\", 'in', 'which', 'into', 'are', 'ain', \"hasn't\", 'theirs', 'ours', 'through', 'we', 'above', 'all', 'our', 'as', 'having', 'very', 'they', 'further', \"mustn't\", 'not', 'being', 'of', 'whom'}\n"
     ]
    }
   ],
   "source": [
    "#importing corpus module\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "#display pre defined stopwords in corpus module\n",
    "print(set(stopwords.words('english')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Part', 'NLTK', 'Corpora', 'WordNet', '.', 'I', 'would', \"n't\", 'totally', 'classify', 'WordNet', 'Corpora', ',', 'anything', 'really', 'giant', 'Lexicon', ',', ',', 'either', 'way', ',', 'super', 'useful', '.', 'With', 'WordNet', 'things', 'like', 'look', 'words', 'meaning', 'according', 'parts', 'speech', ',', 'find', 'synonyms', ',', 'antonyms', ',', 'even', 'examples', 'word', 'use', '.']\n"
     ]
    }
   ],
   "source": [
    "example_info=\"\"\"Part of the NLTK Corpora is WordNet. I wouldn't totally classify WordNet as a Corpora, if anything it is really a giant Lexicon, but, either way, it is super useful. With WordNet we can do things like look up words and their meaning according to their parts of speech, we can find synonyms, antonyms, and even examples of the word in use.\"\"\"\n",
    "\n",
    "words=word_tokenize(example_info)\n",
    "filter_words=[]\n",
    "stop_words=set(stopwords.words('english'))\n",
    "#for w in words:\n",
    " #  print(w)\n",
    "  #  if w not in stop_words:\n",
    "   #     filter_words.append(w)\n",
    "\n",
    "filter_words=[w for w in words if w not in stop_words]\n",
    "print(filter_words)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "python\n",
      "python\n",
      "pythonli\n",
      "python\n",
      "walk\n",
      "ride\n"
     ]
    }
   ],
   "source": [
    "#importing PortStemmer\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "ps=PorterStemmer()\n",
    "example_words=['Python','Pythoning','Pythonly','python']\n",
    "\n",
    "for i in example_words:\n",
    "    print(ps.stem(i))\n",
    "\n",
    "print(ps.stem('Walking'))\n",
    "print(ps.stem('Riding'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chunking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('PRESIDENT', 'NNP'), ('GEORGE', 'NNP'), ('W.', 'NNP'), ('BUSH', 'NNP'), (\"'S\", 'POS'), ('ADDRESS', 'NNP'), ('BEFORE', 'IN'), ('A', 'NNP'), ('JOINT', 'NNP'), ('SESSION', 'NNP'), ('OF', 'IN'), ('THE', 'NNP'), ('CONGRESS', 'NNP'), ('ON', 'NNP'), ('THE', 'NNP'), ('STATE', 'NNP'), ('OF', 'IN'), ('THE', 'NNP'), ('UNION', 'NNP'), ('January', 'NNP'), ('31', 'CD'), (',', ','), ('2006', 'CD'), ('THE', 'NNP'), ('PRESIDENT', 'NNP'), (':', ':'), ('Thank', 'NNP'), ('you', 'PRP'), ('all', 'DT'), ('.', '.')]\n",
      "[('Mr.', 'NNP'), ('Speaker', 'NNP'), (',', ','), ('Vice', 'NNP'), ('President', 'NNP'), ('Cheney', 'NNP'), (',', ','), ('members', 'NNS'), ('of', 'IN'), ('Congress', 'NNP'), (',', ','), ('members', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('Supreme', 'NNP'), ('Court', 'NNP'), ('and', 'CC'), ('diplomatic', 'JJ'), ('corps', 'NN'), (',', ','), ('distinguished', 'JJ'), ('guests', 'NNS'), (',', ','), ('and', 'CC'), ('fellow', 'JJ'), ('citizens', 'NNS'), (':', ':'), ('Today', 'VB'), ('our', 'PRP$'), ('nation', 'NN'), ('lost', 'VBD'), ('a', 'DT'), ('beloved', 'VBN'), (',', ','), ('graceful', 'JJ'), (',', ','), ('courageous', 'JJ'), ('woman', 'NN'), ('who', 'WP'), ('called', 'VBD'), ('America', 'NNP'), ('to', 'TO'), ('its', 'PRP$'), ('founding', 'NN'), ('ideals', 'NNS'), ('and', 'CC'), ('carried', 'VBD'), ('on', 'IN'), ('a', 'DT'), ('noble', 'JJ'), ('dream', 'NN'), ('.', '.')]\n",
      "[('Tonight', 'NN'), ('we', 'PRP'), ('are', 'VBP'), ('comforted', 'VBN'), ('by', 'IN'), ('the', 'DT'), ('hope', 'NN'), ('of', 'IN'), ('a', 'DT'), ('glad', 'JJ'), ('reunion', 'NN'), ('with', 'IN'), ('the', 'DT'), ('husband', 'NN'), ('who', 'WP'), ('was', 'VBD'), ('taken', 'VBN'), ('so', 'RB'), ('long', 'RB'), ('ago', 'RB'), (',', ','), ('and', 'CC'), ('we', 'PRP'), ('are', 'VBP'), ('grateful', 'JJ'), ('for', 'IN'), ('the', 'DT'), ('good', 'JJ'), ('life', 'NN'), ('of', 'IN'), ('Coretta', 'NNP'), ('Scott', 'NNP'), ('King', 'NNP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('President', 'NNP'), ('George', 'NNP'), ('W.', 'NNP'), ('Bush', 'NNP'), ('reacts', 'VBZ'), ('to', 'TO'), ('applause', 'VB'), ('during', 'IN'), ('his', 'PRP$'), ('State', 'NNP'), ('of', 'IN'), ('the', 'DT'), ('Union', 'NNP'), ('Address', 'NNP'), ('at', 'IN'), ('the', 'DT'), ('Capitol', 'NNP'), (',', ','), ('Tuesday', 'NNP'), (',', ','), ('Jan', 'NNP'), ('.', '.')]\n",
      "[('31', 'CD'), (',', ','), ('2006', 'CD'), ('.', '.')]\n",
      "[('White', 'NNP'), ('House', 'NNP'), ('photo', 'NN'), ('by', 'IN'), ('Eric', 'NNP'), ('DraperEvery', 'NNP'), ('time', 'NN'), ('I', 'PRP'), (\"'m\", 'VBP'), ('invited', 'JJ'), ('to', 'TO'), ('this', 'DT'), ('rostrum', 'NN'), (',', ','), ('I', 'PRP'), (\"'m\", 'VBP'), ('humbled', 'VBN'), ('by', 'IN'), ('the', 'DT'), ('privilege', 'NN'), (',', ','), ('and', 'CC'), ('mindful', 'NN'), ('of', 'IN'), ('the', 'DT'), ('history', 'NN'), ('we', 'PRP'), (\"'ve\", 'VBP'), ('seen', 'VBN'), ('together', 'RB'), ('.', '.')]\n",
      "[('We', 'PRP'), ('have', 'VBP'), ('gathered', 'VBN'), ('under', 'IN'), ('this', 'DT'), ('Capitol', 'NNP'), ('dome', 'NN'), ('in', 'IN'), ('moments', 'NNS'), ('of', 'IN'), ('national', 'JJ'), ('mourning', 'NN'), ('and', 'CC'), ('national', 'JJ'), ('achievement', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), ('have', 'VBP'), ('served', 'VBN'), ('America', 'NNP'), ('through', 'IN'), ('one', 'CD'), ('of', 'IN'), ('the', 'DT'), ('most', 'RBS'), ('consequential', 'JJ'), ('periods', 'NNS'), ('of', 'IN'), ('our', 'PRP$'), ('history', 'NN'), ('--', ':'), ('and', 'CC'), ('it', 'PRP'), ('has', 'VBZ'), ('been', 'VBN'), ('my', 'PRP$'), ('honor', 'NN'), ('to', 'TO'), ('serve', 'VB'), ('with', 'IN'), ('you', 'PRP'), ('.', '.')]\n",
      "[('In', 'IN'), ('a', 'DT'), ('system', 'NN'), ('of', 'IN'), ('two', 'CD'), ('parties', 'NNS'), (',', ','), ('two', 'CD'), ('chambers', 'NNS'), (',', ','), ('and', 'CC'), ('two', 'CD'), ('elected', 'JJ'), ('branches', 'NNS'), (',', ','), ('there', 'EX'), ('will', 'MD'), ('always', 'RB'), ('be', 'VB'), ('differences', 'NNS'), ('and', 'CC'), ('debate', 'NN'), ('.', '.')]\n",
      "[('But', 'CC'), ('even', 'RB'), ('tough', 'JJ'), ('debates', 'NNS'), ('can', 'MD'), ('be', 'VB'), ('conducted', 'VBN'), ('in', 'IN'), ('a', 'DT'), ('civil', 'JJ'), ('tone', 'NN'), (',', ','), ('and', 'CC'), ('our', 'PRP$'), ('differences', 'NNS'), ('can', 'MD'), ('not', 'RB'), ('be', 'VB'), ('allowed', 'VBN'), ('to', 'TO'), ('harden', 'VB'), ('into', 'IN'), ('anger', 'NN'), ('.', '.')]\n",
      "[('To', 'TO'), ('confront', 'VB'), ('the', 'DT'), ('great', 'JJ'), ('issues', 'NNS'), ('before', 'IN'), ('us', 'PRP'), (',', ','), ('we', 'PRP'), ('must', 'MD'), ('act', 'VB'), ('in', 'IN'), ('a', 'DT'), ('spirit', 'NN'), ('of', 'IN'), ('goodwill', 'NN'), ('and', 'CC'), ('respect', 'NN'), ('for', 'IN'), ('one', 'CD'), ('another', 'DT'), ('--', ':'), ('and', 'CC'), ('I', 'PRP'), ('will', 'MD'), ('do', 'VB'), ('my', 'PRP$'), ('part', 'NN'), ('.', '.')]\n",
      "[('Tonight', 'NNP'), ('the', 'DT'), ('state', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('Union', 'NNP'), ('is', 'VBZ'), ('strong', 'JJ'), ('--', ':'), ('and', 'CC'), ('together', 'RB'), ('we', 'PRP'), ('will', 'MD'), ('make', 'VB'), ('it', 'PRP'), ('stronger', 'JJR'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('In', 'IN'), ('this', 'DT'), ('decisive', 'JJ'), ('year', 'NN'), (',', ','), ('you', 'PRP'), ('and', 'CC'), ('I', 'PRP'), ('will', 'MD'), ('make', 'VB'), ('choices', 'NNS'), ('that', 'WDT'), ('determine', 'VBP'), ('both', 'DT'), ('the', 'DT'), ('future', 'NN'), ('and', 'CC'), ('the', 'DT'), ('character', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('country', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), ('will', 'MD'), ('choose', 'VB'), ('to', 'TO'), ('act', 'VB'), ('confidently', 'RB'), ('in', 'IN'), ('pursuing', 'VBG'), ('the', 'DT'), ('enemies', 'NNS'), ('of', 'IN'), ('freedom', 'NN'), ('--', ':'), ('or', 'CC'), ('retreat', 'NN'), ('from', 'IN'), ('our', 'PRP$'), ('duties', 'NNS'), ('in', 'IN'), ('the', 'DT'), ('hope', 'NN'), ('of', 'IN'), ('an', 'DT'), ('easier', 'JJR'), ('life', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), ('will', 'MD'), ('choose', 'VB'), ('to', 'TO'), ('build', 'VB'), ('our', 'PRP$'), ('prosperity', 'NN'), ('by', 'IN'), ('leading', 'VBG'), ('the', 'DT'), ('world', 'NN'), ('economy', 'NN'), ('--', ':'), ('or', 'CC'), ('shut', 'VB'), ('ourselves', 'PRP'), ('off', 'RP'), ('from', 'IN'), ('trade', 'NN'), ('and', 'CC'), ('opportunity', 'NN'), ('.', '.')]\n",
      "[('In', 'IN'), ('a', 'DT'), ('complex', 'JJ'), ('and', 'CC'), ('challenging', 'JJ'), ('time', 'NN'), (',', ','), ('the', 'DT'), ('road', 'NN'), ('of', 'IN'), ('isolationism', 'NN'), ('and', 'CC'), ('protectionism', 'NN'), ('may', 'MD'), ('seem', 'VB'), ('broad', 'JJ'), ('and', 'CC'), ('inviting', 'NN'), ('--', ':'), ('yet', 'CC'), ('it', 'PRP'), ('ends', 'VBZ'), ('in', 'IN'), ('danger', 'NN'), ('and', 'CC'), ('decline', 'NN'), ('.', '.')]\n",
      "[('The', 'DT'), ('only', 'JJ'), ('way', 'NN'), ('to', 'TO'), ('protect', 'VB'), ('our', 'PRP$'), ('people', 'NNS'), (',', ','), ('the', 'DT'), ('only', 'JJ'), ('way', 'NN'), ('to', 'TO'), ('secure', 'VB'), ('the', 'DT'), ('peace', 'NN'), (',', ','), ('the', 'DT'), ('only', 'JJ'), ('way', 'NN'), ('to', 'TO'), ('control', 'VB'), ('our', 'PRP$'), ('destiny', 'NN'), ('is', 'VBZ'), ('by', 'IN'), ('our', 'PRP$'), ('leadership', 'NN'), ('--', ':'), ('so', 'IN'), ('the', 'DT'), ('United', 'NNP'), ('States', 'NNPS'), ('of', 'IN'), ('America', 'NNP'), ('will', 'MD'), ('continue', 'VB'), ('to', 'TO'), ('lead', 'VB'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Abroad', 'RB'), (',', ','), ('our', 'PRP$'), ('nation', 'NN'), ('is', 'VBZ'), ('committed', 'VBN'), ('to', 'TO'), ('an', 'DT'), ('historic', 'JJ'), (',', ','), ('long-term', 'JJ'), ('goal', 'NN'), ('--', ':'), ('we', 'PRP'), ('seek', 'VBP'), ('the', 'DT'), ('end', 'NN'), ('of', 'IN'), ('tyranny', 'NN'), ('in', 'IN'), ('our', 'PRP$'), ('world', 'NN'), ('.', '.')]\n",
      "[('Some', 'DT'), ('dismiss', 'VBP'), ('that', 'DT'), ('goal', 'NN'), ('as', 'IN'), ('misguided', 'JJ'), ('idealism', 'NN'), ('.', '.')]\n",
      "[('In', 'IN'), ('reality', 'NN'), (',', ','), ('the', 'DT'), ('future', 'JJ'), ('security', 'NN'), ('of', 'IN'), ('America', 'NNP'), ('depends', 'VBZ'), ('on', 'IN'), ('it', 'PRP'), ('.', '.')]\n",
      "[('On', 'IN'), ('September', 'NNP'), ('the', 'DT'), ('11th', 'CD'), (',', ','), ('2001', 'CD'), (',', ','), ('we', 'PRP'), ('found', 'VBD'), ('that', 'IN'), ('problems', 'NNS'), ('originating', 'VBG'), ('in', 'IN'), ('a', 'DT'), ('failed', 'JJ'), ('and', 'CC'), ('oppressive', 'JJ'), ('state', 'NN'), ('7,000', 'CD'), ('miles', 'NNS'), ('away', 'RB'), ('could', 'MD'), ('bring', 'VB'), ('murder', 'NN'), ('and', 'CC'), ('destruction', 'NN'), ('to', 'TO'), ('our', 'PRP$'), ('country', 'NN'), ('.', '.')]\n",
      "[('Dictatorships', 'NNP'), ('shelter', 'NN'), ('terrorists', 'NNS'), (',', ','), ('and', 'CC'), ('feed', 'VB'), ('resentment', 'NN'), ('and', 'CC'), ('radicalism', 'NN'), (',', ','), ('and', 'CC'), ('seek', 'JJ'), ('weapons', 'NNS'), ('of', 'IN'), ('mass', 'NN'), ('destruction', 'NN'), ('.', '.')]\n",
      "[('Democracies', 'NNS'), ('replace', 'VB'), ('resentment', 'NN'), ('with', 'IN'), ('hope', 'NN'), (',', ','), ('respect', 'VB'), ('the', 'DT'), ('rights', 'NNS'), ('of', 'IN'), ('their', 'PRP$'), ('citizens', 'NNS'), ('and', 'CC'), ('their', 'PRP$'), ('neighbors', 'NNS'), (',', ','), ('and', 'CC'), ('join', 'VB'), ('the', 'DT'), ('fight', 'NN'), ('against', 'IN'), ('terror', 'NN'), ('.', '.')]\n",
      "[('Every', 'DT'), ('step', 'NN'), ('toward', 'IN'), ('freedom', 'NN'), ('in', 'IN'), ('the', 'DT'), ('world', 'NN'), ('makes', 'VBZ'), ('our', 'PRP$'), ('country', 'NN'), ('safer', 'NN'), ('--', ':'), ('so', 'IN'), ('we', 'PRP'), ('will', 'MD'), ('act', 'VB'), ('boldly', 'RB'), ('in', 'IN'), ('freedom', 'NN'), (\"'s\", 'POS'), ('cause', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Far', 'RB'), ('from', 'IN'), ('being', 'VBG'), ('a', 'DT'), ('hopeless', 'NN'), ('dream', 'NN'), (',', ','), ('the', 'DT'), ('advance', 'NN'), ('of', 'IN'), ('freedom', 'NN'), ('is', 'VBZ'), ('the', 'DT'), ('great', 'JJ'), ('story', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('time', 'NN'), ('.', '.')]\n",
      "[('In', 'IN'), ('1945', 'CD'), (',', ','), ('there', 'EX'), ('were', 'VBD'), ('about', 'RB'), ('two', 'CD'), ('dozen', 'NN'), ('lonely', 'RB'), ('democracies', 'VBZ'), ('in', 'IN'), ('the', 'DT'), ('world', 'NN'), ('.', '.')]\n",
      "[('Today', 'NN'), (',', ','), ('there', 'EX'), ('are', 'VBP'), ('122', 'CD'), ('.', '.')]\n",
      "[('And', 'CC'), ('we', 'PRP'), (\"'re\", 'VBP'), ('writing', 'VBG'), ('a', 'DT'), ('new', 'JJ'), ('chapter', 'NN'), ('in', 'IN'), ('the', 'DT'), ('story', 'NN'), ('of', 'IN'), ('self-government', 'JJ'), ('--', ':'), ('with', 'IN'), ('women', 'NNS'), ('lining', 'VBG'), ('up', 'RP'), ('to', 'TO'), ('vote', 'VB'), ('in', 'IN'), ('Afghanistan', 'NNP'), (',', ','), ('and', 'CC'), ('millions', 'NNS'), ('of', 'IN'), ('Iraqis', 'NNP'), ('marking', 'VBG'), ('their', 'PRP$'), ('liberty', 'NN'), ('with', 'IN'), ('purple', 'JJ'), ('ink', 'NN'), (',', ','), ('and', 'CC'), ('men', 'NNS'), ('and', 'CC'), ('women', 'NNS'), ('from', 'IN'), ('Lebanon', 'NNP'), ('to', 'TO'), ('Egypt', 'NNP'), ('debating', 'VBG'), ('the', 'DT'), ('rights', 'NNS'), ('of', 'IN'), ('individuals', 'NNS'), ('and', 'CC'), ('the', 'DT'), ('necessity', 'NN'), ('of', 'IN'), ('freedom', 'NN'), ('.', '.')]\n",
      "[('At', 'IN'), ('the', 'DT'), ('start', 'NN'), ('of', 'IN'), ('2006', 'CD'), (',', ','), ('more', 'JJR'), ('than', 'IN'), ('half', 'PDT'), ('the', 'DT'), ('people', 'NNS'), ('of', 'IN'), ('our', 'PRP$'), ('world', 'NN'), ('live', 'VBP'), ('in', 'IN'), ('democratic', 'JJ'), ('nations', 'NNS'), ('.', '.')]\n",
      "[('And', 'CC'), ('we', 'PRP'), ('do', 'VBP'), ('not', 'RB'), ('forget', 'VB'), ('the', 'DT'), ('other', 'JJ'), ('half', 'NN'), ('--', ':'), ('in', 'IN'), ('places', 'NNS'), ('like', 'IN'), ('Syria', 'NNP'), ('and', 'CC'), ('Burma', 'NNP'), (',', ','), ('Zimbabwe', 'NNP'), (',', ','), ('North', 'NNP'), ('Korea', 'NNP'), (',', ','), ('and', 'CC'), ('Iran', 'NNP'), ('--', ':'), ('because', 'IN'), ('the', 'DT'), ('demands', 'NNS'), ('of', 'IN'), ('justice', 'NN'), (',', ','), ('and', 'CC'), ('the', 'DT'), ('peace', 'NN'), ('of', 'IN'), ('this', 'DT'), ('world', 'NN'), (',', ','), ('require', 'VBP'), ('their', 'PRP$'), ('freedom', 'NN'), (',', ','), ('as', 'RB'), ('well', 'RB'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('President', 'NNP'), ('George', 'NNP'), ('W.', 'NNP'), ('Bush', 'NNP'), ('delivers', 'VBZ'), ('his', 'PRP$'), ('State', 'NN'), ('of', 'IN'), ('the', 'DT'), ('Union', 'NNP'), ('Address', 'NNP'), ('at', 'IN'), ('the', 'DT'), ('Capitol', 'NNP'), (',', ','), ('Tuesday', 'NNP'), (',', ','), ('Jan', 'NNP'), ('.', '.')]\n",
      "[('31', 'CD'), (',', ','), ('2006', 'CD'), ('.', '.')]\n",
      "[('White', 'NNP'), ('House', 'NNP'), ('photo', 'NN'), ('by', 'IN'), ('Eric', 'NNP'), ('Draper', 'NNP'), ('No', 'NNP'), ('one', 'NN'), ('can', 'MD'), ('deny', 'VB'), ('the', 'DT'), ('success', 'NN'), ('of', 'IN'), ('freedom', 'NN'), (',', ','), ('but', 'CC'), ('some', 'DT'), ('men', 'NNS'), ('rage', 'VB'), ('and', 'CC'), ('fight', 'VB'), ('against', 'IN'), ('it', 'PRP'), ('.', '.')]\n",
      "[('And', 'CC'), ('one', 'CD'), ('of', 'IN'), ('the', 'DT'), ('main', 'JJ'), ('sources', 'NNS'), ('of', 'IN'), ('reaction', 'NN'), ('and', 'CC'), ('opposition', 'NN'), ('is', 'VBZ'), ('radical', 'JJ'), ('Islam', 'NNP'), ('--', ':'), ('the', 'DT'), ('perversion', 'NN'), ('by', 'IN'), ('a', 'DT'), ('few', 'JJ'), ('of', 'IN'), ('a', 'DT'), ('noble', 'JJ'), ('faith', 'NN'), ('into', 'IN'), ('an', 'DT'), ('ideology', 'NN'), ('of', 'IN'), ('terror', 'NN'), ('and', 'CC'), ('death', 'NN'), ('.', '.')]\n",
      "[('Terrorists', 'NNS'), ('like', 'IN'), ('bin', 'NN'), ('Laden', 'NNP'), ('are', 'VBP'), ('serious', 'JJ'), ('about', 'IN'), ('mass', 'NN'), ('murder', 'NN'), ('--', ':'), ('and', 'CC'), ('all', 'DT'), ('of', 'IN'), ('us', 'PRP'), ('must', 'MD'), ('take', 'VB'), ('their', 'PRP$'), ('declared', 'JJ'), ('intentions', 'NNS'), ('seriously', 'RB'), ('.', '.')]\n",
      "[('They', 'PRP'), ('seek', 'VBP'), ('to', 'TO'), ('impose', 'VB'), ('a', 'DT'), ('heartless', 'NN'), ('system', 'NN'), ('of', 'IN'), ('totalitarian', 'JJ'), ('control', 'NN'), ('throughout', 'IN'), ('the', 'DT'), ('Middle', 'NNP'), ('East', 'NNP'), (',', ','), ('and', 'CC'), ('arm', 'NN'), ('themselves', 'PRP'), ('with', 'IN'), ('weapons', 'NNS'), ('of', 'IN'), ('mass', 'NN'), ('murder', 'NN'), ('.', '.')]\n",
      "[('Their', 'PRP$'), ('aim', 'NN'), ('is', 'VBZ'), ('to', 'TO'), ('seize', 'VB'), ('power', 'NN'), ('in', 'IN'), ('Iraq', 'NNP'), (',', ','), ('and', 'CC'), ('use', 'VB'), ('it', 'PRP'), ('as', 'IN'), ('a', 'DT'), ('safe', 'JJ'), ('haven', 'NN'), ('to', 'TO'), ('launch', 'VB'), ('attacks', 'NNS'), ('against', 'IN'), ('America', 'NNP'), ('and', 'CC'), ('the', 'DT'), ('world', 'NN'), ('.', '.')]\n",
      "[('Lacking', 'VBG'), ('the', 'DT'), ('military', 'JJ'), ('strength', 'NN'), ('to', 'TO'), ('challenge', 'VB'), ('us', 'PRP'), ('directly', 'RB'), (',', ','), ('the', 'DT'), ('terrorists', 'NNS'), ('have', 'VBP'), ('chosen', 'VBN'), ('the', 'DT'), ('weapon', 'NN'), ('of', 'IN'), ('fear', 'NN'), ('.', '.')]\n",
      "[('When', 'WRB'), ('they', 'PRP'), ('murder', 'VBP'), ('children', 'NNS'), ('at', 'IN'), ('a', 'DT'), ('school', 'NN'), ('in', 'IN'), ('Beslan', 'NNP'), (',', ','), ('or', 'CC'), ('blow', 'VB'), ('up', 'RP'), ('commuters', 'NNS'), ('in', 'IN'), ('London', 'NNP'), (',', ','), ('or', 'CC'), ('behead', 'VB'), ('a', 'DT'), ('bound', 'NN'), ('captive', 'NN'), (',', ','), ('the', 'DT'), ('terrorists', 'NNS'), ('hope', 'VBP'), ('these', 'DT'), ('horrors', 'NNS'), ('will', 'MD'), ('break', 'VB'), ('our', 'PRP$'), ('will', 'MD'), (',', ','), ('allowing', 'VBG'), ('the', 'DT'), ('violent', 'NN'), ('to', 'TO'), ('inherit', 'VB'), ('the', 'DT'), ('Earth', 'NNP'), ('.', '.')]\n",
      "[('But', 'CC'), ('they', 'PRP'), ('have', 'VBP'), ('miscalculated', 'VBN'), (':', ':'), ('We', 'PRP'), ('love', 'VBP'), ('our', 'PRP$'), ('freedom', 'NN'), (',', ','), ('and', 'CC'), ('we', 'PRP'), ('will', 'MD'), ('fight', 'VB'), ('to', 'TO'), ('keep', 'VB'), ('it', 'PRP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('In', 'IN'), ('a', 'DT'), ('time', 'NN'), ('of', 'IN'), ('testing', 'VBG'), (',', ','), ('we', 'PRP'), ('can', 'MD'), ('not', 'RB'), ('find', 'VB'), ('security', 'NN'), ('by', 'IN'), ('abandoning', 'VBG'), ('our', 'PRP$'), ('commitments', 'NNS'), ('and', 'CC'), ('retreating', 'VBG'), ('within', 'IN'), ('our', 'PRP$'), ('borders', 'NNS'), ('.', '.')]\n",
      "[('If', 'IN'), ('we', 'PRP'), ('were', 'VBD'), ('to', 'TO'), ('leave', 'VB'), ('these', 'DT'), ('vicious', 'JJ'), ('attackers', 'NNS'), ('alone', 'RB'), (',', ','), ('they', 'PRP'), ('would', 'MD'), ('not', 'RB'), ('leave', 'VB'), ('us', 'PRP'), ('alone', 'RB'), ('.', '.')]\n",
      "[('They', 'PRP'), ('would', 'MD'), ('simply', 'RB'), ('move', 'VB'), ('the', 'DT'), ('battlefield', 'NN'), ('to', 'TO'), ('our', 'PRP$'), ('own', 'JJ'), ('shores', 'NNS'), ('.', '.')]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('There', 'EX'), ('is', 'VBZ'), ('no', 'DT'), ('peace', 'NN'), ('in', 'IN'), ('retreat', 'NN'), ('.', '.')]\n",
      "[('And', 'CC'), ('there', 'EX'), ('is', 'VBZ'), ('no', 'DT'), ('honor', 'NN'), ('in', 'IN'), ('retreat', 'NN'), ('.', '.')]\n",
      "[('By', 'IN'), ('allowing', 'VBG'), ('radical', 'JJ'), ('Islam', 'NNP'), ('to', 'TO'), ('work', 'VB'), ('its', 'PRP$'), ('will', 'MD'), ('--', ':'), ('by', 'IN'), ('leaving', 'VBG'), ('an', 'DT'), ('assaulted', 'JJ'), ('world', 'NN'), ('to', 'TO'), ('fend', 'VB'), ('for', 'IN'), ('itself', 'PRP'), ('--', ':'), ('we', 'PRP'), ('would', 'MD'), ('signal', 'VB'), ('to', 'TO'), ('all', 'PDT'), ('that', 'IN'), ('we', 'PRP'), ('no', 'DT'), ('longer', 'RBR'), ('believe', 'VBP'), ('in', 'IN'), ('our', 'PRP$'), ('own', 'JJ'), ('ideals', 'NNS'), (',', ','), ('or', 'CC'), ('even', 'RB'), ('in', 'IN'), ('our', 'PRP$'), ('own', 'JJ'), ('courage', 'NN'), ('.', '.')]\n",
      "[('But', 'CC'), ('our', 'PRP$'), ('enemies', 'NNS'), ('and', 'CC'), ('our', 'PRP$'), ('friends', 'NNS'), ('can', 'MD'), ('be', 'VB'), ('certain', 'JJ'), (':', ':'), ('The', 'DT'), ('United', 'NNP'), ('States', 'NNPS'), ('will', 'MD'), ('not', 'RB'), ('retreat', 'VB'), ('from', 'IN'), ('the', 'DT'), ('world', 'NN'), (',', ','), ('and', 'CC'), ('we', 'PRP'), ('will', 'MD'), ('never', 'RB'), ('surrender', 'VB'), ('to', 'TO'), ('evil', 'VB'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('America', 'NNP'), ('rejects', 'VBZ'), ('the', 'DT'), ('false', 'JJ'), ('comfort', 'NN'), ('of', 'IN'), ('isolationism', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), ('are', 'VBP'), ('the', 'DT'), ('nation', 'NN'), ('that', 'IN'), ('saved', 'VBD'), ('liberty', 'NN'), ('in', 'IN'), ('Europe', 'NNP'), (',', ','), ('and', 'CC'), ('liberated', 'VBD'), ('death', 'NN'), ('camps', 'NNS'), (',', ','), ('and', 'CC'), ('helped', 'VBD'), ('raise', 'VB'), ('up', 'RP'), ('democracies', 'NNS'), (',', ','), ('and', 'CC'), ('faced', 'VBD'), ('down', 'IN'), ('an', 'DT'), ('evil', 'JJ'), ('empire', 'NN'), ('.', '.')]\n",
      "[('Once', 'RB'), ('again', 'RB'), (',', ','), ('we', 'PRP'), ('accept', 'VBP'), ('the', 'DT'), ('call', 'NN'), ('of', 'IN'), ('history', 'NN'), ('to', 'TO'), ('deliver', 'VB'), ('the', 'DT'), ('oppressed', 'VBN'), ('and', 'CC'), ('move', 'VB'), ('this', 'DT'), ('world', 'NN'), ('toward', 'IN'), ('peace', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), ('remain', 'VBP'), ('on', 'IN'), ('the', 'DT'), ('offensive', 'JJ'), ('against', 'IN'), ('terror', 'NN'), ('networks', 'NNS'), ('.', '.')]\n",
      "[('We', 'PRP'), ('have', 'VBP'), ('killed', 'VBN'), ('or', 'CC'), ('captured', 'VBN'), ('many', 'JJ'), ('of', 'IN'), ('their', 'PRP$'), ('leaders', 'NNS'), ('--', ':'), ('and', 'CC'), ('for', 'IN'), ('the', 'DT'), ('others', 'NNS'), (',', ','), ('their', 'PRP$'), ('day', 'NN'), ('will', 'MD'), ('come', 'VB'), ('.', '.')]\n",
      "[('President', 'NNP'), ('George', 'NNP'), ('W.', 'NNP'), ('Bush', 'NNP'), ('greets', 'VBZ'), ('members', 'NNS'), ('of', 'IN'), ('Congress', 'NNP'), ('after', 'IN'), ('his', 'PRP$'), ('State', 'NN'), ('of', 'IN'), ('the', 'DT'), ('Union', 'NNP'), ('Address', 'NNP'), ('at', 'IN'), ('the', 'DT'), ('Capitol', 'NNP'), (',', ','), ('Tuesday', 'NNP'), (',', ','), ('Jan', 'NNP'), ('.', '.')]\n",
      "[('31', 'CD'), (',', ','), ('2006', 'CD'), ('.', '.')]\n",
      "[('White', 'NNP'), ('House', 'NNP'), ('photo', 'NN'), ('by', 'IN'), ('Eric', 'NNP'), ('Draper', 'NNP'), ('We', 'PRP'), ('remain', 'VBP'), ('on', 'IN'), ('the', 'DT'), ('offensive', 'JJ'), ('in', 'IN'), ('Afghanistan', 'NNP'), (',', ','), ('where', 'WRB'), ('a', 'DT'), ('fine', 'JJ'), ('President', 'NNP'), ('and', 'CC'), ('a', 'DT'), ('National', 'NNP'), ('Assembly', 'NNP'), ('are', 'VBP'), ('fighting', 'VBG'), ('terror', 'NN'), ('while', 'IN'), ('building', 'VBG'), ('the', 'DT'), ('institutions', 'NNS'), ('of', 'IN'), ('a', 'DT'), ('new', 'JJ'), ('democracy', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), (\"'re\", 'VBP'), ('on', 'IN'), ('the', 'DT'), ('offensive', 'JJ'), ('in', 'IN'), ('Iraq', 'NNP'), (',', ','), ('with', 'IN'), ('a', 'DT'), ('clear', 'JJ'), ('plan', 'NN'), ('for', 'IN'), ('victory', 'NN'), ('.', '.')]\n",
      "[('First', 'RB'), (',', ','), ('we', 'PRP'), (\"'re\", 'VBP'), ('helping', 'VBG'), ('Iraqis', 'NNP'), ('build', 'VB'), ('an', 'DT'), ('inclusive', 'JJ'), ('government', 'NN'), (',', ','), ('so', 'IN'), ('that', 'DT'), ('old', 'JJ'), ('resentments', 'NNS'), ('will', 'MD'), ('be', 'VB'), ('eased', 'VBN'), ('and', 'CC'), ('the', 'DT'), ('insurgency', 'NN'), ('will', 'MD'), ('be', 'VB'), ('marginalized', 'VBN'), ('.', '.')]\n",
      "[('Second', 'JJ'), (',', ','), ('we', 'PRP'), (\"'re\", 'VBP'), ('continuing', 'VBG'), ('reconstruction', 'NN'), ('efforts', 'NNS'), (',', ','), ('and', 'CC'), ('helping', 'VBG'), ('the', 'DT'), ('Iraqi', 'NNP'), ('government', 'NN'), ('to', 'TO'), ('fight', 'VB'), ('corruption', 'NN'), ('and', 'CC'), ('build', 'VB'), ('a', 'DT'), ('modern', 'JJ'), ('economy', 'NN'), (',', ','), ('so', 'IN'), ('all', 'DT'), ('Iraqis', 'NNP'), ('can', 'MD'), ('experience', 'VB'), ('the', 'DT'), ('benefits', 'NNS'), ('of', 'IN'), ('freedom', 'NN'), ('.', '.')]\n",
      "[('And', 'CC'), (',', ','), ('third', 'JJ'), (',', ','), ('we', 'PRP'), (\"'re\", 'VBP'), ('striking', 'VBG'), ('terrorist', 'JJ'), ('targets', 'NNS'), ('while', 'IN'), ('we', 'PRP'), ('train', 'VBP'), ('Iraqi', 'JJ'), ('forces', 'NNS'), ('that', 'WDT'), ('are', 'VBP'), ('increasingly', 'RB'), ('capable', 'JJ'), ('of', 'IN'), ('defeating', 'VBG'), ('the', 'DT'), ('enemy', 'NN'), ('.', '.')]\n",
      "[('Iraqis', 'NNP'), ('are', 'VBP'), ('showing', 'VBG'), ('their', 'PRP$'), ('courage', 'NN'), ('every', 'DT'), ('day', 'NN'), (',', ','), ('and', 'CC'), ('we', 'PRP'), ('are', 'VBP'), ('proud', 'JJ'), ('to', 'TO'), ('be', 'VB'), ('their', 'PRP$'), ('allies', 'NNS'), ('in', 'IN'), ('the', 'DT'), ('cause', 'NN'), ('of', 'IN'), ('freedom', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Our', 'PRP$'), ('work', 'NN'), ('in', 'IN'), ('Iraq', 'NNP'), ('is', 'VBZ'), ('difficult', 'JJ'), ('because', 'IN'), ('our', 'PRP$'), ('enemy', 'NN'), ('is', 'VBZ'), ('brutal', 'JJ'), ('.', '.')]\n",
      "[('But', 'CC'), ('that', 'DT'), ('brutality', 'NN'), ('has', 'VBZ'), ('not', 'RB'), ('stopped', 'VBN'), ('the', 'DT'), ('dramatic', 'JJ'), ('progress', 'NN'), ('of', 'IN'), ('a', 'DT'), ('new', 'JJ'), ('democracy', 'NN'), ('.', '.')]\n",
      "[('In', 'IN'), ('less', 'JJR'), ('than', 'IN'), ('three', 'CD'), ('years', 'NNS'), (',', ','), ('the', 'DT'), ('nation', 'NN'), ('has', 'VBZ'), ('gone', 'VBN'), ('from', 'IN'), ('dictatorship', 'NN'), ('to', 'TO'), ('liberation', 'NN'), (',', ','), ('to', 'TO'), ('sovereignty', 'VB'), (',', ','), ('to', 'TO'), ('a', 'DT'), ('constitution', 'NN'), (',', ','), ('to', 'TO'), ('national', 'JJ'), ('elections', 'NNS'), ('.', '.')]\n",
      "[('At', 'IN'), ('the', 'DT'), ('same', 'JJ'), ('time', 'NN'), (',', ','), ('our', 'PRP$'), ('coalition', 'NN'), ('has', 'VBZ'), ('been', 'VBN'), ('relentless', 'VBN'), ('in', 'IN'), ('shutting', 'VBG'), ('off', 'RP'), ('terrorist', 'JJ'), ('infiltration', 'NN'), (',', ','), ('clearing', 'VBG'), ('out', 'RP'), ('insurgent', 'JJ'), ('strongholds', 'NNS'), (',', ','), ('and', 'CC'), ('turning', 'VBG'), ('over', 'RP'), ('territory', 'NN'), ('to', 'TO'), ('Iraqi', 'NNP'), ('security', 'NN'), ('forces', 'NNS'), ('.', '.')]\n",
      "[('I', 'PRP'), ('am', 'VBP'), ('confident', 'JJ'), ('in', 'IN'), ('our', 'PRP$'), ('plan', 'NN'), ('for', 'IN'), ('victory', 'NN'), (';', ':'), ('I', 'PRP'), ('am', 'VBP'), ('confident', 'JJ'), ('in', 'IN'), ('the', 'DT'), ('will', 'MD'), ('of', 'IN'), ('the', 'DT'), ('Iraqi', 'NNP'), ('people', 'NNS'), (';', ':'), ('I', 'PRP'), ('am', 'VBP'), ('confident', 'JJ'), ('in', 'IN'), ('the', 'DT'), ('skill', 'NN'), ('and', 'CC'), ('spirit', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('military', 'JJ'), ('.', '.')]\n",
      "[('Fellow', 'NNP'), ('citizens', 'NNS'), (',', ','), ('we', 'PRP'), ('are', 'VBP'), ('in', 'IN'), ('this', 'DT'), ('fight', 'NN'), ('to', 'TO'), ('win', 'VB'), (',', ','), ('and', 'CC'), ('we', 'PRP'), ('are', 'VBP'), ('winning', 'VBG'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('The', 'DT'), ('road', 'NN'), ('of', 'IN'), ('victory', 'NN'), ('is', 'VBZ'), ('the', 'DT'), ('road', 'NN'), ('that', 'WDT'), ('will', 'MD'), ('take', 'VB'), ('our', 'PRP$'), ('troops', 'NNS'), ('home', 'NN'), ('.', '.')]\n",
      "[('As', 'IN'), ('we', 'PRP'), ('make', 'VBP'), ('progress', 'NN'), ('on', 'IN'), ('the', 'DT'), ('ground', 'NN'), (',', ','), ('and', 'CC'), ('Iraqi', 'NNP'), ('forces', 'NNS'), ('increasingly', 'RB'), ('take', 'VBP'), ('the', 'DT'), ('lead', 'NN'), (',', ','), ('we', 'PRP'), ('should', 'MD'), ('be', 'VB'), ('able', 'JJ'), ('to', 'TO'), ('further', 'JJ'), ('decrease', 'VB'), ('our', 'PRP$'), ('troop', 'NN'), ('levels', 'NNS'), ('--', ':'), ('but', 'CC'), ('those', 'DT'), ('decisions', 'NNS'), ('will', 'MD'), ('be', 'VB'), ('made', 'VBN'), ('by', 'IN'), ('our', 'PRP$'), ('military', 'JJ'), ('commanders', 'NNS'), (',', ','), ('not', 'RB'), ('by', 'IN'), ('politicians', 'NNS'), ('in', 'IN'), ('Washington', 'NNP'), (',', ','), ('D.C', 'NNP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Our', 'PRP$'), ('coalition', 'NN'), ('has', 'VBZ'), ('learned', 'VBN'), ('from', 'IN'), ('our', 'PRP$'), ('experience', 'NN'), ('in', 'IN'), ('Iraq', 'NNP'), ('.', '.')]\n",
      "[('We', 'PRP'), (\"'ve\", 'VBP'), ('adjusted', 'VBN'), ('our', 'PRP$'), ('military', 'JJ'), ('tactics', 'NNS'), ('and', 'CC'), ('changed', 'VBD'), ('our', 'PRP$'), ('approach', 'NN'), ('to', 'TO'), ('reconstruction', 'NN'), ('.', '.')]\n",
      "[('Along', 'IN'), ('the', 'DT'), ('way', 'NN'), (',', ','), ('we', 'PRP'), ('have', 'VBP'), ('benefitted', 'VBN'), ('from', 'IN'), ('responsible', 'JJ'), ('criticism', 'NN'), ('and', 'CC'), ('counsel', 'NN'), ('offered', 'VBN'), ('by', 'IN'), ('members', 'NNS'), ('of', 'IN'), ('Congress', 'NNP'), ('of', 'IN'), ('both', 'DT'), ('parties', 'NNS'), ('.', '.')]\n",
      "[('In', 'IN'), ('the', 'DT'), ('coming', 'VBG'), ('year', 'NN'), (',', ','), ('I', 'PRP'), ('will', 'MD'), ('continue', 'VB'), ('to', 'TO'), ('reach', 'VB'), ('out', 'RP'), ('and', 'CC'), ('seek', 'VB'), ('your', 'PRP$'), ('good', 'JJ'), ('advice', 'NN'), ('.', '.')]\n",
      "[('Yet', 'RB'), (',', ','), ('there', 'EX'), ('is', 'VBZ'), ('a', 'DT'), ('difference', 'NN'), ('between', 'IN'), ('responsible', 'JJ'), ('criticism', 'NN'), ('that', 'WDT'), ('aims', 'VBZ'), ('for', 'IN'), ('success', 'NN'), (',', ','), ('and', 'CC'), ('defeatism', 'NN'), ('that', 'WDT'), ('refuses', 'VBZ'), ('to', 'TO'), ('acknowledge', 'VB'), ('anything', 'NN'), ('but', 'CC'), ('failure', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Hindsight', 'NNP'), ('alone', 'RB'), ('is', 'VBZ'), ('not', 'RB'), ('wisdom', 'JJ'), (',', ','), ('and', 'CC'), ('second-guessing', 'NN'), ('is', 'VBZ'), ('not', 'RB'), ('a', 'DT'), ('strategy', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('With', 'IN'), ('so', 'RB'), ('much', 'JJ'), ('in', 'IN'), ('the', 'DT'), ('balance', 'NN'), (',', ','), ('those', 'DT'), ('of', 'IN'), ('us', 'PRP'), ('in', 'IN'), ('public', 'JJ'), ('office', 'NN'), ('have', 'VBP'), ('a', 'DT'), ('duty', 'NN'), ('to', 'TO'), ('speak', 'VB'), ('with', 'IN'), ('candor', 'NN'), ('.', '.')]\n",
      "[('A', 'DT'), ('sudden', 'JJ'), ('withdrawal', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('forces', 'NNS'), ('from', 'IN'), ('Iraq', 'NNP'), ('would', 'MD'), ('abandon', 'VB'), ('our', 'PRP$'), ('Iraqi', 'NNP'), ('allies', 'NNS'), ('to', 'TO'), ('death', 'NN'), ('and', 'CC'), ('prison', 'NN'), (',', ','), ('would', 'MD'), ('put', 'VB'), ('men', 'NNS'), ('like', 'IN'), ('bin', 'NN'), ('Laden', 'NNP'), ('and', 'CC'), ('Zarqawi', 'NNP'), ('in', 'IN'), ('charge', 'NN'), ('of', 'IN'), ('a', 'DT'), ('strategic', 'JJ'), ('country', 'NN'), (',', ','), ('and', 'CC'), ('show', 'VBP'), ('that', 'IN'), ('a', 'DT'), ('pledge', 'NN'), ('from', 'IN'), ('America', 'NNP'), ('means', 'VBZ'), ('little', 'JJ'), ('.', '.')]\n",
      "[('Members', 'NNS'), ('of', 'IN'), ('Congress', 'NNP'), (',', ','), ('however', 'RB'), ('we', 'PRP'), ('feel', 'VBP'), ('about', 'IN'), ('the', 'DT'), ('decisions', 'NNS'), ('and', 'CC'), ('debates', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('past', 'NN'), (',', ','), ('our', 'PRP$'), ('nation', 'NN'), ('has', 'VBZ'), ('only', 'RB'), ('one', 'CD'), ('option', 'NN'), (':', ':'), ('We', 'PRP'), ('must', 'MD'), ('keep', 'VB'), ('our', 'PRP$'), ('word', 'NN'), (',', ','), ('defeat', 'VB'), ('our', 'PRP$'), ('enemies', 'NNS'), (',', ','), ('and', 'CC'), ('stand', 'VBP'), ('behind', 'IN'), ('the', 'DT'), ('American', 'JJ'), ('military', 'NN'), ('in', 'IN'), ('this', 'DT'), ('vital', 'JJ'), ('mission', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Laura', 'NNP'), ('Bush', 'NNP'), ('is', 'VBZ'), ('applauded', 'VBN'), ('as', 'IN'), ('she', 'PRP'), ('is', 'VBZ'), ('introduced', 'VBN'), ('Tuesday', 'NNP'), ('evening', 'NN'), (',', ','), ('Jan', 'NNP'), ('.', '.')]\n",
      "[('31', 'CD'), (',', ','), ('2006', 'CD'), ('during', 'IN'), ('the', 'DT'), ('State', 'NNP'), ('of', 'IN'), ('the', 'DT'), ('Union', 'NNP'), ('Address', 'NNP'), ('at', 'IN'), ('United', 'NNP'), ('States', 'NNPS'), ('Capitol', 'NNP'), ('in', 'IN'), ('Washington', 'NNP'), ('.', '.')]\n",
      "[('White', 'NNP'), ('House', 'NNP'), ('photo', 'NN'), ('by', 'IN'), ('Eric', 'NNP'), ('Draper', 'NNP'), ('Our', 'PRP$'), ('men', 'NNS'), ('and', 'CC'), ('women', 'NNS'), ('in', 'IN'), ('uniform', 'JJ'), ('are', 'VBP'), ('making', 'VBG'), ('sacrifices', 'NNS'), ('--', ':'), ('and', 'CC'), ('showing', 'VBG'), ('a', 'DT'), ('sense', 'NN'), ('of', 'IN'), ('duty', 'NN'), ('stronger', 'JJR'), ('than', 'IN'), ('all', 'DT'), ('fear', 'NN'), ('.', '.')]\n",
      "[('They', 'PRP'), ('know', 'VBP'), ('what', 'WP'), ('it', 'PRP'), (\"'s\", 'VBZ'), ('like', 'IN'), ('to', 'TO'), ('fight', 'VB'), ('house', 'NN'), ('to', 'TO'), ('house', 'NN'), ('in', 'IN'), ('a', 'DT'), ('maze', 'NN'), ('of', 'IN'), ('streets', 'NNS'), (',', ','), ('to', 'TO'), ('wear', 'VB'), ('heavy', 'JJ'), ('gear', 'NN'), ('in', 'IN'), ('the', 'DT'), ('desert', 'NN'), ('heat', 'NN'), (',', ','), ('to', 'TO'), ('see', 'VB'), ('a', 'DT'), ('comrade', 'NN'), ('killed', 'VBN'), ('by', 'IN'), ('a', 'DT'), ('roadside', 'NN'), ('bomb', 'NN'), ('.', '.')]\n",
      "[('And', 'CC'), ('those', 'DT'), ('who', 'WP'), ('know', 'VBP'), ('the', 'DT'), ('costs', 'NNS'), ('also', 'RB'), ('know', 'VBP'), ('the', 'DT'), ('stakes', 'NNS'), ('.', '.')]\n",
      "[('Marine', 'JJ'), ('Staff', 'NNP'), ('Sergeant', 'NNP'), ('Dan', 'NNP'), ('Clay', 'NNP'), ('was', 'VBD'), ('killed', 'VBN'), ('last', 'JJ'), ('month', 'NN'), ('fighting', 'VBG'), ('in', 'IN'), ('Fallujah', 'NNP'), ('.', '.')]\n",
      "[('He', 'PRP'), ('left', 'VBD'), ('behind', 'RP'), ('a', 'DT'), ('letter', 'NN'), ('to', 'TO'), ('his', 'PRP$'), ('family', 'NN'), (',', ','), ('but', 'CC'), ('his', 'PRP$'), ('words', 'NNS'), ('could', 'MD'), ('just', 'RB'), ('as', 'RB'), ('well', 'RB'), ('be', 'VB'), ('addressed', 'VBN'), ('to', 'TO'), ('every', 'DT'), ('American', 'NNP'), ('.', '.')]\n",
      "[('Here', 'RB'), ('is', 'VBZ'), ('what', 'WP'), ('Dan', 'NNP'), ('wrote', 'VBD'), (':', ':'), ('``', '``'), ('I', 'PRP'), ('know', 'VBP'), ('what', 'WP'), ('honor', 'NN'), ('is', 'VBZ'), ('.', '.')]\n",
      "[('...', ':'), ('It', 'PRP'), ('has', 'VBZ'), ('been', 'VBN'), ('an', 'DT'), ('honor', 'NN'), ('to', 'TO'), ('protect', 'VB'), ('and', 'CC'), ('serve', 'VB'), ('all', 'DT'), ('of', 'IN'), ('you', 'PRP'), ('.', '.')]\n",
      "[('I', 'PRP'), ('faced', 'VBD'), ('death', 'NN'), ('with', 'IN'), ('the', 'DT'), ('secure', 'NN'), ('knowledge', 'NN'), ('that', 'IN'), ('you', 'PRP'), ('would', 'MD'), ('not', 'RB'), ('have', 'VB'), ('to', 'TO'), ('...', ':'), ('.', '.')]\n",
      "[('Never', 'RB'), ('falter', 'NN'), ('!', '.')]\n",
      "[('Do', 'VBP'), (\"n't\", 'RB'), ('hesitate', 'VB'), ('to', 'TO'), ('honor', 'VB'), ('and', 'CC'), ('support', 'VB'), ('those', 'DT'), ('of', 'IN'), ('us', 'PRP'), ('who', 'WP'), ('have', 'VBP'), ('the', 'DT'), ('honor', 'NN'), ('of', 'IN'), ('protecting', 'VBG'), ('that', 'DT'), ('which', 'WDT'), ('is', 'VBZ'), ('worth', 'JJ'), ('protecting', 'VBG'), ('.', '.'), (\"''\", \"''\")]\n",
      "[('Staff', 'NNP'), ('Sergeant', 'NNP'), ('Dan', 'NNP'), ('Clay', 'NNP'), (\"'s\", 'POS'), ('wife', 'NN'), (',', ','), ('Lisa', 'NNP'), (',', ','), ('and', 'CC'), ('his', 'PRP$'), ('mom', 'NN'), ('and', 'CC'), ('dad', 'NN'), (',', ','), ('Sara', 'NNP'), ('Jo', 'NNP'), ('and', 'CC'), ('Bud', 'NNP'), (',', ','), ('are', 'VBP'), ('with', 'IN'), ('us', 'PRP'), ('this', 'DT'), ('evening', 'NN'), ('.', '.')]\n",
      "[('Welcome', 'NNP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Our', 'PRP$'), ('nation', 'NN'), ('is', 'VBZ'), ('grateful', 'JJ'), ('to', 'TO'), ('the', 'DT'), ('fallen', 'VBN'), (',', ','), ('who', 'WP'), ('live', 'VBP'), ('in', 'IN'), ('the', 'DT'), ('memory', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('country', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), (\"'re\", 'VBP'), ('grateful', 'JJ'), ('to', 'TO'), ('all', 'DT'), ('who', 'WP'), ('volunteer', 'VBP'), ('to', 'TO'), ('wear', 'VB'), ('our', 'PRP$'), ('nation', 'NN'), (\"'s\", 'POS'), ('uniform', 'NN'), ('--', ':'), ('and', 'CC'), ('as', 'IN'), ('we', 'PRP'), ('honor', 'VBP'), ('our', 'PRP$'), ('brave', 'NN'), ('troops', 'NNS'), (',', ','), ('let', 'VB'), ('us', 'PRP'), ('never', 'RB'), ('forget', 'VBP'), ('the', 'DT'), ('sacrifices', 'NNS'), ('of', 'IN'), ('America', 'NNP'), (\"'s\", 'POS'), ('military', 'JJ'), ('families', 'NNS'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Our', 'PRP$'), ('offensive', 'JJ'), ('against', 'IN'), ('terror', 'NN'), ('involves', 'VBZ'), ('more', 'JJR'), ('than', 'IN'), ('military', 'JJ'), ('action', 'NN'), ('.', '.')]\n",
      "[('Ultimately', 'RB'), (',', ','), ('the', 'DT'), ('only', 'JJ'), ('way', 'NN'), ('to', 'TO'), ('defeat', 'VB'), ('the', 'DT'), ('terrorists', 'NNS'), ('is', 'VBZ'), ('to', 'TO'), ('defeat', 'VB'), ('their', 'PRP$'), ('dark', 'JJ'), ('vision', 'NN'), ('of', 'IN'), ('hatred', 'VBN'), ('and', 'CC'), ('fear', 'VBN'), ('by', 'IN'), ('offering', 'VBG'), ('the', 'DT'), ('hopeful', 'JJ'), ('alternative', 'NN'), ('of', 'IN'), ('political', 'JJ'), ('freedom', 'NN'), ('and', 'CC'), ('peaceful', 'JJ'), ('change', 'NN'), ('.', '.')]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('So', 'IN'), ('the', 'DT'), ('United', 'NNP'), ('States', 'NNPS'), ('of', 'IN'), ('America', 'NNP'), ('supports', 'NNS'), ('democratic', 'JJ'), ('reform', 'NN'), ('across', 'IN'), ('the', 'DT'), ('broader', 'JJR'), ('Middle', 'NNP'), ('East', 'NNP'), ('.', '.')]\n",
      "[('Elections', 'NNS'), ('are', 'VBP'), ('vital', 'JJ'), (',', ','), ('but', 'CC'), ('they', 'PRP'), ('are', 'VBP'), ('only', 'RB'), ('the', 'DT'), ('beginning', 'NN'), ('.', '.')]\n",
      "[('Raising', 'VBG'), ('up', 'RP'), ('a', 'DT'), ('democracy', 'NN'), ('requires', 'VBZ'), ('the', 'DT'), ('rule', 'NN'), ('of', 'IN'), ('law', 'NN'), (',', ','), ('and', 'CC'), ('protection', 'NN'), ('of', 'IN'), ('minorities', 'NNS'), (',', ','), ('and', 'CC'), ('strong', 'JJ'), (',', ','), ('accountable', 'JJ'), ('institutions', 'NNS'), ('that', 'IN'), ('last', 'JJ'), ('longer', 'JJR'), ('than', 'IN'), ('a', 'DT'), ('single', 'JJ'), ('vote', 'NN'), ('.', '.')]\n",
      "[('The', 'DT'), ('great', 'JJ'), ('people', 'NNS'), ('of', 'IN'), ('Egypt', 'NNP'), ('have', 'VBP'), ('voted', 'VBN'), ('in', 'IN'), ('a', 'DT'), ('multi-party', 'JJ'), ('presidential', 'JJ'), ('election', 'NN'), ('--', ':'), ('and', 'CC'), ('now', 'RB'), ('their', 'PRP$'), ('government', 'NN'), ('should', 'MD'), ('open', 'VB'), ('paths', 'NNS'), ('of', 'IN'), ('peaceful', 'JJ'), ('opposition', 'NN'), ('that', 'WDT'), ('will', 'MD'), ('reduce', 'VB'), ('the', 'DT'), ('appeal', 'NN'), ('of', 'IN'), ('radicalism', 'NN'), ('.', '.')]\n",
      "[('The', 'DT'), ('Palestinian', 'JJ'), ('people', 'NNS'), ('have', 'VBP'), ('voted', 'VBN'), ('in', 'IN'), ('elections', 'NNS'), ('.', '.')]\n",
      "[('And', 'CC'), ('now', 'RB'), ('the', 'DT'), ('leaders', 'NNS'), ('of', 'IN'), ('Hamas', 'NNP'), ('must', 'MD'), ('recognize', 'VB'), ('Israel', 'NNP'), (',', ','), ('disarm', 'NN'), (',', ','), ('reject', 'JJ'), ('terrorism', 'NN'), (',', ','), ('and', 'CC'), ('work', 'NN'), ('for', 'IN'), ('lasting', 'VBG'), ('peace', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Saudi', 'NNP'), ('Arabia', 'NNP'), ('has', 'VBZ'), ('taken', 'VBN'), ('the', 'DT'), ('first', 'JJ'), ('steps', 'NNS'), ('of', 'IN'), ('reform', 'NN'), ('--', ':'), ('now', 'RB'), ('it', 'PRP'), ('can', 'MD'), ('offer', 'VB'), ('its', 'PRP$'), ('people', 'NNS'), ('a', 'DT'), ('better', 'JJR'), ('future', 'NN'), ('by', 'IN'), ('pressing', 'VBG'), ('forward', 'RB'), ('with', 'IN'), ('those', 'DT'), ('efforts', 'NNS'), ('.', '.')]\n",
      "[('Democracies', 'NNS'), ('in', 'IN'), ('the', 'DT'), ('Middle', 'NNP'), ('East', 'NNP'), ('will', 'MD'), ('not', 'RB'), ('look', 'VB'), ('like', 'IN'), ('our', 'PRP$'), ('own', 'JJ'), (',', ','), ('because', 'IN'), ('they', 'PRP'), ('will', 'MD'), ('reflect', 'VB'), ('the', 'DT'), ('traditions', 'NNS'), ('of', 'IN'), ('their', 'PRP$'), ('own', 'JJ'), ('citizens', 'NNS'), ('.', '.')]\n",
      "[('Yet', 'RB'), ('liberty', 'NN'), ('is', 'VBZ'), ('the', 'DT'), ('future', 'NN'), ('of', 'IN'), ('every', 'DT'), ('nation', 'NN'), ('in', 'IN'), ('the', 'DT'), ('Middle', 'NNP'), ('East', 'NNP'), (',', ','), ('because', 'IN'), ('liberty', 'NN'), ('is', 'VBZ'), ('the', 'DT'), ('right', 'NN'), ('and', 'CC'), ('hope', 'NN'), ('of', 'IN'), ('all', 'DT'), ('humanity', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('President', 'NNP'), ('George', 'NNP'), ('W.', 'NNP'), ('Bush', 'NNP'), ('waves', 'VBZ'), ('toward', 'IN'), ('the', 'DT'), ('upper', 'JJ'), ('visitors', 'NNS'), ('gallery', 'NN'), ('of', 'IN'), ('the', 'DT'), ('House', 'NNP'), ('Chamber', 'NNP'), ('following', 'VBG'), ('his', 'PRP$'), ('State', 'NN'), ('of', 'IN'), ('the', 'DT'), ('Union', 'NNP'), ('remarks', 'NNS'), ('Tuesday', 'NNP'), (',', ','), ('Jan', 'NNP'), ('.', '.')]\n",
      "[('31', 'CD'), (',', ','), ('2006', 'CD'), ('at', 'IN'), ('the', 'DT'), ('United', 'NNP'), ('States', 'NNPS'), ('Capitol', 'NNP'), ('.', '.')]\n",
      "[('White', 'NNP'), ('House', 'NNP'), ('photo', 'NN'), ('by', 'IN'), ('Eric', 'NNP'), ('Draper', 'NNP'), ('The', 'DT'), ('same', 'JJ'), ('is', 'VBZ'), ('true', 'JJ'), ('of', 'IN'), ('Iran', 'NNP'), (',', ','), ('a', 'DT'), ('nation', 'NN'), ('now', 'RB'), ('held', 'VBN'), ('hostage', 'NN'), ('by', 'IN'), ('a', 'DT'), ('small', 'JJ'), ('clerical', 'JJ'), ('elite', 'NN'), ('that', 'WDT'), ('is', 'VBZ'), ('isolating', 'VBG'), ('and', 'CC'), ('repressing', 'VBG'), ('its', 'PRP$'), ('people', 'NNS'), ('.', '.')]\n",
      "[('The', 'DT'), ('regime', 'NN'), ('in', 'IN'), ('that', 'DT'), ('country', 'NN'), ('sponsors', 'NNS'), ('terrorists', 'NNS'), ('in', 'IN'), ('the', 'DT'), ('Palestinian', 'JJ'), ('territories', 'NNS'), ('and', 'CC'), ('in', 'IN'), ('Lebanon', 'NNP'), ('--', ':'), ('and', 'CC'), ('that', 'DT'), ('must', 'MD'), ('come', 'VB'), ('to', 'TO'), ('an', 'DT'), ('end', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('The', 'DT'), ('Iranian', 'JJ'), ('government', 'NN'), ('is', 'VBZ'), ('defying', 'VBG'), ('the', 'DT'), ('world', 'NN'), ('with', 'IN'), ('its', 'PRP$'), ('nuclear', 'JJ'), ('ambitions', 'NNS'), (',', ','), ('and', 'CC'), ('the', 'DT'), ('nations', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('world', 'NN'), ('must', 'MD'), ('not', 'RB'), ('permit', 'VB'), ('the', 'DT'), ('Iranian', 'JJ'), ('regime', 'NN'), ('to', 'TO'), ('gain', 'VB'), ('nuclear', 'JJ'), ('weapons', 'NNS'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('America', 'NNP'), ('will', 'MD'), ('continue', 'VB'), ('to', 'TO'), ('rally', 'VB'), ('the', 'DT'), ('world', 'NN'), ('to', 'TO'), ('confront', 'VB'), ('these', 'DT'), ('threats', 'NNS'), ('.', '.')]\n",
      "[('Tonight', 'NNP'), (',', ','), ('let', 'VB'), ('me', 'PRP'), ('speak', 'VB'), ('directly', 'RB'), ('to', 'TO'), ('the', 'DT'), ('citizens', 'NNS'), ('of', 'IN'), ('Iran', 'NNP'), (':', ':'), ('America', 'NNP'), ('respects', 'VBZ'), ('you', 'PRP'), (',', ','), ('and', 'CC'), ('we', 'PRP'), ('respect', 'VBP'), ('your', 'PRP$'), ('country', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), ('respect', 'VBP'), ('your', 'PRP$'), ('right', 'NN'), ('to', 'TO'), ('choose', 'VB'), ('your', 'PRP$'), ('own', 'JJ'), ('future', 'NN'), ('and', 'CC'), ('win', 'VB'), ('your', 'PRP$'), ('own', 'JJ'), ('freedom', 'NN'), ('.', '.')]\n",
      "[('And', 'CC'), ('our', 'PRP$'), ('nation', 'NN'), ('hopes', 'VBZ'), ('one', 'CD'), ('day', 'NN'), ('to', 'TO'), ('be', 'VB'), ('the', 'DT'), ('closest', 'JJS'), ('of', 'IN'), ('friends', 'NNS'), ('with', 'IN'), ('a', 'DT'), ('free', 'JJ'), ('and', 'CC'), ('democratic', 'JJ'), ('Iran', 'NNP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('To', 'TO'), ('overcome', 'VB'), ('dangers', 'NNS'), ('in', 'IN'), ('our', 'PRP$'), ('world', 'NN'), (',', ','), ('we', 'PRP'), ('must', 'MD'), ('also', 'RB'), ('take', 'VB'), ('the', 'DT'), ('offensive', 'JJ'), ('by', 'IN'), ('encouraging', 'VBG'), ('economic', 'JJ'), ('progress', 'NN'), (',', ','), ('and', 'CC'), ('fighting', 'VBG'), ('disease', 'NN'), (',', ','), ('and', 'CC'), ('spreading', 'VBG'), ('hope', 'NN'), ('in', 'IN'), ('hopeless', 'JJ'), ('lands', 'NNS'), ('.', '.')]\n",
      "[('Isolationism', 'NNP'), ('would', 'MD'), ('not', 'RB'), ('only', 'RB'), ('tie', 'VB'), ('our', 'PRP$'), ('hands', 'NNS'), ('in', 'IN'), ('fighting', 'VBG'), ('enemies', 'NNS'), (',', ','), ('it', 'PRP'), ('would', 'MD'), ('keep', 'VB'), ('us', 'PRP'), ('from', 'IN'), ('helping', 'VBG'), ('our', 'PRP$'), ('friends', 'NNS'), ('in', 'IN'), ('desperate', 'JJ'), ('need', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), ('show', 'VBP'), ('compassion', 'JJ'), ('abroad', 'RB'), ('because', 'IN'), ('Americans', 'NNPS'), ('believe', 'VBP'), ('in', 'IN'), ('the', 'DT'), ('God-given', 'NNP'), ('dignity', 'NN'), ('and', 'CC'), ('worth', 'NN'), ('of', 'IN'), ('a', 'DT'), ('villager', 'NN'), ('with', 'IN'), ('HIV/AIDS', 'NNP'), (',', ','), ('or', 'CC'), ('an', 'DT'), ('infant', 'NN'), ('with', 'IN'), ('malaria', 'NNS'), (',', ','), ('or', 'CC'), ('a', 'DT'), ('refugee', 'JJ'), ('fleeing', 'NN'), ('genocide', 'NN'), (',', ','), ('or', 'CC'), ('a', 'DT'), ('young', 'JJ'), ('girl', 'NN'), ('sold', 'VBN'), ('into', 'IN'), ('slavery', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), ('also', 'RB'), ('show', 'VBP'), ('compassion', 'NN'), ('abroad', 'RB'), ('because', 'IN'), ('regions', 'NNS'), ('overwhelmed', 'VBN'), ('by', 'IN'), ('poverty', 'NN'), (',', ','), ('corruption', 'NN'), (',', ','), ('and', 'CC'), ('despair', 'NN'), ('are', 'VBP'), ('sources', 'NNS'), ('of', 'IN'), ('terrorism', 'NN'), (',', ','), ('and', 'CC'), ('organized', 'VBD'), ('crime', 'NN'), (',', ','), ('and', 'CC'), ('human', 'JJ'), ('trafficking', 'NN'), (',', ','), ('and', 'CC'), ('the', 'DT'), ('drug', 'NN'), ('trade', 'NN'), ('.', '.')]\n",
      "[('In', 'IN'), ('recent', 'JJ'), ('years', 'NNS'), (',', ','), ('you', 'PRP'), ('and', 'CC'), ('I', 'PRP'), ('have', 'VBP'), ('taken', 'VBN'), ('unprecedented', 'JJ'), ('action', 'NN'), ('to', 'TO'), ('fight', 'VB'), ('AIDS', 'NNP'), ('and', 'CC'), ('malaria', 'NNS'), (',', ','), ('expand', 'VBP'), ('the', 'DT'), ('education', 'NN'), ('of', 'IN'), ('girls', 'NNS'), (',', ','), ('and', 'CC'), ('reward', 'RB'), ('developing', 'VBG'), ('nations', 'NNS'), ('that', 'WDT'), ('are', 'VBP'), ('moving', 'VBG'), ('forward', 'RB'), ('with', 'IN'), ('economic', 'JJ'), ('and', 'CC'), ('political', 'JJ'), ('reform', 'NN'), ('.', '.')]\n",
      "[('For', 'IN'), ('people', 'NNS'), ('everywhere', 'RB'), (',', ','), ('the', 'DT'), ('United', 'NNP'), ('States', 'NNPS'), ('is', 'VBZ'), ('a', 'DT'), ('partner', 'NN'), ('for', 'IN'), ('a', 'DT'), ('better', 'JJR'), ('life', 'NN'), ('.', '.')]\n",
      "[('Short-changing', 'VBG'), ('these', 'DT'), ('efforts', 'NNS'), ('would', 'MD'), ('increase', 'VB'), ('the', 'DT'), ('suffering', 'NN'), ('and', 'CC'), ('chaos', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('world', 'NN'), (',', ','), ('undercut', 'JJ'), ('our', 'PRP$'), ('long-term', 'JJ'), ('security', 'NN'), (',', ','), ('and', 'CC'), ('dull', 'VB'), ('the', 'DT'), ('conscience', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('country', 'NN'), ('.', '.')]\n",
      "[('I', 'PRP'), ('urge', 'VBP'), ('members', 'NNS'), ('of', 'IN'), ('Congress', 'NNP'), ('to', 'TO'), ('serve', 'VB'), ('the', 'DT'), ('interests', 'NNS'), ('of', 'IN'), ('America', 'NNP'), ('by', 'IN'), ('showing', 'VBG'), ('the', 'DT'), ('compassion', 'NN'), ('of', 'IN'), ('America', 'NNP'), ('.', '.')]\n",
      "[('Our', 'PRP$'), ('country', 'NN'), ('must', 'MD'), ('also', 'RB'), ('remain', 'VB'), ('on', 'IN'), ('the', 'DT'), ('offensive', 'JJ'), ('against', 'IN'), ('terrorism', 'NN'), ('here', 'RB'), ('at', 'IN'), ('home', 'NN'), ('.', '.')]\n",
      "[('The', 'DT'), ('enemy', 'NN'), ('has', 'VBZ'), ('not', 'RB'), ('lost', 'VBN'), ('the', 'DT'), ('desire', 'NN'), ('or', 'CC'), ('capability', 'NN'), ('to', 'TO'), ('attack', 'VB'), ('us', 'PRP'), ('.', '.')]\n",
      "[('Fortunately', 'RB'), (',', ','), ('this', 'DT'), ('nation', 'NN'), ('has', 'VBZ'), ('superb', 'VBN'), ('professionals', 'NNS'), ('in', 'IN'), ('law', 'NN'), ('enforcement', 'NN'), (',', ','), ('intelligence', 'NN'), (',', ','), ('the', 'DT'), ('military', 'JJ'), (',', ','), ('and', 'CC'), ('homeland', 'VBP'), ('security', 'NN'), ('.', '.')]\n",
      "[('These', 'DT'), ('men', 'NNS'), ('and', 'CC'), ('women', 'NNS'), ('are', 'VBP'), ('dedicating', 'VBG'), ('their', 'PRP$'), ('lives', 'NNS'), (',', ','), ('protecting', 'VBG'), ('us', 'PRP'), ('all', 'DT'), (',', ','), ('and', 'CC'), ('they', 'PRP'), ('deserve', 'VBP'), ('our', 'PRP$'), ('support', 'NN'), ('and', 'CC'), ('our', 'PRP$'), ('thanks', 'NNS'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('They', 'PRP'), ('also', 'RB'), ('deserve', 'VBP'), ('the', 'DT'), ('same', 'JJ'), ('tools', 'NNS'), ('they', 'PRP'), ('already', 'RB'), ('use', 'VBP'), ('to', 'TO'), ('fight', 'VB'), ('drug', 'NN'), ('trafficking', 'NN'), ('and', 'CC'), ('organized', 'VBN'), ('crime', 'NN'), ('--', ':'), ('so', 'RB'), ('I', 'PRP'), ('ask', 'VBP'), ('you', 'PRP'), ('to', 'TO'), ('reauthorize', 'VB'), ('the', 'DT'), ('Patriot', 'NNP'), ('Act', 'NNP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('It', 'PRP'), ('is', 'VBZ'), ('said', 'VBD'), ('that', 'IN'), ('prior', 'JJ'), ('to', 'TO'), ('the', 'DT'), ('attacks', 'NNS'), ('of', 'IN'), ('September', 'NNP'), ('the', 'DT'), ('11th', 'CD'), (',', ','), ('our', 'PRP$'), ('government', 'NN'), ('failed', 'VBD'), ('to', 'TO'), ('connect', 'VB'), ('the', 'DT'), ('dots', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('conspiracy', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), ('now', 'RB'), ('know', 'VBP'), ('that', 'IN'), ('two', 'CD'), ('of', 'IN'), ('the', 'DT'), ('hijackers', 'NNS'), ('in', 'IN'), ('the', 'DT'), ('United', 'NNP'), ('States', 'NNPS'), ('placed', 'VBD'), ('telephone', 'NN'), ('calls', 'NNS'), ('to', 'TO'), ('al', 'VB'), ('Qaeda', 'NNP'), ('operatives', 'VBZ'), ('overseas', 'RB'), ('.', '.')]\n",
      "[('But', 'CC'), ('we', 'PRP'), ('did', 'VBD'), ('not', 'RB'), ('know', 'VB'), ('about', 'IN'), ('their', 'PRP$'), ('plans', 'NNS'), ('until', 'IN'), ('it', 'PRP'), ('was', 'VBD'), ('too', 'RB'), ('late', 'JJ'), ('.', '.')]\n",
      "[('So', 'RB'), ('to', 'TO'), ('prevent', 'VB'), ('another', 'DT'), ('attack', 'NN'), ('--', ':'), ('based', 'VBN'), ('on', 'IN'), ('authority', 'NN'), ('given', 'VBN'), ('to', 'TO'), ('me', 'PRP'), ('by', 'IN'), ('the', 'DT'), ('Constitution', 'NNP'), ('and', 'CC'), ('by', 'IN'), ('statute', 'NN'), ('--', ':'), ('I', 'PRP'), ('have', 'VBP'), ('authorized', 'VBN'), ('a', 'DT'), ('terrorist', 'JJ'), ('surveillance', 'NN'), ('program', 'NN'), ('to', 'TO'), ('aggressively', 'RB'), ('pursue', 'VB'), ('the', 'DT'), ('international', 'JJ'), ('communications', 'NNS'), ('of', 'IN'), ('suspected', 'JJ'), ('al', 'JJ'), ('Qaeda', 'NNP'), ('operatives', 'NNS'), ('and', 'CC'), ('affiliates', 'NNS'), ('to', 'TO'), ('and', 'CC'), ('from', 'IN'), ('America', 'NNP'), ('.', '.')]\n",
      "[('Previous', 'JJ'), ('Presidents', 'NNS'), ('have', 'VBP'), ('used', 'VBN'), ('the', 'DT'), ('same', 'JJ'), ('constitutional', 'JJ'), ('authority', 'NN'), ('I', 'PRP'), ('have', 'VBP'), (',', ','), ('and', 'CC'), ('federal', 'JJ'), ('courts', 'NNS'), ('have', 'VBP'), ('approved', 'VBN'), ('the', 'DT'), ('use', 'NN'), ('of', 'IN'), ('that', 'DT'), ('authority', 'NN'), ('.', '.')]\n",
      "[('Appropriate', 'JJ'), ('members', 'NNS'), ('of', 'IN'), ('Congress', 'NNP'), ('have', 'VBP'), ('been', 'VBN'), ('kept', 'VBN'), ('informed', 'VBN'), ('.', '.')]\n",
      "[('The', 'DT'), ('terrorist', 'JJ'), ('surveillance', 'NN'), ('program', 'NN'), ('has', 'VBZ'), ('helped', 'VBN'), ('prevent', 'VB'), ('terrorist', 'JJ'), ('attacks', 'NNS'), ('.', '.')]\n",
      "[('It', 'PRP'), ('remains', 'VBZ'), ('essential', 'JJ'), ('to', 'TO'), ('the', 'DT'), ('security', 'NN'), ('of', 'IN'), ('America', 'NNP'), ('.', '.')]\n",
      "[('If', 'IN'), ('there', 'EX'), ('are', 'VBP'), ('people', 'NNS'), ('inside', 'IN'), ('our', 'PRP$'), ('country', 'NN'), ('who', 'WP'), ('are', 'VBP'), ('talking', 'VBG'), ('with', 'IN'), ('al', 'NN'), ('Qaeda', 'NNP'), (',', ','), ('we', 'PRP'), ('want', 'VBP'), ('to', 'TO'), ('know', 'VB'), ('about', 'IN'), ('it', 'PRP'), (',', ','), ('because', 'IN'), ('we', 'PRP'), ('will', 'MD'), ('not', 'RB'), ('sit', 'VB'), ('back', 'RB'), ('and', 'CC'), ('wait', 'NN'), ('to', 'TO'), ('be', 'VB'), ('hit', 'VBN'), ('again', 'RB'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('In', 'IN'), ('all', 'PDT'), ('these', 'DT'), ('areas', 'NNS'), ('--', ':'), ('from', 'IN'), ('the', 'DT'), ('disruption', 'NN'), ('of', 'IN'), ('terror', 'NN'), ('networks', 'NNS'), (',', ','), ('to', 'TO'), ('victory', 'NN'), ('in', 'IN'), ('Iraq', 'NNP'), (',', ','), ('to', 'TO'), ('the', 'DT'), ('spread', 'NN'), ('of', 'IN'), ('freedom', 'NN'), ('and', 'CC'), ('hope', 'NN'), ('in', 'IN'), ('troubled', 'JJ'), ('regions', 'NNS'), ('--', ':'), ('we', 'PRP'), ('need', 'VBP'), ('the', 'DT'), ('support', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('friends', 'NNS'), ('and', 'CC'), ('allies', 'NNS'), ('.', '.')]\n",
      "[('To', 'TO'), ('draw', 'VB'), ('that', 'DT'), ('support', 'NN'), (',', ','), ('we', 'PRP'), ('must', 'MD'), ('always', 'RB'), ('be', 'VB'), ('clear', 'JJ'), ('in', 'IN'), ('our', 'PRP$'), ('principles', 'NNS'), ('and', 'CC'), ('willing', 'JJ'), ('to', 'TO'), ('act', 'VB'), ('.', '.')]\n",
      "[('The', 'DT'), ('only', 'JJ'), ('alternative', 'NN'), ('to', 'TO'), ('American', 'JJ'), ('leadership', 'NN'), ('is', 'VBZ'), ('a', 'DT'), ('dramatically', 'RB'), ('more', 'RBR'), ('dangerous', 'JJ'), ('and', 'CC'), ('anxious', 'JJ'), ('world', 'NN'), ('.', '.')]\n",
      "[('Yet', 'CC'), ('we', 'PRP'), ('also', 'RB'), ('choose', 'VBP'), ('to', 'TO'), ('lead', 'VB'), ('because', 'IN'), ('it', 'PRP'), ('is', 'VBZ'), ('a', 'DT'), ('privilege', 'NN'), ('to', 'TO'), ('serve', 'VB'), ('the', 'DT'), ('values', 'NNS'), ('that', 'WDT'), ('gave', 'VBD'), ('us', 'PRP'), ('birth', 'NN'), ('.', '.')]\n",
      "[('American', 'JJ'), ('leaders', 'NNS'), ('--', ':'), ('from', 'IN'), ('Roosevelt', 'NNP'), ('to', 'TO'), ('Truman', 'NNP'), ('to', 'TO'), ('Kennedy', 'NNP'), ('to', 'TO'), ('Reagan', 'NNP'), ('--', ':'), ('rejected', 'VBD'), ('isolation', 'NN'), ('and', 'CC'), ('retreat', 'NN'), (',', ','), ('because', 'IN'), ('they', 'PRP'), ('knew', 'VBD'), ('that', 'IN'), ('America', 'NNP'), ('is', 'VBZ'), ('always', 'RB'), ('more', 'RBR'), ('secure', 'JJ'), ('when', 'WRB'), ('freedom', 'NN'), ('is', 'VBZ'), ('on', 'IN'), ('the', 'DT'), ('march', 'NN'), ('.', '.')]\n",
      "[('Our', 'PRP$'), ('own', 'JJ'), ('generation', 'NN'), ('is', 'VBZ'), ('in', 'IN'), ('a', 'DT'), ('long', 'JJ'), ('war', 'NN'), ('against', 'IN'), ('a', 'DT'), ('determined', 'JJ'), ('enemy', 'NN'), ('--', ':'), ('a', 'DT'), ('war', 'NN'), ('that', 'WDT'), ('will', 'MD'), ('be', 'VB'), ('fought', 'VBN'), ('by', 'IN'), ('Presidents', 'NNS'), ('of', 'IN'), ('both', 'DT'), ('parties', 'NNS'), (',', ','), ('who', 'WP'), ('will', 'MD'), ('need', 'VB'), ('steady', 'JJ'), ('bipartisan', 'JJ'), ('support', 'NN'), ('from', 'IN'), ('the', 'DT'), ('Congress', 'NNP'), ('.', '.')]\n",
      "[('And', 'CC'), ('tonight', 'NN'), ('I', 'PRP'), ('ask', 'VBP'), ('for', 'IN'), ('yours', 'NNS'), ('.', '.')]\n",
      "[('Together', 'RB'), (',', ','), ('let', 'VB'), ('us', 'PRP'), ('protect', 'VB'), ('our', 'PRP$'), ('country', 'NN'), (',', ','), ('support', 'VB'), ('the', 'DT'), ('men', 'NNS'), ('and', 'CC'), ('women', 'NNS'), ('who', 'WP'), ('defend', 'VBP'), ('us', 'PRP'), (',', ','), ('and', 'CC'), ('lead', 'VB'), ('this', 'DT'), ('world', 'NN'), ('toward', 'IN'), ('freedom', 'NN'), ('.', '.')]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Here', 'RB'), ('at', 'IN'), ('home', 'NN'), (',', ','), ('America', 'NNP'), ('also', 'RB'), ('has', 'VBZ'), ('a', 'DT'), ('great', 'JJ'), ('opportunity', 'NN'), (':', ':'), ('We', 'PRP'), ('will', 'MD'), ('build', 'VB'), ('the', 'DT'), ('prosperity', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('country', 'NN'), ('by', 'IN'), ('strengthening', 'VBG'), ('our', 'PRP$'), ('economic', 'JJ'), ('leadership', 'NN'), ('in', 'IN'), ('the', 'DT'), ('world', 'NN'), ('.', '.')]\n",
      "[('Our', 'PRP$'), ('economy', 'NN'), ('is', 'VBZ'), ('healthy', 'JJ'), ('and', 'CC'), ('vigorous', 'JJ'), (',', ','), ('and', 'CC'), ('growing', 'VBG'), ('faster', 'RBR'), ('than', 'IN'), ('other', 'JJ'), ('major', 'JJ'), ('industrialized', 'VBN'), ('nations', 'NNS'), ('.', '.')]\n",
      "[('In', 'IN'), ('the', 'DT'), ('last', 'JJ'), ('two-and-a-half', 'JJ'), ('years', 'NNS'), (',', ','), ('America', 'NNP'), ('has', 'VBZ'), ('created', 'VBN'), ('4.6', 'CD'), ('million', 'CD'), ('new', 'JJ'), ('jobs', 'NNS'), ('--', ':'), ('more', 'JJR'), ('than', 'IN'), ('Japan', 'NNP'), ('and', 'CC'), ('the', 'DT'), ('European', 'NNP'), ('Union', 'NNP'), ('combined', 'VBD'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Even', 'RB'), ('in', 'IN'), ('the', 'DT'), ('face', 'NN'), ('of', 'IN'), ('higher', 'JJR'), ('energy', 'NN'), ('prices', 'NNS'), ('and', 'CC'), ('natural', 'JJ'), ('disasters', 'NNS'), (',', ','), ('the', 'DT'), ('American', 'JJ'), ('people', 'NNS'), ('have', 'VBP'), ('turned', 'VBN'), ('in', 'IN'), ('an', 'DT'), ('economic', 'JJ'), ('performance', 'NN'), ('that', 'WDT'), ('is', 'VBZ'), ('the', 'DT'), ('envy', 'NN'), ('of', 'IN'), ('the', 'DT'), ('world', 'NN'), ('.', '.')]\n",
      "[('The', 'DT'), ('American', 'JJ'), ('economy', 'NN'), ('is', 'VBZ'), ('preeminent', 'JJ'), (',', ','), ('but', 'CC'), ('we', 'PRP'), ('can', 'MD'), ('not', 'RB'), ('afford', 'VB'), ('to', 'TO'), ('be', 'VB'), ('complacent', 'JJ'), ('.', '.')]\n",
      "[('In', 'IN'), ('a', 'DT'), ('dynamic', 'JJ'), ('world', 'NN'), ('economy', 'NN'), (',', ','), ('we', 'PRP'), ('are', 'VBP'), ('seeing', 'VBG'), ('new', 'JJ'), ('competitors', 'NNS'), (',', ','), ('like', 'IN'), ('China', 'NNP'), ('and', 'CC'), ('India', 'NNP'), (',', ','), ('and', 'CC'), ('this', 'DT'), ('creates', 'VBZ'), ('uncertainty', 'NN'), (',', ','), ('which', 'WDT'), ('makes', 'VBZ'), ('it', 'PRP'), ('easier', 'JJR'), ('to', 'TO'), ('feed', 'VB'), ('people', 'NNS'), (\"'s\", 'POS'), ('fears', 'NNS'), ('.', '.')]\n",
      "[('So', 'IN'), ('we', 'PRP'), (\"'re\", 'VBP'), ('seeing', 'VBG'), ('some', 'DT'), ('old', 'JJ'), ('temptations', 'NNS'), ('return', 'NN'), ('.', '.')]\n",
      "[('Protectionists', 'NNS'), ('want', 'VBP'), ('to', 'TO'), ('escape', 'VB'), ('competition', 'NN'), (',', ','), ('pretending', 'VBG'), ('that', 'IN'), ('we', 'PRP'), ('can', 'MD'), ('keep', 'VB'), ('our', 'PRP$'), ('high', 'JJ'), ('standard', 'NN'), ('of', 'IN'), ('living', 'NN'), ('while', 'IN'), ('walling', 'VBG'), ('off', 'RP'), ('our', 'PRP$'), ('economy', 'NN'), ('.', '.')]\n",
      "[('Others', 'NNS'), ('say', 'VBP'), ('that', 'IN'), ('the', 'DT'), ('government', 'NN'), ('needs', 'VBZ'), ('to', 'TO'), ('take', 'VB'), ('a', 'DT'), ('larger', 'JJR'), ('role', 'NN'), ('in', 'IN'), ('directing', 'VBG'), ('the', 'DT'), ('economy', 'NN'), (',', ','), ('centralizing', 'VBG'), ('more', 'JJR'), ('power', 'NN'), ('in', 'IN'), ('Washington', 'NNP'), ('and', 'CC'), ('increasing', 'VBG'), ('taxes', 'NNS'), ('.', '.')]\n",
      "[('We', 'PRP'), ('hear', 'VBP'), ('claims', 'NNS'), ('that', 'IN'), ('immigrants', 'NNS'), ('are', 'VBP'), ('somehow', 'RB'), ('bad', 'JJ'), ('for', 'IN'), ('the', 'DT'), ('economy', 'NN'), ('--', ':'), ('even', 'RB'), ('though', 'IN'), ('this', 'DT'), ('economy', 'NN'), ('could', 'MD'), ('not', 'RB'), ('function', 'VB'), ('without', 'IN'), ('them', 'PRP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('All', 'PDT'), ('these', 'DT'), ('are', 'VBP'), ('forms', 'NNS'), ('of', 'IN'), ('economic', 'JJ'), ('retreat', 'NN'), (',', ','), ('and', 'CC'), ('they', 'PRP'), ('lead', 'VBP'), ('in', 'IN'), ('the', 'DT'), ('same', 'JJ'), ('direction', 'NN'), ('--', ':'), ('toward', 'IN'), ('a', 'DT'), ('stagnant', 'JJ'), ('and', 'CC'), ('second-rate', 'JJ'), ('economy', 'NN'), ('.', '.')]\n",
      "[('Tonight', 'NNP'), ('I', 'PRP'), ('will', 'MD'), ('set', 'VB'), ('out', 'RP'), ('a', 'DT'), ('better', 'JJR'), ('path', 'NN'), (':', ':'), ('an', 'DT'), ('agenda', 'NN'), ('for', 'IN'), ('a', 'DT'), ('nation', 'NN'), ('that', 'WDT'), ('competes', 'VBZ'), ('with', 'IN'), ('confidence', 'NN'), (';', ':'), ('an', 'DT'), ('agenda', 'NN'), ('that', 'WDT'), ('will', 'MD'), ('raise', 'VB'), ('standards', 'NNS'), ('of', 'IN'), ('living', 'NN'), ('and', 'CC'), ('generate', 'VB'), ('new', 'JJ'), ('jobs', 'NNS'), ('.', '.')]\n",
      "[('Americans', 'NNPS'), ('should', 'MD'), ('not', 'RB'), ('fear', 'VB'), ('our', 'PRP$'), ('economic', 'JJ'), ('future', 'NN'), (',', ','), ('because', 'IN'), ('we', 'PRP'), ('intend', 'VBP'), ('to', 'TO'), ('shape', 'VB'), ('it', 'PRP'), ('.', '.')]\n",
      "[('Keeping', 'VBG'), ('America', 'NNP'), ('competitive', 'JJ'), ('begins', 'NNS'), ('with', 'IN'), ('keeping', 'VBG'), ('our', 'PRP$'), ('economy', 'NN'), ('growing', 'VBG'), ('.', '.')]\n",
      "[('And', 'CC'), ('our', 'PRP$'), ('economy', 'NN'), ('grows', 'VBZ'), ('when', 'WRB'), ('Americans', 'NNPS'), ('have', 'VBP'), ('more', 'JJR'), ('of', 'IN'), ('their', 'PRP$'), ('own', 'JJ'), ('money', 'NN'), ('to', 'TO'), ('spend', 'VB'), (',', ','), ('save', 'VB'), (',', ','), ('and', 'CC'), ('invest', 'JJS'), ('.', '.')]\n",
      "[('In', 'IN'), ('the', 'DT'), ('last', 'JJ'), ('five', 'CD'), ('years', 'NNS'), (',', ','), ('the', 'DT'), ('tax', 'NN'), ('relief', 'NN'), ('you', 'PRP'), ('passed', 'VBN'), ('has', 'VBZ'), ('left', 'VBN'), ('$', '$'), ('880', 'CD'), ('billion', 'CD'), ('in', 'IN'), ('the', 'DT'), ('hands', 'NNS'), ('of', 'IN'), ('American', 'JJ'), ('workers', 'NNS'), (',', ','), ('investors', 'NNS'), (',', ','), ('small', 'JJ'), ('businesses', 'NNS'), (',', ','), ('and', 'CC'), ('families', 'NNS'), ('--', ':'), ('and', 'CC'), ('they', 'PRP'), ('have', 'VBP'), ('used', 'VBN'), ('it', 'PRP'), ('to', 'TO'), ('help', 'VB'), ('produce', 'VB'), ('more', 'JJR'), ('than', 'IN'), ('four', 'CD'), ('years', 'NNS'), ('of', 'IN'), ('uninterrupted', 'JJ'), ('economic', 'JJ'), ('growth', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Yet', 'RB'), ('the', 'DT'), ('tax', 'NN'), ('relief', 'NN'), ('is', 'VBZ'), ('set', 'VBN'), ('to', 'TO'), ('expire', 'VB'), ('in', 'IN'), ('the', 'DT'), ('next', 'JJ'), ('few', 'JJ'), ('years', 'NNS'), ('.', '.')]\n",
      "[('If', 'IN'), ('we', 'PRP'), ('do', 'VBP'), ('nothing', 'NN'), (',', ','), ('American', 'NNP'), ('families', 'NNS'), ('will', 'MD'), ('face', 'VB'), ('a', 'DT'), ('massive', 'JJ'), ('tax', 'NN'), ('increase', 'NN'), ('they', 'PRP'), ('do', 'VBP'), ('not', 'RB'), ('expect', 'VB'), ('and', 'CC'), ('will', 'MD'), ('not', 'RB'), ('welcome', 'VB'), ('.', '.')]\n",
      "[('Because', 'IN'), ('America', 'NNP'), ('needs', 'VBZ'), ('more', 'JJR'), ('than', 'IN'), ('a', 'DT'), ('temporary', 'JJ'), ('expansion', 'NN'), (',', ','), ('we', 'PRP'), ('need', 'VBP'), ('more', 'JJR'), ('than', 'IN'), ('temporary', 'JJ'), ('tax', 'NN'), ('relief', 'NN'), ('.', '.')]\n",
      "[('I', 'PRP'), ('urge', 'VBP'), ('the', 'DT'), ('Congress', 'NNP'), ('to', 'TO'), ('act', 'VB'), ('responsibly', 'RB'), (',', ','), ('and', 'CC'), ('make', 'VB'), ('the', 'DT'), ('tax', 'NN'), ('cuts', 'NNS'), ('permanent', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Keeping', 'VBG'), ('America', 'NNP'), ('competitive', 'JJ'), ('requires', 'VBZ'), ('us', 'PRP'), ('to', 'TO'), ('be', 'VB'), ('good', 'JJ'), ('stewards', 'NNS'), ('of', 'IN'), ('tax', 'NN'), ('dollars', 'NNS'), ('.', '.')]\n",
      "[('Every', 'DT'), ('year', 'NN'), ('of', 'IN'), ('my', 'PRP$'), ('presidency', 'NN'), (',', ','), ('we', 'PRP'), (\"'ve\", 'VBP'), ('reduced', 'VBN'), ('the', 'DT'), ('growth', 'NN'), ('of', 'IN'), ('non-security', 'JJ'), ('discretionary', 'JJ'), ('spending', 'NN'), (',', ','), ('and', 'CC'), ('last', 'JJ'), ('year', 'NN'), ('you', 'PRP'), ('passed', 'VBD'), ('bills', 'NNS'), ('that', 'IN'), ('cut', 'VBD'), ('this', 'DT'), ('spending', 'NN'), ('.', '.')]\n",
      "[('This', 'DT'), ('year', 'NN'), ('my', 'PRP$'), ('budget', 'NN'), ('will', 'MD'), ('cut', 'VB'), ('it', 'PRP'), ('again', 'RB'), (',', ','), ('and', 'CC'), ('reduce', 'VB'), ('or', 'CC'), ('eliminate', 'VB'), ('more', 'JJR'), ('than', 'IN'), ('140', 'CD'), ('programs', 'NNS'), ('that', 'WDT'), ('are', 'VBP'), ('performing', 'VBG'), ('poorly', 'RB'), ('or', 'CC'), ('not', 'RB'), ('fulfilling', 'JJ'), ('essential', 'JJ'), ('priorities', 'NNS'), ('.', '.')]\n",
      "[('By', 'IN'), ('passing', 'VBG'), ('these', 'DT'), ('reforms', 'NNS'), (',', ','), ('we', 'PRP'), ('will', 'MD'), ('save', 'VB'), ('the', 'DT'), ('American', 'NNP'), ('taxpayer', 'NN'), ('another', 'DT'), ('$', '$'), ('14', 'CD'), ('billion', 'CD'), ('next', 'JJ'), ('year', 'NN'), (',', ','), ('and', 'CC'), ('stay', 'VB'), ('on', 'IN'), ('track', 'NN'), ('to', 'TO'), ('cut', 'VB'), ('the', 'DT'), ('deficit', 'NN'), ('in', 'IN'), ('half', 'NN'), ('by', 'IN'), ('2009', 'CD'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('I', 'PRP'), ('am', 'VBP'), ('pleased', 'JJ'), ('that', 'IN'), ('members', 'NNS'), ('of', 'IN'), ('Congress', 'NNP'), ('are', 'VBP'), ('working', 'VBG'), ('on', 'IN'), ('earmark', 'NN'), ('reform', 'NN'), (',', ','), ('because', 'IN'), ('the', 'DT'), ('federal', 'JJ'), ('budget', 'NN'), ('has', 'VBZ'), ('too', 'RB'), ('many', 'JJ'), ('special', 'JJ'), ('interest', 'NN'), ('projects', 'NNS'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('And', 'CC'), ('we', 'PRP'), ('can', 'MD'), ('tackle', 'VB'), ('this', 'DT'), ('problem', 'NN'), ('together', 'RB'), (',', ','), ('if', 'IN'), ('you', 'PRP'), ('pass', 'VBP'), ('the', 'DT'), ('line-item', 'JJ'), ('veto', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('We', 'PRP'), ('must', 'MD'), ('also', 'RB'), ('confront', 'VB'), ('the', 'DT'), ('larger', 'JJR'), ('challenge', 'NN'), ('of', 'IN'), ('mandatory', 'JJ'), ('spending', 'NN'), (',', ','), ('or', 'CC'), ('entitlements', 'NNS'), ('.', '.')]\n",
      "[('This', 'DT'), ('year', 'NN'), (',', ','), ('the', 'DT'), ('first', 'JJ'), ('of', 'IN'), ('about', 'RB'), ('78', 'CD'), ('million', 'CD'), ('baby', 'NN'), ('boomers', 'NNS'), ('turn', 'VBP'), ('60', 'CD'), (',', ','), ('including', 'VBG'), ('two', 'CD'), ('of', 'IN'), ('my', 'PRP$'), ('Dad', 'NNP'), (\"'s\", 'POS'), ('favorite', 'JJ'), ('people', 'NNS'), ('--', ':'), ('me', 'PRP'), ('and', 'CC'), ('President', 'NNP'), ('Clinton', 'NNP'), ('.', '.')]\n",
      "[('(', '('), ('Laughter', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('This', 'DT'), ('milestone', 'NN'), ('is', 'VBZ'), ('more', 'JJR'), ('than', 'IN'), ('a', 'DT'), ('personal', 'JJ'), ('crisis', 'NN'), ('--', ':'), ('(', '('), ('laughter', 'NN'), (')', ')'), ('--', ':'), ('it', 'PRP'), ('is', 'VBZ'), ('a', 'DT'), ('national', 'JJ'), ('challenge', 'NN'), ('.', '.')]\n",
      "[('The', 'DT'), ('retirement', 'NN'), ('of', 'IN'), ('the', 'DT'), ('baby', 'NN'), ('boom', 'NN'), ('generation', 'NN'), ('will', 'MD'), ('put', 'VB'), ('unprecedented', 'JJ'), ('strains', 'NNS'), ('on', 'IN'), ('the', 'DT'), ('federal', 'JJ'), ('government', 'NN'), ('.', '.')]\n",
      "[('By', 'IN'), ('2030', 'CD'), (',', ','), ('spending', 'VBG'), ('for', 'IN'), ('Social', 'NNP'), ('Security', 'NNP'), (',', ','), ('Medicare', 'NNP'), ('and', 'CC'), ('Medicaid', 'NNP'), ('alone', 'RB'), ('will', 'MD'), ('be', 'VB'), ('almost', 'RB'), ('60', 'CD'), ('percent', 'NN'), ('of', 'IN'), ('the', 'DT'), ('entire', 'JJ'), ('federal', 'JJ'), ('budget', 'NN'), ('.', '.')]\n",
      "[('And', 'CC'), ('that', 'DT'), ('will', 'MD'), ('present', 'VB'), ('future', 'JJ'), ('Congresses', 'NNS'), ('with', 'IN'), ('impossible', 'JJ'), ('choices', 'NNS'), ('--', ':'), ('staggering', 'VBG'), ('tax', 'NN'), ('increases', 'NNS'), (',', ','), ('immense', 'JJ'), ('deficits', 'NNS'), (',', ','), ('or', 'CC'), ('deep', 'JJ'), ('cuts', 'NNS'), ('in', 'IN'), ('every', 'DT'), ('category', 'NN'), ('of', 'IN'), ('spending', 'NN'), ('.', '.')]\n",
      "[('Congress', 'NNP'), ('did', 'VBD'), ('not', 'RB'), ('act', 'VB'), ('last', 'JJ'), ('year', 'NN'), ('on', 'IN'), ('my', 'PRP$'), ('proposal', 'NN'), ('to', 'TO'), ('save', 'VB'), ('Social', 'NNP'), ('Security', 'NNP'), ('--', ':'), ('(', '('), ('applause', 'NN'), (')', ')'), ('--', ':'), ('yet', 'RB'), ('the', 'DT'), ('rising', 'VBG'), ('cost', 'NN'), ('of', 'IN'), ('entitlements', 'NNS'), ('is', 'VBZ'), ('a', 'DT'), ('problem', 'NN'), ('that', 'WDT'), ('is', 'VBZ'), ('not', 'RB'), ('going', 'VBG'), ('away', 'RB'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('And', 'CC'), ('every', 'DT'), ('year', 'NN'), ('we', 'PRP'), ('fail', 'VBP'), ('to', 'TO'), ('act', 'VB'), (',', ','), ('the', 'DT'), ('situation', 'NN'), ('gets', 'VBZ'), ('worse', 'JJR'), ('.', '.')]\n",
      "[('So', 'RB'), ('tonight', 'JJ'), (',', ','), ('I', 'PRP'), ('ask', 'VBP'), ('you', 'PRP'), ('to', 'TO'), ('join', 'VB'), ('me', 'PRP'), ('in', 'IN'), ('creating', 'VBG'), ('a', 'DT'), ('commission', 'NN'), ('to', 'TO'), ('examine', 'VB'), ('the', 'DT'), ('full', 'JJ'), ('impact', 'NN'), ('of', 'IN'), ('baby', 'NN'), ('boom', 'NN'), ('retirements', 'NNS'), ('on', 'IN'), ('Social', 'NNP'), ('Security', 'NNP'), (',', ','), ('Medicare', 'NNP'), (',', ','), ('and', 'CC'), ('Medicaid', 'NNP'), ('.', '.')]\n",
      "[('This', 'DT'), ('commission', 'NN'), ('should', 'MD'), ('include', 'VB'), ('members', 'NNS'), ('of', 'IN'), ('Congress', 'NNP'), ('of', 'IN'), ('both', 'DT'), ('parties', 'NNS'), (',', ','), ('and', 'CC'), ('offer', 'VBP'), ('bipartisan', 'JJ'), ('solutions', 'NNS'), ('.', '.')]\n",
      "[('We', 'PRP'), ('need', 'VBP'), ('to', 'TO'), ('put', 'VB'), ('aside', 'RP'), ('partisan', 'JJ'), ('politics', 'NNS'), ('and', 'CC'), ('work', 'NN'), ('together', 'RB'), ('and', 'CC'), ('get', 'VB'), ('this', 'DT'), ('problem', 'NN'), ('solved', 'VBD'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Keeping', 'VBG'), ('America', 'NNP'), ('competitive', 'JJ'), ('requires', 'VBZ'), ('us', 'PRP'), ('to', 'TO'), ('open', 'VB'), ('more', 'JJR'), ('markets', 'NNS'), ('for', 'IN'), ('all', 'DT'), ('that', 'DT'), ('Americans', 'NNPS'), ('make', 'VBP'), ('and', 'CC'), ('grow', 'VB'), ('.', '.')]\n",
      "[('One', 'CD'), ('out', 'NN'), ('of', 'IN'), ('every', 'DT'), ('five', 'CD'), ('factory', 'NN'), ('jobs', 'NNS'), ('in', 'IN'), ('America', 'NNP'), ('is', 'VBZ'), ('related', 'VBN'), ('to', 'TO'), ('global', 'JJ'), ('trade', 'NN'), (',', ','), ('and', 'CC'), ('we', 'PRP'), ('want', 'VBP'), ('people', 'NNS'), ('everywhere', 'RB'), ('to', 'TO'), ('buy', 'VB'), ('American', 'NNP'), ('.', '.')]\n",
      "[('With', 'IN'), ('open', 'JJ'), ('markets', 'NNS'), ('and', 'CC'), ('a', 'DT'), ('level', 'JJ'), ('playing', 'NN'), ('field', 'NN'), (',', ','), ('no', 'DT'), ('one', 'NN'), ('can', 'MD'), ('out-produce', 'VB'), ('or', 'CC'), ('out-compete', 'VB'), ('the', 'DT'), ('American', 'JJ'), ('worker', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Keeping', 'VBG'), ('America', 'NNP'), ('competitive', 'JJ'), ('requires', 'VBZ'), ('an', 'DT'), ('immigration', 'NN'), ('system', 'NN'), ('that', 'WDT'), ('upholds', 'VBZ'), ('our', 'PRP$'), ('laws', 'NNS'), (',', ','), ('reflects', 'VBZ'), ('our', 'PRP$'), ('values', 'NNS'), (',', ','), ('and', 'CC'), ('serves', 'VBZ'), ('the', 'DT'), ('interests', 'NNS'), ('of', 'IN'), ('our', 'PRP$'), ('economy', 'NN'), ('.', '.')]\n",
      "[('Our', 'PRP$'), ('nation', 'NN'), ('needs', 'VBZ'), ('orderly', 'JJ'), ('and', 'CC'), ('secure', 'JJ'), ('borders', 'NNS'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('To', 'TO'), ('meet', 'VB'), ('this', 'DT'), ('goal', 'NN'), (',', ','), ('we', 'PRP'), ('must', 'MD'), ('have', 'VB'), ('stronger', 'JJR'), ('immigration', 'NN'), ('enforcement', 'NN'), ('and', 'CC'), ('border', 'NN'), ('protection', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('And', 'CC'), ('we', 'PRP'), ('must', 'MD'), ('have', 'VB'), ('a', 'DT'), ('rational', 'JJ'), (',', ','), ('humane', 'JJ'), ('guest', 'JJS'), ('worker', 'NN'), ('program', 'NN'), ('that', 'WDT'), ('rejects', 'VBZ'), ('amnesty', 'JJ'), (',', ','), ('allows', 'VBZ'), ('temporary', 'JJ'), ('jobs', 'NNS'), ('for', 'IN'), ('people', 'NNS'), ('who', 'WP'), ('seek', 'VBP'), ('them', 'PRP'), ('legally', 'RB'), (',', ','), ('and', 'CC'), ('reduces', 'NNS'), ('smuggling', 'VBG'), ('and', 'CC'), ('crime', 'NN'), ('at', 'IN'), ('the', 'DT'), ('border', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Keeping', 'VBG'), ('America', 'NNP'), ('competitive', 'JJ'), ('requires', 'VBZ'), ('affordable', 'JJ'), ('health', 'NN'), ('care', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Our', 'PRP$'), ('government', 'NN'), ('has', 'VBZ'), ('a', 'DT'), ('responsibility', 'NN'), ('to', 'TO'), ('provide', 'VB'), ('health', 'NN'), ('care', 'NN'), ('for', 'IN'), ('the', 'DT'), ('poor', 'JJ'), ('and', 'CC'), ('the', 'DT'), ('elderly', 'JJ'), (',', ','), ('and', 'CC'), ('we', 'PRP'), ('are', 'VBP'), ('meeting', 'VBG'), ('that', 'IN'), ('responsibility', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('For', 'IN'), ('all', 'DT'), ('Americans', 'NNPS'), ('--', ':'), ('for', 'IN'), ('all', 'DT'), ('Americans', 'NNPS'), (',', ','), ('we', 'PRP'), ('must', 'MD'), ('confront', 'VB'), ('the', 'DT'), ('rising', 'VBG'), ('cost', 'NN'), ('of', 'IN'), ('care', 'NN'), (',', ','), ('strengthen', 'VB'), ('the', 'DT'), ('doctor-patient', 'JJ'), ('relationship', 'NN'), (',', ','), ('and', 'CC'), ('help', 'NN'), ('people', 'NNS'), ('afford', 'VBP'), ('the', 'DT'), ('insurance', 'NN'), ('coverage', 'NN'), ('they', 'PRP'), ('need', 'VBP'), ('.', '.')]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('We', 'PRP'), ('will', 'MD'), ('make', 'VB'), ('wider', 'JJR'), ('use', 'NN'), ('of', 'IN'), ('electronic', 'JJ'), ('records', 'NNS'), ('and', 'CC'), ('other', 'JJ'), ('health', 'NN'), ('information', 'NN'), ('technology', 'NN'), (',', ','), ('to', 'TO'), ('help', 'VB'), ('control', 'VB'), ('costs', 'NNS'), ('and', 'CC'), ('reduce', 'VB'), ('dangerous', 'JJ'), ('medical', 'JJ'), ('errors', 'NNS'), ('.', '.')]\n",
      "[('We', 'PRP'), ('will', 'MD'), ('strengthen', 'VB'), ('health', 'NN'), ('savings', 'NNS'), ('accounts', 'NNS'), ('--', ':'), ('making', 'VBG'), ('sure', 'JJ'), ('individuals', 'NNS'), ('and', 'CC'), ('small', 'JJ'), ('business', 'NN'), ('employees', 'NNS'), ('can', 'MD'), ('buy', 'VB'), ('insurance', 'NN'), ('with', 'IN'), ('the', 'DT'), ('same', 'JJ'), ('advantages', 'VBZ'), ('that', 'IN'), ('people', 'NNS'), ('working', 'VBG'), ('for', 'IN'), ('big', 'JJ'), ('businesses', 'NNS'), ('now', 'RB'), ('get', 'VBP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('We', 'PRP'), ('will', 'MD'), ('do', 'VB'), ('more', 'JJR'), ('to', 'TO'), ('make', 'VB'), ('this', 'DT'), ('coverage', 'NN'), ('portable', 'JJ'), (',', ','), ('so', 'IN'), ('workers', 'NNS'), ('can', 'MD'), ('switch', 'VB'), ('jobs', 'NNS'), ('without', 'IN'), ('having', 'VBG'), ('to', 'TO'), ('worry', 'VB'), ('about', 'IN'), ('losing', 'VBG'), ('their', 'PRP$'), ('health', 'NN'), ('insurance', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('And', 'CC'), ('because', 'IN'), ('lawsuits', 'NNS'), ('are', 'VBP'), ('driving', 'VBG'), ('many', 'JJ'), ('good', 'JJ'), ('doctors', 'NNS'), ('out', 'IN'), ('of', 'IN'), ('practice', 'NN'), ('--', ':'), ('leaving', 'VBG'), ('women', 'NNS'), ('in', 'IN'), ('nearly', 'RB'), ('1,500', 'CD'), ('American', 'JJ'), ('counties', 'NNS'), ('without', 'IN'), ('a', 'DT'), ('single', 'JJ'), ('OB/GYN', 'NNP'), ('--', ':'), ('I', 'PRP'), ('ask', 'VBP'), ('the', 'DT'), ('Congress', 'NNP'), ('to', 'TO'), ('pass', 'VB'), ('medical', 'JJ'), ('liability', 'NN'), ('reform', 'NN'), ('this', 'DT'), ('year', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Keeping', 'VBG'), ('America', 'NNP'), ('competitive', 'JJ'), ('requires', 'VBZ'), ('affordable', 'JJ'), ('energy', 'NN'), ('.', '.')]\n",
      "[('And', 'CC'), ('here', 'RB'), ('we', 'PRP'), ('have', 'VBP'), ('a', 'DT'), ('serious', 'JJ'), ('problem', 'NN'), (':', ':'), ('America', 'NNP'), ('is', 'VBZ'), ('addicted', 'VBN'), ('to', 'TO'), ('oil', 'NN'), (',', ','), ('which', 'WDT'), ('is', 'VBZ'), ('often', 'RB'), ('imported', 'VBN'), ('from', 'IN'), ('unstable', 'JJ'), ('parts', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('world', 'NN'), ('.', '.')]\n",
      "[('The', 'DT'), ('best', 'JJS'), ('way', 'NN'), ('to', 'TO'), ('break', 'VB'), ('this', 'DT'), ('addiction', 'NN'), ('is', 'VBZ'), ('through', 'IN'), ('technology', 'NN'), ('.', '.')]\n",
      "[('Since', 'IN'), ('2001', 'CD'), (',', ','), ('we', 'PRP'), ('have', 'VBP'), ('spent', 'VBN'), ('nearly', 'RB'), ('$', '$'), ('10', 'CD'), ('billion', 'CD'), ('to', 'TO'), ('develop', 'VB'), ('cleaner', 'JJR'), (',', ','), ('cheaper', 'JJR'), (',', ','), ('and', 'CC'), ('more', 'RBR'), ('reliable', 'JJ'), ('alternative', 'JJ'), ('energy', 'NN'), ('sources', 'NNS'), ('--', ':'), ('and', 'CC'), ('we', 'PRP'), ('are', 'VBP'), ('on', 'IN'), ('the', 'DT'), ('threshold', 'NN'), ('of', 'IN'), ('incredible', 'JJ'), ('advances', 'NNS'), ('.', '.')]\n",
      "[('So', 'RB'), ('tonight', 'JJ'), (',', ','), ('I', 'PRP'), ('announce', 'VBP'), ('the', 'DT'), ('Advanced', 'NNP'), ('Energy', 'NNP'), ('Initiative', 'NNP'), ('--', ':'), ('a', 'DT'), ('22-percent', 'JJ'), ('increase', 'NN'), ('in', 'IN'), ('clean-energy', 'JJ'), ('research', 'NN'), ('--', ':'), ('at', 'IN'), ('the', 'DT'), ('Department', 'NNP'), ('of', 'IN'), ('Energy', 'NNP'), (',', ','), ('to', 'TO'), ('push', 'VB'), ('for', 'IN'), ('breakthroughs', 'NNS'), ('in', 'IN'), ('two', 'CD'), ('vital', 'JJ'), ('areas', 'NNS'), ('.', '.')]\n",
      "[('To', 'TO'), ('change', 'VB'), ('how', 'WRB'), ('we', 'PRP'), ('power', 'NN'), ('our', 'PRP$'), ('homes', 'NNS'), ('and', 'CC'), ('offices', 'NNS'), (',', ','), ('we', 'PRP'), ('will', 'MD'), ('invest', 'VB'), ('more', 'RBR'), ('in', 'IN'), ('zero-emission', 'JJ'), ('coal-fired', 'JJ'), ('plants', 'NNS'), (',', ','), ('revolutionary', 'JJ'), ('solar', 'NN'), ('and', 'CC'), ('wind', 'NN'), ('technologies', 'NNS'), (',', ','), ('and', 'CC'), ('clean', 'JJ'), (',', ','), ('safe', 'JJ'), ('nuclear', 'JJ'), ('energy', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('We', 'PRP'), ('must', 'MD'), ('also', 'RB'), ('change', 'VB'), ('how', 'WRB'), ('we', 'PRP'), ('power', 'NN'), ('our', 'PRP$'), ('automobiles', 'NNS'), ('.', '.')]\n",
      "[('We', 'PRP'), ('will', 'MD'), ('increase', 'VB'), ('our', 'PRP$'), ('research', 'NN'), ('in', 'IN'), ('better', 'JJR'), ('batteries', 'NNS'), ('for', 'IN'), ('hybrid', 'JJ'), ('and', 'CC'), ('electric', 'JJ'), ('cars', 'NNS'), (',', ','), ('and', 'CC'), ('in', 'IN'), ('pollution-free', 'JJ'), ('cars', 'NNS'), ('that', 'WDT'), ('run', 'VBP'), ('on', 'IN'), ('hydrogen', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), (\"'ll\", 'MD'), ('also', 'RB'), ('fund', 'VB'), ('additional', 'JJ'), ('research', 'NN'), ('in', 'IN'), ('cutting-edge', 'JJ'), ('methods', 'NNS'), ('of', 'IN'), ('producing', 'VBG'), ('ethanol', 'NN'), (',', ','), ('not', 'RB'), ('just', 'RB'), ('from', 'IN'), ('corn', 'NN'), (',', ','), ('but', 'CC'), ('from', 'IN'), ('wood', 'NN'), ('chips', 'NNS'), ('and', 'CC'), ('stalks', 'NNS'), (',', ','), ('or', 'CC'), ('switch', 'VB'), ('grass', 'NN'), ('.', '.')]\n",
      "[('Our', 'PRP$'), ('goal', 'NN'), ('is', 'VBZ'), ('to', 'TO'), ('make', 'VB'), ('this', 'DT'), ('new', 'JJ'), ('kind', 'NN'), ('of', 'IN'), ('ethanol', 'JJ'), ('practical', 'JJ'), ('and', 'CC'), ('competitive', 'JJ'), ('within', 'IN'), ('six', 'CD'), ('years', 'NNS'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Breakthroughs', 'NNS'), ('on', 'IN'), ('this', 'DT'), ('and', 'CC'), ('other', 'JJ'), ('new', 'JJ'), ('technologies', 'NNS'), ('will', 'MD'), ('help', 'VB'), ('us', 'PRP'), ('reach', 'VB'), ('another', 'DT'), ('great', 'JJ'), ('goal', 'NN'), (':', ':'), ('to', 'TO'), ('replace', 'VB'), ('more', 'JJR'), ('than', 'IN'), ('75', 'CD'), ('percent', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('oil', 'NN'), ('imports', 'NNS'), ('from', 'IN'), ('the', 'DT'), ('Middle', 'NNP'), ('East', 'NNP'), ('by', 'IN'), ('2025', 'CD'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('By', 'IN'), ('applying', 'VBG'), ('the', 'DT'), ('talent', 'NN'), ('and', 'CC'), ('technology', 'NN'), ('of', 'IN'), ('America', 'NNP'), (',', ','), ('this', 'DT'), ('country', 'NN'), ('can', 'MD'), ('dramatically', 'RB'), ('improve', 'VB'), ('our', 'PRP$'), ('environment', 'NN'), (',', ','), ('move', 'VB'), ('beyond', 'IN'), ('a', 'DT'), ('petroleum-based', 'JJ'), ('economy', 'NN'), (',', ','), ('and', 'CC'), ('make', 'VB'), ('our', 'PRP$'), ('dependence', 'NN'), ('on', 'IN'), ('Middle', 'NNP'), ('Eastern', 'NNP'), ('oil', 'NN'), ('a', 'DT'), ('thing', 'NN'), ('of', 'IN'), ('the', 'DT'), ('past', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('And', 'CC'), ('to', 'TO'), ('keep', 'VB'), ('America', 'NNP'), ('competitive', 'JJ'), (',', ','), ('one', 'CD'), ('commitment', 'NN'), ('is', 'VBZ'), ('necessary', 'JJ'), ('above', 'IN'), ('all', 'DT'), (':', ':'), ('We', 'PRP'), ('must', 'MD'), ('continue', 'VB'), ('to', 'TO'), ('lead', 'VB'), ('the', 'DT'), ('world', 'NN'), ('in', 'IN'), ('human', 'JJ'), ('talent', 'NN'), ('and', 'CC'), ('creativity', 'NN'), ('.', '.')]\n",
      "[('Our', 'PRP$'), ('greatest', 'JJS'), ('advantage', 'NN'), ('in', 'IN'), ('the', 'DT'), ('world', 'NN'), ('has', 'VBZ'), ('always', 'RB'), ('been', 'VBN'), ('our', 'PRP$'), ('educated', 'VBN'), (',', ','), ('hardworking', 'VBG'), (',', ','), ('ambitious', 'JJ'), ('people', 'NNS'), ('--', ':'), ('and', 'CC'), ('we', 'PRP'), (\"'re\", 'VBP'), ('going', 'VBG'), ('to', 'TO'), ('keep', 'VB'), ('that', 'DT'), ('edge', 'NN'), ('.', '.')]\n",
      "[('Tonight', 'NN'), ('I', 'PRP'), ('announce', 'VBP'), ('an', 'DT'), ('American', 'JJ'), ('Competitiveness', 'NNP'), ('Initiative', 'NNP'), (',', ','), ('to', 'TO'), ('encourage', 'VB'), ('innovation', 'NN'), ('throughout', 'IN'), ('our', 'PRP$'), ('economy', 'NN'), (',', ','), ('and', 'CC'), ('to', 'TO'), ('give', 'VB'), ('our', 'PRP$'), ('nation', 'NN'), (\"'s\", 'POS'), ('children', 'NNS'), ('a', 'DT'), ('firm', 'NN'), ('grounding', 'VBG'), ('in', 'IN'), ('math', 'NN'), ('and', 'CC'), ('science', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('First', 'RB'), (',', ','), ('I', 'PRP'), ('propose', 'VBP'), ('to', 'TO'), ('double', 'VB'), ('the', 'DT'), ('federal', 'JJ'), ('commitment', 'NN'), ('to', 'TO'), ('the', 'DT'), ('most', 'RBS'), ('critical', 'JJ'), ('basic', 'JJ'), ('research', 'NN'), ('programs', 'NNS'), ('in', 'IN'), ('the', 'DT'), ('physical', 'JJ'), ('sciences', 'NNS'), ('over', 'IN'), ('the', 'DT'), ('next', 'JJ'), ('10', 'CD'), ('years', 'NNS'), ('.', '.')]\n",
      "[('This', 'DT'), ('funding', 'NN'), ('will', 'MD'), ('support', 'VB'), ('the', 'DT'), ('work', 'NN'), ('of', 'IN'), ('America', 'NNP'), (\"'s\", 'POS'), ('most', 'RBS'), ('creative', 'JJ'), ('minds', 'NNS'), ('as', 'IN'), ('they', 'PRP'), ('explore', 'VBP'), ('promising', 'VBG'), ('areas', 'NNS'), ('such', 'JJ'), ('as', 'IN'), ('nanotechnology', 'NN'), (',', ','), ('supercomputing', 'NN'), (',', ','), ('and', 'CC'), ('alternative', 'JJ'), ('energy', 'NN'), ('sources', 'NNS'), ('.', '.')]\n",
      "[('Second', 'JJ'), (',', ','), ('I', 'PRP'), ('propose', 'VBP'), ('to', 'TO'), ('make', 'VB'), ('permanent', 'JJ'), ('the', 'DT'), ('research', 'NN'), ('and', 'CC'), ('development', 'NN'), ('tax', 'NN'), ('credit', 'NN'), ('--', ':'), ('(', '('), ('applause', 'NN'), (')', ')'), ('--', ':'), ('to', 'TO'), ('encourage', 'VB'), ('bolder', 'VB'), ('private-sector', 'JJ'), ('initiatives', 'NNS'), ('in', 'IN'), ('technology', 'NN'), ('.', '.')]\n",
      "[('With', 'IN'), ('more', 'JJR'), ('research', 'NN'), ('in', 'IN'), ('both', 'CC'), ('the', 'DT'), ('public', 'NN'), ('and', 'CC'), ('private', 'JJ'), ('sectors', 'NNS'), (',', ','), ('we', 'PRP'), ('will', 'MD'), ('improve', 'VB'), ('our', 'PRP$'), ('quality', 'NN'), ('of', 'IN'), ('life', 'NN'), ('--', ':'), ('and', 'CC'), ('ensure', 'VB'), ('that', 'DT'), ('America', 'NNP'), ('will', 'MD'), ('lead', 'VB'), ('the', 'DT'), ('world', 'NN'), ('in', 'IN'), ('opportunity', 'NN'), ('and', 'CC'), ('innovation', 'NN'), ('for', 'IN'), ('decades', 'NNS'), ('to', 'TO'), ('come', 'VB'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Third', 'NNP'), (',', ','), ('we', 'PRP'), ('need', 'VBP'), ('to', 'TO'), ('encourage', 'VB'), ('children', 'NNS'), ('to', 'TO'), ('take', 'VB'), ('more', 'JJR'), ('math', 'NN'), ('and', 'CC'), ('science', 'NN'), (',', ','), ('and', 'CC'), ('to', 'TO'), ('make', 'VB'), ('sure', 'JJ'), ('those', 'DT'), ('courses', 'NNS'), ('are', 'VBP'), ('rigorous', 'JJ'), ('enough', 'RB'), ('to', 'TO'), ('compete', 'VB'), ('with', 'IN'), ('other', 'JJ'), ('nations', 'NNS'), ('.', '.')]\n",
      "[('We', 'PRP'), (\"'ve\", 'VBP'), ('made', 'VBN'), ('a', 'DT'), ('good', 'JJ'), ('start', 'NN'), ('in', 'IN'), ('the', 'DT'), ('early', 'JJ'), ('grades', 'NNS'), ('with', 'IN'), ('the', 'DT'), ('No', 'NNP'), ('Child', 'NNP'), ('Left', 'NNP'), ('Behind', 'NNP'), ('Act', 'NNP'), (',', ','), ('which', 'WDT'), ('is', 'VBZ'), ('raising', 'VBG'), ('standards', 'NNS'), ('and', 'CC'), ('lifting', 'VBG'), ('test', 'NN'), ('scores', 'NNS'), ('across', 'IN'), ('our', 'PRP$'), ('country', 'NN'), ('.', '.')]\n",
      "[('Tonight', 'NNP'), ('I', 'PRP'), ('propose', 'VBP'), ('to', 'TO'), ('train', 'VB'), ('70,000', 'CD'), ('high', 'JJ'), ('school', 'NN'), ('teachers', 'NNS'), ('to', 'TO'), ('lead', 'VB'), ('advanced-placement', 'JJ'), ('courses', 'NNS'), ('in', 'IN'), ('math', 'NN'), ('and', 'CC'), ('science', 'NN'), (',', ','), ('bring', 'VBG'), ('30,000', 'CD'), ('math', 'NN'), ('and', 'CC'), ('science', 'NN'), ('professionals', 'NNS'), ('to', 'TO'), ('teach', 'VB'), ('in', 'IN'), ('classrooms', 'NNS'), (',', ','), ('and', 'CC'), ('give', 'VB'), ('early', 'JJ'), ('help', 'NN'), ('to', 'TO'), ('students', 'NNS'), ('who', 'WP'), ('struggle', 'VBP'), ('with', 'IN'), ('math', 'NN'), (',', ','), ('so', 'IN'), ('they', 'PRP'), ('have', 'VBP'), ('a', 'DT'), ('better', 'JJR'), ('chance', 'NN'), ('at', 'IN'), ('good', 'JJ'), (',', ','), ('high-wage', 'JJ'), ('jobs', 'NNS'), ('.', '.')]\n",
      "[('If', 'IN'), ('we', 'PRP'), ('ensure', 'VB'), ('that', 'IN'), ('America', 'NNP'), (\"'s\", 'POS'), ('children', 'NNS'), ('succeed', 'VB'), ('in', 'IN'), ('life', 'NN'), (',', ','), ('they', 'PRP'), ('will', 'MD'), ('ensure', 'VB'), ('that', 'IN'), ('America', 'NNP'), ('succeeds', 'VBZ'), ('in', 'IN'), ('the', 'DT'), ('world', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Preparing', 'VBG'), ('our', 'PRP$'), ('nation', 'NN'), ('to', 'TO'), ('compete', 'VB'), ('in', 'IN'), ('the', 'DT'), ('world', 'NN'), ('is', 'VBZ'), ('a', 'DT'), ('goal', 'NN'), ('that', 'IN'), ('all', 'DT'), ('of', 'IN'), ('us', 'PRP'), ('can', 'MD'), ('share', 'NN'), ('.', '.')]\n",
      "[('I', 'PRP'), ('urge', 'VBP'), ('you', 'PRP'), ('to', 'TO'), ('support', 'VB'), ('the', 'DT'), ('American', 'JJ'), ('Competitiveness', 'NNP'), ('Initiative', 'NNP'), (',', ','), ('and', 'CC'), ('together', 'RB'), ('we', 'PRP'), ('will', 'MD'), ('show', 'VB'), ('the', 'DT'), ('world', 'NN'), ('what', 'WP'), ('the', 'DT'), ('American', 'JJ'), ('people', 'NNS'), ('can', 'MD'), ('achieve', 'VB'), ('.', '.')]\n",
      "[('America', 'NNP'), ('is', 'VBZ'), ('a', 'DT'), ('great', 'JJ'), ('force', 'NN'), ('for', 'IN'), ('freedom', 'NN'), ('and', 'CC'), ('prosperity', 'NN'), ('.', '.')]\n",
      "[('Yet', 'RB'), ('our', 'PRP$'), ('greatness', 'NN'), ('is', 'VBZ'), ('not', 'RB'), ('measured', 'VBN'), ('in', 'IN'), ('power', 'NN'), ('or', 'CC'), ('luxuries', 'NNS'), (',', ','), ('but', 'CC'), ('by', 'IN'), ('who', 'WP'), ('we', 'PRP'), ('are', 'VBP'), ('and', 'CC'), ('how', 'WRB'), ('we', 'PRP'), ('treat', 'VBP'), ('one', 'CD'), ('another', 'DT'), ('.', '.')]\n",
      "[('So', 'IN'), ('we', 'PRP'), ('strive', 'VBP'), ('to', 'TO'), ('be', 'VB'), ('a', 'DT'), ('compassionate', 'NN'), (',', ','), ('decent', 'NN'), (',', ','), ('hopeful', 'JJ'), ('society', 'NN'), ('.', '.')]\n",
      "[('In', 'IN'), ('recent', 'JJ'), ('years', 'NNS'), (',', ','), ('America', 'NNP'), ('has', 'VBZ'), ('become', 'VBN'), ('a', 'DT'), ('more', 'RBR'), ('hopeful', 'JJ'), ('nation', 'NN'), ('.', '.')]\n",
      "[('Violent', 'JJ'), ('crime', 'NN'), ('rates', 'NNS'), ('have', 'VBP'), ('fallen', 'VBN'), ('to', 'TO'), ('their', 'PRP$'), ('lowest', 'JJS'), ('levels', 'NNS'), ('since', 'IN'), ('the', 'DT'), ('1970s', 'CD'), ('.', '.')]\n",
      "[('Welfare', 'NN'), ('cases', 'NNS'), ('have', 'VBP'), ('dropped', 'VBN'), ('by', 'IN'), ('more', 'JJR'), ('than', 'IN'), ('half', 'NN'), ('over', 'IN'), ('the', 'DT'), ('past', 'JJ'), ('decade', 'NN'), ('.', '.')]\n",
      "[('Drug', 'NN'), ('use', 'NN'), ('among', 'IN'), ('youth', 'NN'), ('is', 'VBZ'), ('down', 'RB'), ('19', 'CD'), ('percent', 'NN'), ('since', 'IN'), ('2001', 'CD'), ('.', '.')]\n",
      "[('There', 'EX'), ('are', 'VBP'), ('fewer', 'JJR'), ('abortions', 'NNS'), ('in', 'IN'), ('America', 'NNP'), ('than', 'IN'), ('at', 'IN'), ('any', 'DT'), ('point', 'NN'), ('in', 'IN'), ('the', 'DT'), ('last', 'JJ'), ('three', 'CD'), ('decades', 'NNS'), (',', ','), ('and', 'CC'), ('the', 'DT'), ('number', 'NN'), ('of', 'IN'), ('children', 'NNS'), ('born', 'VBN'), ('to', 'TO'), ('teenage', 'VB'), ('mothers', 'NNS'), ('has', 'VBZ'), ('been', 'VBN'), ('falling', 'VBG'), ('for', 'IN'), ('a', 'DT'), ('dozen', 'NN'), ('years', 'NNS'), ('in', 'IN'), ('a', 'DT'), ('row', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('These', 'DT'), ('gains', 'NNS'), ('are', 'VBP'), ('evidence', 'NN'), ('of', 'IN'), ('a', 'DT'), ('quiet', 'JJ'), ('transformation', 'NN'), ('--', ':'), ('a', 'DT'), ('revolution', 'NN'), ('of', 'IN'), ('conscience', 'NN'), (',', ','), ('in', 'IN'), ('which', 'WDT'), ('a', 'DT'), ('rising', 'VBG'), ('generation', 'NN'), ('is', 'VBZ'), ('finding', 'VBG'), ('that', 'IN'), ('a', 'DT'), ('life', 'NN'), ('of', 'IN'), ('personal', 'JJ'), ('responsibility', 'NN'), ('is', 'VBZ'), ('a', 'DT'), ('life', 'NN'), ('of', 'IN'), ('fulfillment', 'NN'), ('.', '.')]\n",
      "[('Government', 'NNP'), ('has', 'VBZ'), ('played', 'VBN'), ('a', 'DT'), ('role', 'NN'), ('.', '.')]\n",
      "[('Wise', 'NNP'), ('policies', 'NNS'), (',', ','), ('such', 'JJ'), ('as', 'IN'), ('welfare', 'NN'), ('reform', 'NN'), ('and', 'CC'), ('drug', 'NN'), ('education', 'NN'), ('and', 'CC'), ('support', 'NN'), ('for', 'IN'), ('abstinence', 'NN'), ('and', 'CC'), ('adoption', 'NN'), ('have', 'VBP'), ('made', 'VBN'), ('a', 'DT'), ('difference', 'NN'), ('in', 'IN'), ('the', 'DT'), ('character', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('country', 'NN'), ('.', '.')]\n",
      "[('And', 'CC'), ('everyone', 'NN'), ('here', 'RB'), ('tonight', 'RB'), (',', ','), ('Democrat', 'NNP'), ('and', 'CC'), ('Republican', 'NNP'), (',', ','), ('has', 'VBZ'), ('a', 'DT'), ('right', 'NN'), ('to', 'TO'), ('be', 'VB'), ('proud', 'JJ'), ('of', 'IN'), ('this', 'DT'), ('record', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Yet', 'RB'), ('many', 'JJ'), ('Americans', 'NNPS'), (',', ','), ('especially', 'RB'), ('parents', 'NNS'), (',', ','), ('still', 'RB'), ('have', 'VBP'), ('deep', 'JJ'), ('concerns', 'NNS'), ('about', 'IN'), ('the', 'DT'), ('direction', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('culture', 'NN'), (',', ','), ('and', 'CC'), ('the', 'DT'), ('health', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('most', 'JJS'), ('basic', 'JJ'), ('institutions', 'NNS'), ('.', '.')]\n",
      "[('They', 'PRP'), (\"'re\", 'VBP'), ('concerned', 'VBN'), ('about', 'IN'), ('unethical', 'JJ'), ('conduct', 'NN'), ('by', 'IN'), ('public', 'JJ'), ('officials', 'NNS'), (',', ','), ('and', 'CC'), ('discouraged', 'VBN'), ('by', 'IN'), ('activist', 'NN'), ('courts', 'NNS'), ('that', 'WDT'), ('try', 'VBP'), ('to', 'TO'), ('redefine', 'VB'), ('marriage', 'NN'), ('.', '.')]\n",
      "[('They', 'PRP'), ('worry', 'VBP'), ('about', 'IN'), ('children', 'NNS'), ('in', 'IN'), ('our', 'PRP$'), ('society', 'NN'), ('who', 'WP'), ('need', 'VBP'), ('direction', 'NN'), ('and', 'CC'), ('love', 'NN'), (',', ','), ('and', 'CC'), ('about', 'IN'), ('fellow', 'JJ'), ('citizens', 'NNS'), ('still', 'RB'), ('displaced', 'VBN'), ('by', 'IN'), ('natural', 'JJ'), ('disaster', 'NN'), (',', ','), ('and', 'CC'), ('about', 'IN'), ('suffering', 'VBG'), ('caused', 'VBN'), ('by', 'IN'), ('treatable', 'JJ'), ('diseases', 'NNS'), ('.', '.')]\n",
      "[('As', 'IN'), ('we', 'PRP'), ('look', 'VBP'), ('at', 'IN'), ('these', 'DT'), ('challenges', 'NNS'), (',', ','), ('we', 'PRP'), ('must', 'MD'), ('never', 'RB'), ('give', 'VB'), ('in', 'IN'), ('to', 'TO'), ('the', 'DT'), ('belief', 'NN'), ('that', 'IN'), ('America', 'NNP'), ('is', 'VBZ'), ('in', 'IN'), ('decline', 'NN'), (',', ','), ('or', 'CC'), ('that', 'IN'), ('our', 'PRP$'), ('culture', 'NN'), ('is', 'VBZ'), ('doomed', 'VBN'), ('to', 'TO'), ('unravel', 'VB'), ('.', '.')]\n",
      "[('The', 'DT'), ('American', 'JJ'), ('people', 'NNS'), ('know', 'VBP'), ('better', 'JJR'), ('than', 'IN'), ('that', 'DT'), ('.', '.')]\n",
      "[('We', 'PRP'), ('have', 'VBP'), ('proven', 'VBN'), ('the', 'DT'), ('pessimists', 'NNS'), ('wrong', 'JJ'), ('before', 'RB'), ('--', ':'), ('and', 'CC'), ('we', 'PRP'), ('will', 'MD'), ('do', 'VB'), ('it', 'PRP'), ('again', 'RB'), ('.', '.')]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('A', 'DT'), ('hopeful', 'JJ'), ('society', 'NN'), ('depends', 'VBZ'), ('on', 'IN'), ('courts', 'NNS'), ('that', 'IN'), ('deliver', 'VBP'), ('equal', 'JJ'), ('justice', 'NN'), ('under', 'IN'), ('the', 'DT'), ('law', 'NN'), ('.', '.')]\n",
      "[('The', 'DT'), ('Supreme', 'NNP'), ('Court', 'NNP'), ('now', 'RB'), ('has', 'VBZ'), ('two', 'CD'), ('superb', 'JJ'), ('new', 'JJ'), ('members', 'NNS'), ('--', ':'), ('new', 'JJ'), ('members', 'NNS'), ('on', 'IN'), ('its', 'PRP$'), ('bench', 'NN'), (':', ':'), ('Chief', 'JJ'), ('Justice', 'NNP'), ('John', 'NNP'), ('Roberts', 'NNP'), ('and', 'CC'), ('Justice', 'NNP'), ('Sam', 'NNP'), ('Alito', 'NNP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('I', 'PRP'), ('thank', 'VBD'), ('the', 'DT'), ('Senate', 'NNP'), ('for', 'IN'), ('confirming', 'VBG'), ('both', 'DT'), ('of', 'IN'), ('them', 'PRP'), ('.', '.')]\n",
      "[('I', 'PRP'), ('will', 'MD'), ('continue', 'VB'), ('to', 'TO'), ('nominate', 'VB'), ('men', 'NNS'), ('and', 'CC'), ('women', 'NNS'), ('who', 'WP'), ('understand', 'VBP'), ('that', 'IN'), ('judges', 'NNS'), ('must', 'MD'), ('be', 'VB'), ('servants', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('law', 'NN'), (',', ','), ('and', 'CC'), ('not', 'RB'), ('legislate', 'VB'), ('from', 'IN'), ('the', 'DT'), ('bench', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Today', 'NN'), ('marks', 'VBZ'), ('the', 'DT'), ('official', 'JJ'), ('retirement', 'NN'), ('of', 'IN'), ('a', 'DT'), ('very', 'RB'), ('special', 'JJ'), ('American', 'NNP'), ('.', '.')]\n",
      "[('For', 'IN'), ('24', 'CD'), ('years', 'NNS'), ('of', 'IN'), ('faithful', 'JJ'), ('service', 'NN'), ('to', 'TO'), ('our', 'PRP$'), ('nation', 'NN'), (',', ','), ('the', 'DT'), ('United', 'NNP'), ('States', 'NNPS'), ('is', 'VBZ'), ('grateful', 'JJ'), ('to', 'TO'), ('Justice', 'NNP'), ('Sandra', 'NNP'), ('Day', 'NNP'), (\"O'Connor\", 'NNP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('A', 'DT'), ('hopeful', 'JJ'), ('society', 'NN'), ('has', 'VBZ'), ('institutions', 'NNS'), ('of', 'IN'), ('science', 'NN'), ('and', 'CC'), ('medicine', 'NN'), ('that', 'WDT'), ('do', 'VBP'), ('not', 'RB'), ('cut', 'VB'), ('ethical', 'JJ'), ('corners', 'NNS'), (',', ','), ('and', 'CC'), ('that', 'IN'), ('recognize', 'VBP'), ('the', 'DT'), ('matchless', 'NN'), ('value', 'NN'), ('of', 'IN'), ('every', 'DT'), ('life', 'NN'), ('.', '.')]\n",
      "[('Tonight', 'NNP'), ('I', 'PRP'), ('ask', 'VBP'), ('you', 'PRP'), ('to', 'TO'), ('pass', 'VB'), ('legislation', 'NN'), ('to', 'TO'), ('prohibit', 'VB'), ('the', 'DT'), ('most', 'RBS'), ('egregious', 'JJ'), ('abuses', 'NNS'), ('of', 'IN'), ('medical', 'JJ'), ('research', 'NN'), (':', ':'), ('human', 'JJ'), ('cloning', 'VBG'), ('in', 'IN'), ('all', 'DT'), ('its', 'PRP$'), ('forms', 'NNS'), (',', ','), ('creating', 'VBG'), ('or', 'CC'), ('implanting', 'VBG'), ('embryos', 'NN'), ('for', 'IN'), ('experiments', 'NNS'), (',', ','), ('creating', 'VBG'), ('human-animal', 'JJ'), ('hybrids', 'NNS'), (',', ','), ('and', 'CC'), ('buying', 'NN'), (',', ','), ('selling', 'NN'), (',', ','), ('or', 'CC'), ('patenting', 'VBG'), ('human', 'JJ'), ('embryos', 'NN'), ('.', '.')]\n",
      "[('Human', 'NNP'), ('life', 'NN'), ('is', 'VBZ'), ('a', 'DT'), ('gift', 'NN'), ('from', 'IN'), ('our', 'PRP$'), ('Creator', 'NNP'), ('--', ':'), ('and', 'CC'), ('that', 'IN'), ('gift', 'NN'), ('should', 'MD'), ('never', 'RB'), ('be', 'VB'), ('discarded', 'VBN'), (',', ','), ('devalued', 'VBD'), ('or', 'CC'), ('put', 'VB'), ('up', 'RP'), ('for', 'IN'), ('sale', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('A', 'DT'), ('hopeful', 'JJ'), ('society', 'NN'), ('expects', 'VBZ'), ('elected', 'VBN'), ('officials', 'NNS'), ('to', 'TO'), ('uphold', 'VB'), ('the', 'DT'), ('public', 'JJ'), ('trust', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Honorable', 'JJ'), ('people', 'NNS'), ('in', 'IN'), ('both', 'DT'), ('parties', 'NNS'), ('are', 'VBP'), ('working', 'VBG'), ('on', 'IN'), ('reforms', 'NNS'), ('to', 'TO'), ('strengthen', 'VB'), ('the', 'DT'), ('ethical', 'JJ'), ('standards', 'NNS'), ('of', 'IN'), ('Washington', 'NNP'), ('--', ':'), ('I', 'PRP'), ('support', 'VBP'), ('your', 'PRP$'), ('efforts', 'NNS'), ('.', '.')]\n",
      "[('Each', 'DT'), ('of', 'IN'), ('us', 'PRP'), ('has', 'VBZ'), ('made', 'VBN'), ('a', 'DT'), ('pledge', 'NN'), ('to', 'TO'), ('be', 'VB'), ('worthy', 'JJ'), ('of', 'IN'), ('public', 'JJ'), ('responsibility', 'NN'), ('--', ':'), ('and', 'CC'), ('that', 'DT'), ('is', 'VBZ'), ('a', 'DT'), ('pledge', 'NN'), ('we', 'PRP'), ('must', 'MD'), ('never', 'RB'), ('forget', 'VB'), (',', ','), ('never', 'RB'), ('dismiss', 'NN'), (',', ','), ('and', 'CC'), ('never', 'RB'), ('betray', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('As', 'IN'), ('we', 'PRP'), ('renew', 'VBP'), ('the', 'DT'), ('promise', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('institutions', 'NNS'), (',', ','), ('let', 'VB'), ('us', 'PRP'), ('also', 'RB'), ('show', 'VBP'), ('the', 'DT'), ('character', 'NN'), ('of', 'IN'), ('America', 'NNP'), ('in', 'IN'), ('our', 'PRP$'), ('compassion', 'NN'), ('and', 'CC'), ('care', 'NN'), ('for', 'IN'), ('one', 'CD'), ('another', 'DT'), ('.', '.')]\n",
      "[('A', 'DT'), ('hopeful', 'JJ'), ('society', 'NN'), ('gives', 'VBZ'), ('special', 'JJ'), ('attention', 'NN'), ('to', 'TO'), ('children', 'NNS'), ('who', 'WP'), ('lack', 'VBP'), ('direction', 'NN'), ('and', 'CC'), ('love', 'NN'), ('.', '.')]\n",
      "[('Through', 'IN'), ('the', 'DT'), ('Helping', 'NNP'), ('America', 'NNP'), (\"'s\", 'POS'), ('Youth', 'NNP'), ('Initiative', 'NNP'), (',', ','), ('we', 'PRP'), ('are', 'VBP'), ('encouraging', 'VBG'), ('caring', 'VBG'), ('adults', 'NNS'), ('to', 'TO'), ('get', 'VB'), ('involved', 'VBN'), ('in', 'IN'), ('the', 'DT'), ('life', 'NN'), ('of', 'IN'), ('a', 'DT'), ('child', 'NN'), ('--', ':'), ('and', 'CC'), ('this', 'DT'), ('good', 'JJ'), ('work', 'NN'), ('is', 'VBZ'), ('being', 'VBG'), ('led', 'VBN'), ('by', 'IN'), ('our', 'PRP$'), ('First', 'NNP'), ('Lady', 'NNP'), (',', ','), ('Laura', 'NNP'), ('Bush', 'NNP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('This', 'DT'), ('year', 'NN'), ('we', 'PRP'), ('will', 'MD'), ('add', 'VB'), ('resources', 'NNS'), ('to', 'TO'), ('encourage', 'VB'), ('young', 'JJ'), ('people', 'NNS'), ('to', 'TO'), ('stay', 'VB'), ('in', 'IN'), ('school', 'NN'), (',', ','), ('so', 'RB'), ('more', 'JJR'), ('of', 'IN'), ('America', 'NNP'), (\"'s\", 'POS'), ('youth', 'NN'), ('can', 'MD'), ('raise', 'VB'), ('their', 'PRP$'), ('sights', 'NNS'), ('and', 'CC'), ('achieve', 'VBP'), ('their', 'PRP$'), ('dreams', 'NNS'), ('.', '.')]\n",
      "[('A', 'DT'), ('hopeful', 'JJ'), ('society', 'NN'), ('comes', 'VBZ'), ('to', 'TO'), ('the', 'DT'), ('aid', 'NN'), ('of', 'IN'), ('fellow', 'JJ'), ('citizens', 'NNS'), ('in', 'IN'), ('times', 'NNS'), ('of', 'IN'), ('suffering', 'NN'), ('and', 'CC'), ('emergency', 'NN'), ('--', ':'), ('and', 'CC'), ('stays', 'NNS'), ('at', 'IN'), ('it', 'PRP'), ('until', 'IN'), ('they', 'PRP'), (\"'re\", 'VBP'), ('back', 'RB'), ('on', 'IN'), ('their', 'PRP$'), ('feet', 'NNS'), ('.', '.')]\n",
      "[('So', 'RB'), ('far', 'RB'), ('the', 'DT'), ('federal', 'JJ'), ('government', 'NN'), ('has', 'VBZ'), ('committed', 'VBN'), ('$', '$'), ('85', 'CD'), ('billion', 'CD'), ('to', 'TO'), ('the', 'DT'), ('people', 'NNS'), ('of', 'IN'), ('the', 'DT'), ('Gulf', 'NNP'), ('Coast', 'NNP'), ('and', 'CC'), ('New', 'NNP'), ('Orleans', 'NNP'), ('.', '.')]\n",
      "[('We', 'PRP'), (\"'re\", 'VBP'), ('removing', 'VBG'), ('debris', 'NN'), ('and', 'CC'), ('repairing', 'NN'), ('highways', 'NNS'), ('and', 'CC'), ('rebuilding', 'VBG'), ('stronger', 'JJR'), ('levees', 'NNS'), ('.', '.')]\n",
      "[('We', 'PRP'), (\"'re\", 'VBP'), ('providing', 'VBG'), ('business', 'NN'), ('loans', 'NNS'), ('and', 'CC'), ('housing', 'NN'), ('assistance', 'NN'), ('.', '.')]\n",
      "[('Yet', 'RB'), ('as', 'IN'), ('we', 'PRP'), ('meet', 'VBP'), ('these', 'DT'), ('immediate', 'JJ'), ('needs', 'NNS'), (',', ','), ('we', 'PRP'), ('must', 'MD'), ('also', 'RB'), ('address', 'VB'), ('deeper', 'JJR'), ('challenges', 'NNS'), ('that', 'WDT'), ('existed', 'VBD'), ('before', 'IN'), ('the', 'DT'), ('storm', 'NN'), ('arrived', 'VBD'), ('.', '.')]\n",
      "[('In', 'IN'), ('New', 'NNP'), ('Orleans', 'NNP'), ('and', 'CC'), ('in', 'IN'), ('other', 'JJ'), ('places', 'NNS'), (',', ','), ('many', 'JJ'), ('of', 'IN'), ('our', 'PRP$'), ('fellow', 'JJ'), ('citizens', 'NNS'), ('have', 'VBP'), ('felt', 'VBN'), ('excluded', 'VBN'), ('from', 'IN'), ('the', 'DT'), ('promise', 'NN'), ('of', 'IN'), ('our', 'PRP$'), ('country', 'NN'), ('.', '.')]\n",
      "[('The', 'DT'), ('answer', 'NN'), ('is', 'VBZ'), ('not', 'RB'), ('only', 'RB'), ('temporary', 'JJ'), ('relief', 'NN'), (',', ','), ('but', 'CC'), ('schools', 'NNS'), ('that', 'WDT'), ('teach', 'VBP'), ('every', 'DT'), ('child', 'NN'), (',', ','), ('and', 'CC'), ('job', 'NN'), ('skills', 'NNS'), ('that', 'IN'), ('bring', 'VBG'), ('upward', 'JJ'), ('mobility', 'NN'), (',', ','), ('and', 'CC'), ('more', 'JJR'), ('opportunities', 'NNS'), ('to', 'TO'), ('own', 'VB'), ('a', 'DT'), ('home', 'NN'), ('and', 'CC'), ('start', 'VB'), ('a', 'DT'), ('business', 'NN'), ('.', '.')]\n",
      "[('As', 'IN'), ('we', 'PRP'), ('recover', 'VBP'), ('from', 'IN'), ('a', 'DT'), ('disaster', 'NN'), (',', ','), ('let', 'VB'), ('us', 'PRP'), ('also', 'RB'), ('work', 'NN'), ('for', 'IN'), ('the', 'DT'), ('day', 'NN'), ('when', 'WRB'), ('all', 'DT'), ('Americans', 'NNPS'), ('are', 'VBP'), ('protected', 'VBN'), ('by', 'IN'), ('justice', 'NN'), (',', ','), ('equal', 'JJ'), ('in', 'IN'), ('hope', 'NN'), (',', ','), ('and', 'CC'), ('rich', 'JJ'), ('in', 'IN'), ('opportunity', 'NN'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('A', 'DT'), ('hopeful', 'JJ'), ('society', 'NN'), ('acts', 'NNS'), ('boldly', 'RB'), ('to', 'TO'), ('fight', 'VB'), ('diseases', 'NNS'), ('like', 'IN'), ('HIV/AIDS', 'NNP'), (',', ','), ('which', 'WDT'), ('can', 'MD'), ('be', 'VB'), ('prevented', 'VBN'), (',', ','), ('and', 'CC'), ('treated', 'VBD'), (',', ','), ('and', 'CC'), ('defeated', 'VBD'), ('.', '.')]\n",
      "[('More', 'JJR'), ('than', 'IN'), ('a', 'DT'), ('million', 'CD'), ('Americans', 'NNPS'), ('live', 'VBP'), ('with', 'IN'), ('HIV', 'NNP'), (',', ','), ('and', 'CC'), ('half', 'NN'), ('of', 'IN'), ('all', 'DT'), ('AIDS', 'NNP'), ('cases', 'NNS'), ('occur', 'VBP'), ('among', 'IN'), ('African', 'JJ'), ('Americans', 'NNPS'), ('.', '.')]\n",
      "[('I', 'PRP'), ('ask', 'VBP'), ('Congress', 'NNP'), ('to', 'TO'), ('reform', 'VB'), ('and', 'CC'), ('reauthorize', 'VB'), ('the', 'DT'), ('Ryan', 'NNP'), ('White', 'NNP'), ('Act', 'NNP'), (',', ','), ('and', 'CC'), ('provide', 'VB'), ('new', 'JJ'), ('funding', 'NN'), ('to', 'TO'), ('states', 'NNS'), (',', ','), ('so', 'IN'), ('we', 'PRP'), ('end', 'VBP'), ('the', 'DT'), ('waiting', 'NN'), ('lists', 'NNS'), ('for', 'IN'), ('AIDS', 'NNP'), ('medicines', 'NNS'), ('in', 'IN'), ('America', 'NNP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('We', 'PRP'), ('will', 'MD'), ('also', 'RB'), ('lead', 'VB'), ('a', 'DT'), ('nationwide', 'JJ'), ('effort', 'NN'), (',', ','), ('working', 'VBG'), ('closely', 'RB'), ('with', 'IN'), ('African', 'JJ'), ('American', 'JJ'), ('churches', 'NNS'), ('and', 'CC'), ('faith-based', 'JJ'), ('groups', 'NNS'), (',', ','), ('to', 'TO'), ('deliver', 'VB'), ('rapid', 'JJ'), ('HIV', 'NNP'), ('tests', 'NNS'), ('to', 'TO'), ('millions', 'NNS'), (',', ','), ('end', 'VBP'), ('the', 'DT'), ('stigma', 'NN'), ('of', 'IN'), ('AIDS', 'NNP'), (',', ','), ('and', 'CC'), ('come', 'VB'), ('closer', 'JJR'), ('to', 'TO'), ('the', 'DT'), ('day', 'NN'), ('when', 'WRB'), ('there', 'EX'), ('are', 'VBP'), ('no', 'DT'), ('new', 'JJ'), ('infections', 'NNS'), ('in', 'IN'), ('America', 'NNP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n",
      "[('Fellow', 'NNP'), ('citizens', 'NNS'), (',', ','), ('we', 'PRP'), (\"'ve\", 'VBP'), ('been', 'VBN'), ('called', 'VBN'), ('to', 'TO'), ('leadership', 'NN'), ('in', 'IN'), ('a', 'DT'), ('period', 'NN'), ('of', 'IN'), ('consequence', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), (\"'ve\", 'VBP'), ('entered', 'VBN'), ('a', 'DT'), ('great', 'JJ'), ('ideological', 'JJ'), ('conflict', 'NN'), ('we', 'PRP'), ('did', 'VBD'), ('nothing', 'NN'), ('to', 'TO'), ('invite', 'VB'), ('.', '.')]\n",
      "[('We', 'PRP'), ('see', 'VBP'), ('great', 'JJ'), ('changes', 'NNS'), ('in', 'IN'), ('science', 'NN'), ('and', 'CC'), ('commerce', 'NN'), ('that', 'WDT'), ('will', 'MD'), ('influence', 'VB'), ('all', 'DT'), ('our', 'PRP$'), ('lives', 'NNS'), ('.', '.')]\n",
      "[('Sometimes', 'RB'), ('it', 'PRP'), ('can', 'MD'), ('seem', 'VB'), ('that', 'DT'), ('history', 'NN'), ('is', 'VBZ'), ('turning', 'VBG'), ('in', 'IN'), ('a', 'DT'), ('wide', 'JJ'), ('arc', 'NN'), (',', ','), ('toward', 'IN'), ('an', 'DT'), ('unknown', 'JJ'), ('shore', 'NN'), ('.', '.')]\n",
      "[('Yet', 'CC'), ('the', 'DT'), ('destination', 'NN'), ('of', 'IN'), ('history', 'NN'), ('is', 'VBZ'), ('determined', 'VBN'), ('by', 'IN'), ('human', 'JJ'), ('action', 'NN'), (',', ','), ('and', 'CC'), ('every', 'DT'), ('great', 'JJ'), ('movement', 'NN'), ('of', 'IN'), ('history', 'NN'), ('comes', 'VBZ'), ('to', 'TO'), ('a', 'DT'), ('point', 'NN'), ('of', 'IN'), ('choosing', 'NN'), ('.', '.')]\n",
      "[('Lincoln', 'NNP'), ('could', 'MD'), ('have', 'VB'), ('accepted', 'VBN'), ('peace', 'NN'), ('at', 'IN'), ('the', 'DT'), ('cost', 'NN'), ('of', 'IN'), ('disunity', 'NN'), ('and', 'CC'), ('continued', 'JJ'), ('slavery', 'NN'), ('.', '.')]\n",
      "[('Martin', 'NNP'), ('Luther', 'NNP'), ('King', 'NNP'), ('could', 'MD'), ('have', 'VB'), ('stopped', 'VBN'), ('at', 'IN'), ('Birmingham', 'NNP'), ('or', 'CC'), ('at', 'IN'), ('Selma', 'NNP'), (',', ','), ('and', 'CC'), ('achieved', 'VBD'), ('only', 'RB'), ('half', 'PDT'), ('a', 'DT'), ('victory', 'NN'), ('over', 'IN'), ('segregation', 'NN'), ('.', '.')]\n",
      "[('The', 'DT'), ('United', 'NNP'), ('States', 'NNPS'), ('could', 'MD'), ('have', 'VB'), ('accepted', 'VBN'), ('the', 'DT'), ('permanent', 'JJ'), ('division', 'NN'), ('of', 'IN'), ('Europe', 'NNP'), (',', ','), ('and', 'CC'), ('been', 'VBN'), ('complicit', 'NNS'), ('in', 'IN'), ('the', 'DT'), ('oppression', 'NN'), ('of', 'IN'), ('others', 'NNS'), ('.', '.')]\n",
      "[('Today', 'NN'), (',', ','), ('having', 'VBG'), ('come', 'VBN'), ('far', 'RB'), ('in', 'IN'), ('our', 'PRP$'), ('own', 'JJ'), ('historical', 'JJ'), ('journey', 'NN'), (',', ','), ('we', 'PRP'), ('must', 'MD'), ('decide', 'VB'), (':', ':'), ('Will', 'MD'), ('we', 'PRP'), ('turn', 'VB'), ('back', 'RP'), (',', ','), ('or', 'CC'), ('finish', 'VB'), ('well', 'RB'), ('?', '.')]\n",
      "[('Before', 'IN'), ('history', 'NN'), ('is', 'VBZ'), ('written', 'VBN'), ('down', 'RP'), ('in', 'IN'), ('books', 'NNS'), (',', ','), ('it', 'PRP'), ('is', 'VBZ'), ('written', 'VBN'), ('in', 'IN'), ('courage', 'NN'), ('.', '.')]\n",
      "[('Like', 'IN'), ('Americans', 'NNPS'), ('before', 'IN'), ('us', 'PRP'), (',', ','), ('we', 'PRP'), ('will', 'MD'), ('show', 'VB'), ('that', 'DT'), ('courage', 'NN'), ('and', 'CC'), ('we', 'PRP'), ('will', 'MD'), ('finish', 'VB'), ('well', 'RB'), ('.', '.')]\n",
      "[('We', 'PRP'), ('will', 'MD'), ('lead', 'VB'), ('freedom', 'NN'), (\"'s\", 'POS'), ('advance', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), ('will', 'MD'), ('compete', 'VB'), ('and', 'CC'), ('excel', 'VB'), ('in', 'IN'), ('the', 'DT'), ('global', 'JJ'), ('economy', 'NN'), ('.', '.')]\n",
      "[('We', 'PRP'), ('will', 'MD'), ('renew', 'VB'), ('the', 'DT'), ('defining', 'VBG'), ('moral', 'JJ'), ('commitments', 'NNS'), ('of', 'IN'), ('this', 'DT'), ('land', 'NN'), ('.', '.')]\n",
      "[('And', 'CC'), ('so', 'RB'), ('we', 'PRP'), ('move', 'VBP'), ('forward', 'RB'), ('--', ':'), ('optimistic', 'JJ'), ('about', 'IN'), ('our', 'PRP$'), ('country', 'NN'), (',', ','), ('faithful', 'JJ'), ('to', 'TO'), ('its', 'PRP$'), ('cause', 'NN'), (',', ','), ('and', 'CC'), ('confident', 'NN'), ('of', 'IN'), ('the', 'DT'), ('victories', 'NNS'), ('to', 'TO'), ('come', 'VB'), ('.', '.')]\n",
      "[('May', 'NNP'), ('God', 'NNP'), ('bless', 'NN'), ('America', 'NNP'), ('.', '.')]\n",
      "[('(', '('), ('Applause', 'NNP'), ('.', '.'), (')', ')')]\n"
     ]
    }
   ],
   "source": [
    "#Another example\n",
    "from nltk.corpus import state_union\n",
    "from nltk.tokenize import PunktSentenceTokenizer\n",
    "\n",
    "train_text = state_union.raw(\"2005-GWBush.txt\")\n",
    "sample_text = state_union.raw(\"2006-GWBush.txt\")\n",
    "#print(train_text)\n",
    "\n",
    "custom_sent_tokenizer=PunktSentenceTokenizer(train_text)\n",
    "\n",
    "tokenized=custom_sent_tokenizer.tokenize(sample_text)\n",
    "\n",
    "def process_content():\n",
    "    try:\n",
    "        for i in tokenized:\n",
    "            words=nltk.word_tokenize(i)\n",
    "            #print(words)\n",
    "            #print(\"Wait\")\n",
    "            tagged=nltk.pos_tag(words)\n",
    "            print(tagged)\n",
    "    except Exception as e:\n",
    "        print(str(e))\n",
    "process_content()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_content():\n",
    "    try:\n",
    "        for i in tokenized:\n",
    "            words=nltk.word_tokenize(i)\n",
    "            #print(words)\n",
    "            #print(\"Wait\")\n",
    "            tagged=nltk.pos_tag(words)\n",
    "            #prepare chunk parser format.\n",
    "            chunkGram = r\"\"\"Chunk: {<RB.?>*<VB.?>*<NNP>+<NN>?}\"\"\"\n",
    "            chunkParser = nltk.RegexpParser(chunkGram)\n",
    "            \n",
    "            chunked = chunkParser.parse(tagged)\n",
    "            #chunked.draw()           ##to see chunks uncommit this line     \n",
    "            \n",
    "    except Exception as e:\n",
    "        print(str(e))\n",
    "process_content()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chinking (removal something from chunk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_content():\n",
    "    try:\n",
    "        for i in tokenized[0:5]:\n",
    "            words=nltk.word_tokenize(i)\n",
    "            #print(words)\n",
    "            #print(\"Wait\")\n",
    "            tagged=nltk.pos_tag(words)\n",
    "            #prepare chunk parser format.\n",
    "            chunkGram = r\"\"\"Chunk: {<.*>+}\n",
    "                                    }<VB.?|IN|DT|TO>+{\"\"\"\n",
    "\n",
    "            chunkParser = nltk.RegexpParser(chunkGram)\n",
    "            \n",
    "            chunked = chunkParser.parse(tagged)\n",
    "            chunked.draw()           ##to see chunks uncommit this line     \n",
    "            \n",
    "    except Exception as e:\n",
    "        print(str(e))\n",
    "process_content()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Named entity identifing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_content():\n",
    "    try:\n",
    "        for i in tokenized[0:5]:\n",
    "            words=nltk.word_tokenize(i)\n",
    "            #print(words)\n",
    "            #print(\"Wait\")\n",
    "            tagged=nltk.pos_tag(words)\n",
    "            #prepare chunk parser format.\n",
    "            namedEnt = nltk.ne_chunk(tagged, binary=True)\n",
    "            namedEnt.draw()    ##to see chunks uncommit this line     \n",
    "            \n",
    "    except Exception as e:\n",
    "        print(str(e))\n",
    "process_content()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lemmatizing with NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat\n",
      "laptop\n",
      "goose\n",
      "rock\n",
      "python\n",
      "good\n",
      "best\n",
      "run\n",
      "run\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "lemmatizer=WordNetLemmatizer()\n",
    "\n",
    "print(lemmatizer.lemmatize('cats'))\n",
    "print(lemmatizer.lemmatize('laptopss'))\n",
    "print(lemmatizer.lemmatize(\"geese\"))\n",
    "print(lemmatizer.lemmatize(\"rocks\"))\n",
    "print(lemmatizer.lemmatize(\"python\"))\n",
    "print(lemmatizer.lemmatize(\"better\", pos=\"a\"))\n",
    "print(lemmatizer.lemmatize(\"best\", pos=\"a\"))\n",
    "print(lemmatizer.lemmatize(\"run\"))\n",
    "print(lemmatizer.lemmatize(\"run\",'v'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thamm\\Anaconda3\\lib\\site-packages\\nltk\\__init__.py\n"
     ]
    }
   ],
   "source": [
    "print(nltk.__file__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Accessing files from corpora directory(built in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[The King James Bible]\n",
      "\n",
      "The Old Testament of the King James Bible\n",
      "\n",
      "The First Book of Moses:  Called Genesis\n",
      "\n",
      "\n",
      "1:1 In the beginning God created the heaven and the earth.\n",
      "1:2 And the earth was without form, and void; and darkness was upon\n",
      "the face of the deep.\n",
      "And the Spirit of God moved upon the face of the\n",
      "waters.\n",
      "1:3 And God said, Let there be light: and there was light.\n",
      "1:4 And God saw the light, that it was good: and God divided the light\n",
      "from the darkness.\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk.corpus import gutenberg\n",
    "\n",
    "# sample text\n",
    "sample = gutenberg.raw(\"bible-kjv.txt\")\n",
    "\n",
    "tok = sent_tokenize(sample)\n",
    "\n",
    "for x in range(5):\n",
    "    print(tok[x])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "######  You can use WordNet alongside the NLTK module to find the meanings of words, synonyms, antonyms, and more. Let's cover some examples."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All set values :  [Synset('plan.n.01'), Synset('program.n.02'), Synset('broadcast.n.02'), Synset('platform.n.02'), Synset('program.n.05'), Synset('course_of_study.n.01'), Synset('program.n.07'), Synset('program.n.08'), Synset('program.v.01'), Synset('program.v.02')]\n",
      "\n",
      "First one :  plan.n.01\n",
      "Word (synonum) : plan\n",
      "Defination : a series of steps to be carried out or goals to be accomplished\n",
      "Examples : ['they drew up a six-step plan', 'they discussed plans for a new bond issue']\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import wordnet\n",
    "\n",
    "syns=wordnet.synsets('Program')\n",
    "\n",
    "print(\"All set values : \",syns)  #all set values\n",
    "\n",
    "print(\"\\nFirst one : \",syns[0].name())\n",
    "\n",
    "print(\"Word (synonum) :\", syns[0].lemmas()[0].name())\n",
    "\n",
    "print(\"Defination :\", syns[0].definition())\n",
    "\n",
    "print(\"Examples :\",syns[0].examples())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'respectable', 'salutary', 'thoroughly', 'unspoilt', 'upright', 'dependable', 'trade_good', 'right', 'goodness', 'skilful', 'expert', 'undecomposed', 'soundly', 'sound', 'practiced', 'beneficial', 'in_force', 'unspoiled', 'full', 'commodity', 'honorable', 'ripe', 'proficient', 'near', 'honest', 'secure', 'serious', 'adept', 'estimable', 'skillful', 'good', 'effective', 'just', 'dear', 'well', 'safe', 'in_effect'}\n",
      "{'bad', 'evilness', 'evil', 'ill', 'badness'}\n"
     ]
    }
   ],
   "source": [
    "synonyms = []\n",
    "antonyms = []\n",
    "\n",
    "for syn in wordnet.synsets(\"good\"):\n",
    "    for l in syn.lemmas():\n",
    "        synonyms.append(l.name())\n",
    "        if l.antonyms():\n",
    "            antonyms.append(l.antonyms()[0].name())\n",
    "\n",
    "print(set(synonyms))\n",
    "print(set(antonyms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9090909090909091\n"
     ]
    }
   ],
   "source": [
    "w1 = wordnet.synset('ship.n.01')  #comparing words meaning. determine similarity between words/synonyms\n",
    "w2 = wordnet.synset('boat.n.01')\n",
    "print(w1.wup_similarity(w2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6956521739130435\n"
     ]
    }
   ],
   "source": [
    "w1 = wordnet.synset('ship.n.01')\n",
    "w2 = wordnet.synset('car.n.01')\n",
    "print(w1.wup_similarity(w2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.32\n"
     ]
    }
   ],
   "source": [
    "w1 = wordnet.synset('ship.n.01')\n",
    "w2 = wordnet.synset('cat.n.01')\n",
    "print(w1.wup_similarity(w2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text Classification with NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import random\n",
    "from nltk.corpus import movie_reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(['i', 'was', 'fortunate', 'enough', 'to', 'attend', 'an', 'advance', 'screening', 'for', 'the', 'upcoming', 'thriller', 'conspiracy', 'theory', '.', 'this', 'was', ',', 'of', 'course', ',', 'a', 'big', 'deal', 'for', 'me', 'because', 'reviewing', 'movies', 'is', 'basically', 'just', 'a', 'hobby', 'for', 'me', 'and', 'i', 'never', 'get', 'a', 'chance', 'at', 'something', 'like', 'this', '.', 'not', 'only', 'did', 'i', 'get', 'to', 'see', 'an', 'advance', 'screening', ',', 'i', 'was', 'able', 'to', 'see', 'an', 'advance', 'screening', 'of', 'a', '*', 'very', 'good', '*', 'movie', '.', 'the', 'very', 'fast', '-', 'paced', 'film', 'stars', 'mel', 'gibson', 'as', 'jerry', 'fletcher', ',', 'a', 'fast', '-', 'talking', ',', 'witty', ',', 'comical', 'taxi', 'driver', 'in', 'new', 'york', 'city', '.', 'gibson', \"'\", 's', 'performance', 'is', 'terrific', ',', 'and', 'his', 'character', 'is', 'similar', 'to', 'that', 'of', 'martin', 'riggs', 'in', 'the', 'lethal', 'weapon', 'films', '.', 'gibson', 'again', 'teams', 'up', 'with', 'richard', 'donner', ',', 'as', 'he', 'did', 'in', 'the', 'lethal', 'weapon', 'films', 'and', 'maverick', ',', 'and', 'this', 'time', 'around', ',', 'the', 'combination', 'works', 'even', 'better', '.', 'the', 'character', 'of', 'jerry', 'fletcher', 'is', 'indeed', 'unique', '.', 'try', 'to', 'imagine', 'a', 'toned', '-', 'downed', 'version', 'of', 'travis', 'bickle', 'who', ',', 'this', 'time', 'around', ',', 'is', 'a', 'jittery', 'guy', 'with', 'knowledge', 'of', 'government', 'conspiracy', 'cover', '-', 'ups', '.', 'if', 'you', 'can', 'imagine', 'that', ',', 'then', 'you', \"'\", 've', 'basically', 'got', 'jerry', 'fletcher', '.', 'in', 'many', 'ways', ',', 'i', 'was', 'surprised', 'by', 'this', 'movie', '.', 'to', 'begin', 'with', ',', 'i', 'was', 'surprised', 'at', 'how', 'good', 'it', 'was', '.', 'don', \"'\", 't', 'get', 'me', 'wrong', ',', 'it', \"'\", 's', 'not', 'going', 'to', 'be', 'accepting', 'any', 'gold', 'trophies', 'next', 'spring', ',', 'but', 'it', 'was', 'a', 'very', 'enjoyable', 'movie', '.', 'secondly', ',', 'i', 'was', 'surprised', 'at', 'mel', 'gibson', \"'\", 's', 'performance', '.', 'he', 'provided', 'a', 'fantastic', 'performance', '.', 'the', 'previews', 'of', 'this', 'film', 'led', 'me', 'to', 'believe', 'it', 'was', 'an', 'all', '-', 'out', 'action', 'flick', ',', 'which', 'after', 'viewing', ',', 'it', 'was', 'almost', 'the', 'opposite', 'in', 'a', 'certain', 'sense', '.', 'gibson', \"'\", 's', 'character', 'has', 'an', 'almost', 'uncountable', 'number', 'of', 'one', '-', 'liners', ',', 'hilarious', 'situations', ',', 'and', 'his', 'character', 'is', 'one', 'that', 'the', 'audience', 'tends', 'to', 'side', 'with', 'throughout', 'the', 'film', '.', 'for', 'instance', ',', 'jerry', \"'\", 's', 'apartment', '(', 'and', 'especially', 'his', 'security', ')', 'is', 'memorable', '.', 'and', 'last', ',', 'i', 'was', 'surprised', 'at', 'how', 'good', 'the', 'plot', 'is', '.', 'writer', 'brian', 'helgeland', 'has', 'created', 'a', 'terrific', 'story', ',', 'and', 'when', 'watching', 'this', 'film', ',', 'you', 'are', 'left', 'to', 'wonder', 'if', 'all', 'of', 'jerry', 'fletcher', \"'\", 's', 'far', '-', 'fetched', '(', 'or', 'so', 'they', 'seem', 'at', 'first', ')', 'conspiracy', 'theories', 'are', 'helgeland', \"'\", 's', 'own', 'opinions', '.', 'mel', 'gibson', 'isn', \"'\", 't', 'the', 'only', 'stand', '-', 'out', 'in', 'the', 'film', '.', 'julia', 'roberts', 'is', 'very', 'good', 'in', 'her', 'performance', 'as', 'alice', 'sutton', ',', 'the', 'department', 'of', 'justice', 'employee', 'who', 'can', \"'\", 't', 'seem', 'to', 'stay', 'away', 'from', 'jerry', 'fletcher', ',', 'who', 'continually', 'visits', 'alice', 'in', 'hopes', 'that', 'she', 'will', 'do', 'something', 'about', 'his', 'theories', '.', 'he', 'also', 'seems', 'obsessed', 'and', 'in', 'love', 'with', 'her', '.', 'but', 'whenever', 'he', 'approaches', 'her', 'with', 'another', 'one', 'of', 'his', 'theories', ',', 'she', 'shrugs', 'him', 'and', 'his', 'beliefs', 'off', ',', 'continually', 'noting', 'that', 'one', 'day', 'she', 'is', 'going', 'to', 'slap', 'a', 'restraining', 'order', 'on', 'him', '.', 'jerry', ',', 'obviously', 'determined', 'to', 'continually', 'seek', 'out', 'the', 'truth', ',', 'continues', 'to', 'research', 'information', 'for', 'his', 'next', 'conspiracy', 'theory', ',', 'which', 'will', 'be', 'printed', 'in', 'his', 'newsletter', '(', 'same', 'title', 'as', 'the', 'movie', ')', '.', 'although', 'he', 'only', 'has', 'five', 'subscribers', ',', 'he', 'puts', 'a', 'large', 'amount', 'of', 'time', 'and', 'effort', 'into', 'his', 'work', ',', 'and', 'publishes', 'his', 'next', 'edition', '.', 'very', 'soon', 'after', 'he', 'does', 'so', ',', 'he', 'is', 'abducted', 'and', 'tortured', 'by', 'a', 'sinister', 'man', 'who', 'refers', 'to', 'himself', 'as', 'dr', '.', 'jonas', '(', 'patrick', 'stewart', ')', '.', 'after', 'barely', 'escaping', 'alive', ',', 'jerry', 'is', 'forced', 'to', 'turn', 'to', 'the', 'only', 'person', 'he', 'can', 'trust', ':', 'alice', 'sutton', '.', 'the', 'remainder', 'of', 'the', 'film', 'is', 'almost', 'always', 'fast', '-', 'paced', 'and', 'full', 'of', 'action', 'and', 'suspense', ',', 'with', 'jerry', \"'\", 's', 'life', 'being', 'constantly', 'put', 'into', 'jeopardy', '.', 'and', 'more', 'along', 'the', 'way', ',', 'the', 'audience', '(', 'and', 'alice', ')', 'learn', 'more', 'and', 'more', 'about', 'jerry', \"'\", 's', 'life', '.', 'many', 'things', 'are', 'explained', 'throughout', 'this', 'film', ',', 'both', 'to', 'the', 'characters', 'and', 'to', 'the', 'audience', '.', 'for', 'instance', ',', 'the', 'meaning', 'of', 'the', 'book', '\"', 'the', 'catcher', 'in', 'the', 'rye', '\"', 'and', 'its', 'ties', 'to', 'assassins', ',', 'the', 'reason', 'lone', 'gunmen', 'have', 'three', 'names', '(', 'e', '.', 'g', '.', 'lee', 'harvey', 'oswald', 'and', 'james', 'earl', 'ray', ')', ',', 'and', 'the', 'real', 'truth', 'behind', 'the', 'grateful', 'dead', '.', 'make', 'sure', 'listen', 'closely', 'throughout', 'the', 'film', ',', 'as', 'jerry', 'is', 'constantly', 'throwing', 'out', 'interesting', 'tidbits', 'such', 'as', 'the', 'above', '.', 'definitely', ',', 'when', 'conspiracy', 'theory', 'hits', 'theaters', 'august', '8th', ',', 'make', 'sure', 'you', 'are', 'standing', 'in', 'line', 'to', 'see', 'it', '.', 'i', 'am', 'nearly', 'positive', 'everyone', 'should', 'enjoy', 'this', 'film', ',', 'especially', 'if', 'you', 'are', 'into', 'an', 'action', '-', 'thriller', 'with', 'witty', 'dialogue', 'and', 'numerous', 'suspenseful', 'situations', '.', 'and', 'even', 'if', 'you', 'aren', \"'\", 't', ',', 'you', 'should', 'still', 'like', 'this', 'film', '.'], 'pos')\n"
     ]
    }
   ],
   "source": [
    "documents = [(list(movie_reviews.words(fileid)), category)\n",
    "             for category in movie_reviews.categories()\n",
    "             for fileid in movie_reviews.fileids(category)]\n",
    "\n",
    "#documents[0]\n",
    "#print(movie_reviews.categories())\n",
    "#print(movie_reviews.fileids(['pos','neg']))\n",
    "random.shuffle(documents)  #The shuffle() method randomizes the items of a list in place.\n",
    "\n",
    "print(documents[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(',', 77717), ('the', 76529), ('.', 65876), ('a', 38106), ('and', 35576), ('of', 34123), ('to', 31937), (\"'\", 30585), ('is', 25195), ('in', 21822), ('s', 18513), ('\"', 17612), ('it', 16107), ('that', 15924), ('-', 15595)]\n",
      "253\n"
     ]
    }
   ],
   "source": [
    "all_words = []\n",
    "for w in movie_reviews.words():\n",
    "    all_words.append(w.lower())\n",
    "all_words = nltk.FreqDist(all_words)\n",
    "print(all_words.most_common(15))\n",
    "print(all_words[\"stupid\"])  #to count how many times a word present."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_features = list(all_words.keys())[:3000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['deal',\n",
       " '?',\n",
       " 'watch',\n",
       " 'movie',\n",
       " '\"',\n",
       " 'sorta',\n",
       " 'find',\n",
       " 'out',\n",
       " 'critique',\n",
       " 'mind']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_features[40:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_features(document):\n",
    "    words = set(document)\n",
    "    features = {}\n",
    "    for w in word_features:\n",
    "        features[w] = (w in words)\n",
    "\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'plot': True, ':': True, 'two': True, 'teen': True, 'couples': True, 'go': True, 'to': True, 'a': True, 'church': True, 'party': True, ',': True, 'drink': True, 'and': True, 'then': True, 'drive': True, '.': True, 'they': True, 'get': True, 'into': True, 'an': True, 'accident': True, 'one': True, 'of': True, 'the': True, 'guys': True, 'dies': True, 'but': True, 'his': True, 'girlfriend': True, 'continues': True, 'see': True, 'him': True, 'in': True, 'her': True, 'life': True, 'has': True, 'nightmares': True, 'what': True, \"'\": True, 's': True, 'deal': True, '?': True, 'watch': True, 'movie': True, '\"': True, 'sorta': True, 'find': True, 'out': True, 'critique': True, 'mind': True, '-': True, 'fuck': True, 'for': True, 'generation': True, 'that': True, 'touches': True, 'on': True, 'very': True, 'cool': True, 'idea': True, 'presents': True, 'it': True, 'bad': True, 'package': True, 'which': True, 'is': True, 'makes': True, 'this': True, 'review': True, 'even': True, 'harder': True, 'write': True, 'since': True, 'i': True, 'generally': True, 'applaud': True, 'films': True, 'attempt': True, 'break': True, 'mold': True, 'mess': True, 'with': True, 'your': True, 'head': True, 'such': True, '(': True, 'lost': True, 'highway': True, '&': True, 'memento': True, ')': True, 'there': True, 'are': True, 'good': True, 'ways': True, 'making': True, 'all': True, 'types': True, 'these': True, 'folks': True, 'just': True, 'didn': True, 't': True, 'snag': True, 'correctly': True, 'seem': True, 'have': True, 'taken': True, 'pretty': True, 'neat': True, 'concept': True, 'executed': True, 'terribly': True, 'so': True, 'problems': True, 'well': True, 'its': True, 'main': True, 'problem': True, 'simply': True, 'too': True, 'jumbled': True, 'starts': True, 'off': True, 'normal': True, 'downshifts': True, 'fantasy': True, 'world': True, 'you': True, 'as': True, 'audience': True, 'member': True, 'no': True, 'going': True, 'dreams': True, 'characters': True, 'coming': True, 'back': True, 'from': True, 'dead': True, 'others': True, 'who': True, 'look': True, 'like': True, 'strange': True, 'apparitions': True, 'disappearances': True, 'looooot': True, 'chase': True, 'scenes': True, 'tons': True, 'weird': True, 'things': True, 'happen': True, 'most': True, 'not': True, 'explained': True, 'now': True, 'personally': True, 'don': True, 'trying': True, 'unravel': True, 'film': True, 'every': True, 'when': True, 'does': True, 'give': True, 'me': True, 'same': True, 'clue': True, 'over': True, 'again': True, 'kind': True, 'fed': True, 'up': True, 'after': True, 'while': True, 'biggest': True, 'obviously': True, 'got': True, 'big': True, 'secret': True, 'hide': True, 'seems': True, 'want': True, 'completely': True, 'until': True, 'final': True, 'five': True, 'minutes': True, 'do': True, 'make': True, 'entertaining': True, 'thrilling': True, 'or': True, 'engaging': True, 'meantime': True, 'really': True, 'sad': True, 'part': True, 'arrow': True, 'both': True, 'dig': True, 'flicks': True, 'we': True, 'actually': True, 'figured': True, 'by': True, 'half': True, 'way': True, 'point': True, 'strangeness': True, 'did': True, 'start': True, 'little': True, 'bit': True, 'sense': True, 'still': True, 'more': True, 'guess': True, 'bottom': True, 'line': True, 'movies': True, 'should': True, 'always': True, 'sure': True, 'before': True, 'given': True, 'password': True, 'enter': True, 'understanding': True, 'mean': True, 'showing': True, 'melissa': True, 'sagemiller': True, 'running': True, 'away': True, 'visions': True, 'about': True, '20': True, 'throughout': True, 'plain': True, 'lazy': True, '!': True, 'okay': True, 'people': True, 'chasing': True, 'know': True, 'need': True, 'how': True, 'giving': True, 'us': True, 'different': True, 'offering': True, 'further': True, 'insight': True, 'down': True, 'apparently': True, 'studio': True, 'took': True, 'director': True, 'chopped': True, 'themselves': True, 'shows': True, 'might': True, 've': True, 'been': True, 'decent': True, 'here': True, 'somewhere': True, 'suits': True, 'decided': True, 'turning': True, 'music': True, 'video': True, 'edge': True, 'would': True, 'actors': True, 'although': True, 'wes': True, 'bentley': True, 'seemed': True, 'be': True, 'playing': True, 'exact': True, 'character': True, 'he': True, 'american': True, 'beauty': True, 'only': True, 'new': True, 'neighborhood': True, 'my': True, 'kudos': True, 'holds': True, 'own': True, 'entire': True, 'feeling': True, 'unraveling': True, 'overall': True, 'doesn': True, 'stick': True, 'because': True, 'entertain': True, 'confusing': True, 'rarely': True, 'excites': True, 'feels': True, 'redundant': True, 'runtime': True, 'despite': True, 'ending': True, 'explanation': True, 'craziness': True, 'came': True, 'oh': True, 'horror': True, 'slasher': True, 'flick': True, 'packaged': True, 'someone': True, 'assuming': True, 'genre': True, 'hot': True, 'kids': True, 'also': True, 'wrapped': True, 'production': True, 'years': True, 'ago': True, 'sitting': True, 'shelves': True, 'ever': True, 'whatever': True, 'skip': True, 'where': True, 'joblo': True, 'nightmare': True, 'elm': True, 'street': True, '3': True, '7': True, '/': True, '10': True, 'blair': True, 'witch': True, '2': True, 'crow': True, '9': True, 'salvation': True, '4': True, 'stir': True, 'echoes': True, '8': True, 'happy': False, 'bastard': False, 'quick': False, 'damn': False, 'y2k': False, 'bug': False, 'starring': False, 'jamie': False, 'lee': False, 'curtis': False, 'another': False, 'baldwin': False, 'brother': False, 'william': False, 'time': False, 'story': False, 'regarding': False, 'crew': False, 'tugboat': False, 'comes': False, 'across': False, 'deserted': False, 'russian': False, 'tech': False, 'ship': False, 'kick': False, 'power': False, 'within': False, 'gore': False, 'bringing': False, 'few': False, 'action': False, 'sequences': False, 'virus': False, 'empty': False, 'flash': False, 'substance': False, 'why': False, 'was': False, 'middle': False, 'nowhere': False, 'origin': False, 'pink': False, 'flashy': False, 'thing': False, 'hit': False, 'mir': False, 'course': False, 'donald': False, 'sutherland': False, 'stumbling': False, 'around': False, 'drunkenly': False, 'hey': False, 'let': False, 'some': False, 'robots': False, 'acting': False, 'below': False, 'average': False, 'likes': False, 're': False, 'likely': False, 'work': False, 'halloween': False, 'h20': False, 'wasted': False, 'real': False, 'star': False, 'stan': False, 'winston': False, 'robot': False, 'design': False, 'schnazzy': False, 'cgi': False, 'occasional': False, 'shot': False, 'picking': False, 'brain': False, 'if': False, 'body': False, 'parts': False, 'turn': False, 'otherwise': False, 'much': False, 'sunken': False, 'jaded': False, 'viewer': False, 'thankful': False, 'invention': False, 'timex': False, 'indiglo': False, 'based': False, 'late': False, '1960': False, 'television': False, 'show': False, 'name': False, 'mod': False, 'squad': False, 'tells': False, 'tale': False, 'three': False, 'reformed': False, 'criminals': False, 'under': False, 'employ': False, 'police': False, 'undercover': False, 'however': False, 'wrong': False, 'evidence': False, 'gets': False, 'stolen': False, 'immediately': False, 'suspicion': False, 'ads': False, 'cuts': False, 'claire': False, 'dane': False, 'nice': False, 'hair': False, 'cute': False, 'outfits': False, 'car': False, 'chases': False, 'stuff': False, 'blowing': False, 'sounds': False, 'first': False, 'fifteen': False, 'quickly': False, 'becomes': False, 'apparent': False, 'certainly': False, 'slick': False, 'looking': False, 'complete': False, 'costumes': False, 'isn': False, 'enough': False, 'best': False, 'described': False, 'cross': False, 'between': False, 'hour': False, 'long': False, 'cop': False, 'stretched': False, 'span': False, 'single': False, 'clich': False, 'matter': False, 'elements': False, 'recycled': False, 'everything': False, 'already': False, 'seen': False, 'nothing': False, 'spectacular': False, 'sometimes': False, 'bordering': False, 'wooden': False, 'danes': False, 'omar': False, 'epps': False, 'deliver': False, 'their': False, 'lines': False, 'bored': False, 'transfers': False, 'onto': False, 'escape': False, 'relatively': False, 'unscathed': False, 'giovanni': False, 'ribisi': False, 'plays': False, 'resident': False, 'crazy': False, 'man': False, 'ultimately': False, 'being': False, 'worth': False, 'watching': False, 'unfortunately': False, 'save': False, 'convoluted': False, 'apart': False, 'occupying': False, 'screen': False, 'young': False, 'cast': False, 'clothes': False, 'hip': False, 'soundtrack': False, 'appears': False, 'geared': False, 'towards': False, 'teenage': False, 'mindset': False, 'r': False, 'rating': False, 'content': False, 'justify': False, 'juvenile': False, 'older': False, 'information': False, 'literally': False, 'spoon': False, 'hard': False, 'instead': False, 'telling': False, 'dialogue': False, 'poorly': False, 'written': False, 'extremely': False, 'predictable': False, 'progresses': False, 'won': False, 'care': False, 'heroes': False, 'any': False, 'jeopardy': False, 'll': False, 'aren': False, 'basing': False, 'nobody': False, 'remembers': False, 'questionable': False, 'wisdom': False, 'especially': False, 'considers': False, 'target': False, 'fact': False, 'number': False, 'memorable': False, 'can': False, 'counted': False, 'hand': False, 'missing': False, 'finger': False, 'times': False, 'checked': False, 'six': False, 'clear': False, 'indication': False, 'them': False, 'than': False, 'cash': False, 'spending': False, 'dollar': False, 'judging': False, 'rash': False, 'awful': False, 'seeing': False, 'avoid': False, 'at': False, 'costs': False, 'quest': False, 'camelot': False, 'warner': False, 'bros': False, 'feature': False, 'length': False, 'fully': False, 'animated': False, 'steal': False, 'clout': False, 'disney': False, 'cartoon': False, 'empire': False, 'mouse': False, 'reason': False, 'worried': False, 'other': False, 'recent': False, 'challenger': False, 'throne': False, 'last': False, 'fall': False, 'promising': False, 'flawed': False, '20th': False, 'century': False, 'fox': False, 'anastasia': False, 'hercules': False, 'lively': False, 'colorful': False, 'palate': False, 'had': False, 'beat': False, 'hands': False, 'crown': False, '1997': False, 'piece': False, 'animation': False, 'year': False, 'contest': False, 'arrival': False, 'magic': False, 'kingdom': False, 'mediocre': False, '--': False, 'd': False, 'pocahontas': False, 'those': False, 'keeping': False, 'score': False, 'nearly': False, 'dull': False, 'revolves': False, 'adventures': False, 'free': False, 'spirited': False, 'kayley': False, 'voiced': False, 'jessalyn': False, 'gilsig': False, 'early': False, 'daughter': False, 'belated': False, 'knight': False, 'king': False, 'arthur': False, 'round': False, 'table': False, 'dream': False, 'follow': False, 'father': False, 'footsteps': False, 'she': False, 'chance': False, 'evil': False, 'warlord': False, 'ruber': False, 'gary': False, 'oldman': False, 'ex': False, 'gone': False, 'steals': False, 'magical': False, 'sword': False, 'excalibur': False, 'accidentally': False, 'loses': False, 'dangerous': False, 'booby': False, 'trapped': False, 'forest': False, 'help': False, 'hunky': False, 'blind': False, 'timberland': False, 'dweller': False, 'garrett': False, 'carey': False, 'elwes': False, 'headed': False, 'dragon': False, 'eric': False, 'idle': False, 'rickles': False, 'arguing': False, 'itself': False, 'able': False, 'medieval': False, 'sexist': False, 'prove': False, 'fighter': False, 'side': False, 'pure': False, 'showmanship': False, 'essential': False, 'element': False, 'expected': False, 'climb': False, 'high': False, 'ranks': False, 'differentiates': False, 'something': False, 'saturday': False, 'morning': False, 'subpar': False, 'instantly': False, 'forgettable': False, 'songs': False, 'integrated': False, 'computerized': False, 'footage': False, 'compare': False, 'run': False, 'angry': False, 'ogre': False, 'herc': False, 'battle': False, 'hydra': False, 'rest': False, 'case': False, 'stink': False, 'none': False, 'remotely': False, 'interesting': False, 'race': False, 'bland': False, 'end': False, 'tie': False, 'win': False, 'comedy': False, 'shtick': False, 'awfully': False, 'cloying': False, 'least': False, 'signs': False, 'pulse': False, 'fans': False, \"-'\": False, '90s': False, 'tgif': False, 'will': False, 'thrilled': False, 'jaleel': False, 'urkel': False, 'white': False, 'bronson': False, 'balki': False, 'pinchot': False, 'sharing': False, 'nicely': False, 'realized': False, 'though': False, 'm': False, 'loss': False, 'recall': False, 'specific': False, 'providing': False, 'voice': False, 'talent': False, 'enthusiastic': False, 'paired': False, 'singers': False, 'sound': False, 'musical': False, 'moments': False, 'jane': False, 'seymour': False, 'celine': False, 'dion': False, 'must': False, 'strain': False, 'through': False, 'aside': False, 'children': False, 'probably': False, 'adults': False, 'grievous': False, 'error': False, 'lack': False, 'personality': False, 'learn': False, 'goes': False, 'synopsis': False, 'mentally': False, 'unstable': False, 'undergoing': False, 'psychotherapy': False, 'saves': False, 'boy': False, 'potentially': False, 'fatal': False, 'falls': False, 'love': False, 'mother': False, 'fledgling': False, 'restauranteur': False, 'unsuccessfully': False, 'attempting': False, 'gain': False, 'woman': False, 'favor': False, 'takes': False, 'pictures': False, 'kills': False, 'comments': False, 'stalked': False, 'yet': False, 'seemingly': False, 'endless': False, 'string': False, 'spurned': False, 'psychos': False, 'getting': False, 'revenge': False, 'type': False, 'stable': False, 'category': False, '1990s': False, 'industry': False, 'theatrical': False, 'direct': False, 'proliferation': False, 'may': False, 'due': False, 'typically': False, 'inexpensive': False, 'produce': False, 'special': False, 'effects': False, 'stars': False, 'serve': False, 'vehicles': False, 'nudity': False, 'allowing': False, 'frequent': False, 'night': False, 'cable': False, 'wavers': False, 'slightly': False, 'norm': False, 'respect': False, 'psycho': False, 'never': False, 'affair': False, ';': False, 'contrary': False, 'rejected': False, 'rather': False, 'lover': False, 'wife': False, 'husband': False, 'entry': False, 'doomed': False, 'collect': False, 'dust': False, 'viewed': False, 'midnight': False, 'provide': False, 'suspense': False, 'sets': False, 'interspersed': False, 'opening': False, 'credits': False, 'instance': False, 'serious': False, 'sounding': False, 'narrator': False, 'spouts': False, 'statistics': False, 'stalkers': False, 'ponders': False, 'cause': False, 'stalk': False, 'implicitly': False, 'implied': False, 'men': False, 'shown': False, 'snapshot': False, 'actor': False, 'jay': False, 'underwood': False, 'states': False, 'daryl': False, 'gleason': False, 'stalker': False, 'brooke': False, 'daniels': False, 'meant': False, 'called': False, 'guesswork': False, 'required': False, 'proceeds': False, 'begins': False, 'obvious': False, 'sequence': False, 'contrived': False, 'quite': False, 'brings': False, 'victim': False, 'together': False, 'obsesses': False, 'follows': False, 'tries': False, 'woo': False, 'plans': False, 'become': False, 'desperate': False, 'elaborate': False, 'include': False, 'cliche': False, 'murdered': False, 'pet': False, 'require': False, 'found': False, 'exception': False, 'cat': False, 'shower': False, 'events': False, 'lead': False, 'inevitable': False, 'showdown': False, 'survives': False, 'invariably': False, 'conclusion': False, 'turkey': False, 'uniformly': False, 'adequate': False, 'anything': False, 'home': False, 'either': False, 'turns': False, 'toward': False, 'melodrama': False, 'overdoes': False, 'words': False, 'manages': False, 'creepy': False, 'pass': False, 'demands': False, 'maryam': False, 'abo': False, 'close': False, 'played': False, 'bond': False, 'chick': False, 'living': False, 'daylights': False, 'equally': False, 'title': False, 'ditzy': False, 'strong': False, 'independent': False, 'business': False, 'owner': False, 'needs': False, 'proceed': False, 'example': False, 'suspicions': False, 'ensure': False, 'use': False, 'excuse': False, 'decides': False, 'return': False, 'toolbox': False, 'left': False, 'place': False, 'house': False, 'leave': False, 'door': False, 'answers': False, 'opens': False, 'wanders': False, 'returns': False, 'enters': False, 'our': False, 'heroine': False, 'danger': False, 'somehow': False, 'parked': False, 'front': False, 'right': False, 'oblivious': False, 'presence': False, 'inside': False, 'whole': False, 'episode': False, 'places': False, 'incredible': False, 'suspension': False, 'disbelief': False, 'questions': False, 'validity': False, 'intelligence': False, 'receives': False, 'highly': False, 'derivative': False, 'somewhat': False, 'boring': False, 'cannot': False, 'watched': False, 'rated': False, 'mostly': False, 'several': False, 'murder': False, 'brief': False, 'strip': False, 'bar': False, 'offensive': False, 'many': False, 'thrillers': False, 'mood': False, 'stake': False, 'else': False, 'capsule': False, '2176': False, 'planet': False, 'mars': False, 'taking': False, 'custody': False, 'accused': False, 'murderer': False, 'face': False, 'menace': False, 'lot': False, 'fighting': False, 'john': False, 'carpenter': False, 'reprises': False, 'ideas': False, 'previous': False, 'assault': False, 'precinct': False, '13': False, 'homage': False, 'himself': False, '0': False, '+': False, 'believes': False, 'fight': False, 'horrible': False, 'writer': False, 'supposedly': False, 'expert': False, 'mistake': False, 'ghosts': False, 'drawn': False, 'humans': False, 'surprisingly': False, 'low': False, 'powered': False, 'alien': False, 'addition': False, 'anybody': False, 'made': False, 'grounds': False, 'sue': False, 'chock': False, 'full': False, 'pieces': False, 'prince': False, 'darkness': False, 'surprising': False, 'managed': False, 'fit': False, 'admittedly': False, 'novel': False, 'science': False, 'fiction': False, 'experience': False, 'terraformed': False, 'walk': False, 'surface': False, 'without': False, 'breathing': False, 'gear': False, 'budget': False, 'mentioned': False, 'gravity': False, 'increased': False, 'earth': False, 'easier': False, 'society': False, 'changed': False, 'advanced': False, 'culture': False, 'women': False, 'positions': False, 'control': False, 'view': False, 'stagnated': False, 'female': False, 'beyond': False, 'minor': False, 'technological': False, 'advances': False, 'less': False, '175': False, 'expect': False, 'change': False, 'ten': False, 'basic': False, 'common': False, 'except': False, 'yes': False, 'replaced': False, 'tacky': False, 'rundown': False, 'martian': False, 'mining': False, 'colony': False, 'having': False, 'criminal': False, 'napolean': False, 'wilson': False, 'desolation': False, 'williams': False, 'facing': False, 'hoodlums': False, 'automatic': False, 'weapons': False, 'nature': False, 'behave': False, 'manner': False, 'essentially': False, 'human': False, 'savages': False, 'lapse': False, 'imagination': False, 'told': False, 'flashback': False, 'entirely': False, 'filmed': False, 'almost': False, 'tones': False, 'red': False, 'yellow': False, 'black': False, 'powerful': False, 'scene': False, 'train': False, 'rushing': False, 'heavy': False, 'sadly': False, 'buildup': False, 'terror': False, 'creates': False, 'looks': False, 'fugitive': False, 'wannabes': False, 'rock': False, 'band': False, 'kiss': False, 'building': False, 'bunch': False, 'sudden': False, 'jump': False, 'sucker': False, 'thinking': False, 'scary': False, 'happening': False, 'standard': False, 'haunted': False, 'shock': False, 'great': False, 'newer': False, 'unimpressive': False, 'digital': False, 'decapitations': False, 'fights': False, 'short': False, 'stretch': False, 'release': False, 'mission': False, 'panned': False, 'reviewers': False, 'better': False, 'rate': False, 'scale': False, 'following': False, 'showed': False, 'liked': False, 'moderately': False, 'classic': False, 'comment': False, 'twice': False, 'ask': False, 'yourself': False, '8mm': False, 'eight': False, 'millimeter': False, 'wholesome': False, 'surveillance': False, 'sight': False, 'values': False, 'becoming': False, 'enmeshed': False, 'seedy': False, 'sleazy': False, 'underworld': False, 'hardcore': False, 'pornography': False, 'bubbling': False, 'beneath': False, 'town': False, 'americana': False, 'sordid': False, 'sick': False, 'depraved': False, 'necessarily': False, 'stop': False, 'order': False, 'satisfy': False, 'twisted': False, 'desires': False, 'position': False, 'influence': False, 'kinds': False, 'demented': False, 'talking': False, 'snuff': False, 'supposed': False, 'documentaries': False, 'victims': False, 'brutalized': False, 'killed': False, 'camera': False, 'joel': False, 'schumacher': False, 'credit': False, 'batman': False, 'robin': False, 'kill': False, 'forever': False, 'client': False, 'thirds': False, 'unwind': False, 'fairly': False, 'conventional': False, 'persons': False, 'drama': False, 'albeit': False, 'particularly': False, 'unsavory': False, 'core': False, 'threatening': False, 'along': False, 'explodes': False, 'violence': False, 'think': False, 'finally': False, 'tags': False, 'ridiculous': False, 'self': False, 'righteous': False, 'finale': False, 'drags': False, 'unpleasant': False, 'trust': False, 'waste': False, 'hours': False, 'nicolas': False, 'snake': False, 'eyes': False, 'cage': False, 'private': False, 'investigator': False, 'tom': False, 'welles': False, 'hired': False, 'wealthy': False, 'philadelphia': False, 'widow': False, 'determine': False, 'whether': False, 'reel': False, 'safe': False, 'documents': False, 'girl': False, 'assignment': False, 'factly': False, 'puzzle': False, 'neatly': False, 'specialized': False, 'skills': False, 'training': False, 'easy': False, 'cops': False, 'toilet': False, 'tanks': False, 'clues': False, 'deeper': False, 'digs': False, 'investigation': False, 'obsessed': False, 'george': False, 'c': False, 'scott': False, 'paul': False, 'schrader': False, 'occasionally': False, 'flickering': False, 'whirs': False, 'sprockets': False, 'winding': False, 'projector': False, 'reminding': False, 'task': False, 'hints': False, 'toll': False, 'lovely': False, 'catherine': False, 'keener': False, 'frustrated': False, 'cleveland': False, 'ugly': False, 'split': False, 'level': False, 'harrisburg': False, 'pa': False, 'condemn': False, 'condone': False, 'subject': False, 'exploits': False, 'irony': False, 'seven': False, 'scribe': False, 'andrew': False, 'kevin': False, 'walker': False, 'vision': False, 'lane': False, 'limited': False, 'hollywood': False, 'product': False, 'snippets': False, 'covering': False, 'later': False, 'joaquin': False, 'phoenix': False, 'far': False, 'adult': False, 'bookstore': False, 'flunky': False, 'max': False, 'california': False, 'cover': False, 'horrid': False, 'screened': False, 'familiar': False, 'revelation': False, 'sexual': False, 'deviants': False, 'indeed': False, 'monsters': False, 'everyday': False, 'neither': False, 'super': False, 'nor': False, 'shocking': False, 'banality': False, 'exactly': False, 'felt': False, 'weren': False, 'nine': False, 'laughs': False, 'months': False, 'terrible': False, 'mr': False, 'hugh': False, 'grant': False, 'huge': False, 'dork': False, 'oral': False, 'sex': False, 'prostitution': False, 'referring': False, 'bugs': False, 'annoying': False, 'adam': False, 'sandler': False, 'jim': False, 'carrey': False, 'eye': False, 'flutters': False, 'nervous': False, 'smiles': False, 'slapstick': False, 'fistfight': False, 'delivery': False, 'room': False, 'culminating': False, 'joan': False, 'cusack': False, 'lap': False, 'paid': False, '$': False, '60': False, 'included': False, 'obscene': False, 'double': False, 'entendres': False, 'obstetrician': False, 'pregnant': False, 'pussy': False, 'size': False, 'hairs': False, 'coat': False, 'nonetheless': False, 'exchange': False, 'cookie': False, 'cutter': False, 'originality': False, 'humor': False, 'successful': False, 'child': False, 'psychiatrist': False, 'psychologist': False, 'scriptwriters': False, 'could': False, 'inject': False, 'unfunny': False, 'kid': False, 'dad': False, 'asshole': False, 'eyelashes': False, 'offers': False, 'smile': False, 'responds': False, 'english': False, 'accent': False, 'attitude': False, 'possibly': False, '_huge_': False, 'beside': False, 'includes': False, 'needlessly': False, 'stupid': False, 'jokes': False, 'olds': False, 'everyone': False, 'shakes': False, 'anyway': False, 'finds': False, 'usual': False, 'reaction': False, 'fluttered': False, 'paves': False, 'possible': False, 'pregnancy': False, 'birth': False, 'gag': False, 'book': False, 'friend': False, 'arnold': False, 'provides': False, 'cacophonous': False, 'funny': False, 'beats': False, 'costumed': False, 'arnie': False, 'dinosaur': False, 'draw': False, 'parallels': False, 'toy': False, 'store': False, 'jeff': False, 'goldblum': False, 'hid': False, 'dreadful': False, 'hideaway': False, 'artist': False, 'fear': False, 'simultaneous': False, 'longing': False, 'commitment': False, 'doctor': False, 'recently': False, 'switch': False, 'veterinary': False, 'medicine': False, 'obstetrics': False, 'joke': False, 'old': False, 'foreign': False, 'guy': False, 'mispronounces': False, 'stereotype': False, 'say': False, 'yakov': False, 'smirnov': False, 'favorite': False, 'vodka': False, 'hence': False, 'take': False, 'volvo': False, 'nasty': False, 'unamusing': False, 'heads': False, 'simultaneously': False, 'groan': False, 'failure': False, 'loud': False, 'failed': False, 'uninspired': False, 'lunacy': False, 'sunset': False, 'boulevard': False, 'arrest': False, 'please': False, 'caught': False, 'pants': False, 'bring': False, 'theaters': False, 'faces': False, '90': False, 'forced': False, 'unauthentic': False, 'anyone': False, 'q': False, '80': False, 'sorry': False, 'money': False, 'unfulfilled': False, 'desire': False, 'spend': False, 'bucks': False, 'call': False, 'road': False, 'trip': False, 'walking': False, 'wounded': False, 'stellan': False, 'skarsg': False, 'rd': False, 'convincingly': False, 'zombified': False, 'drunken': False, 'loser': False, 'difficult': False, 'smelly': False, 'boozed': False, 'reliable': False, 'swedish': False, 'adds': False, 'depth': False, 'significance': False, 'plodding': False, 'aberdeen': False, 'sentimental': False, 'painfully': False, 'mundane': False, 'european': False, 'playwright': False, 'august': False, 'strindberg': False, 'built': False, 'career': False, 'families': False, 'relationships': False, 'paralyzed': False, 'secrets': False, 'unable': False, 'express': False, 'longings': False, 'accurate': False, 'reflection': False, 'strives': False, 'focusing': False, 'pairing': False, 'alcoholic': False, 'tomas': False, 'alienated': False, 'openly': False, 'hostile': False, 'yuppie': False, 'kaisa': False, 'lena': False, 'headey': False, 'gossip': False, 'haven': False, 'spoken': False, 'wouldn': False, 'norway': False, 'scotland': False, 'automobile': False, 'charlotte': False, 'rampling': False, 'sand': False, 'rotting': False, 'hospital': False, 'bed': False, 'cancer': False, 'soap': False, 'opera': False, 'twist': False, 'days': False, 'live': False, 'blitzed': False, 'step': False, 'foot': False, 'plane': False, 'hits': False, 'open': False, 'loathing': False, 'each': False, 'periodic': False, 'stops': False, 'puke': False, 'dashboard': False, 'whenever': False, 'muttering': False, 'rotten': False, 'turned': False, 'sloshed': False, 'viewpoint': False, 'recognizes': False, 'apple': False, 'hasn': False, 'fallen': False, 'tree': False, 'nosebleeds': False, 'snorting': False, 'coke': False, 'sabotages': False, 'personal': False, 'indifference': False, 'restrain': False, 'vindictive': False, 'temper': False, 'ain': False, 'pair': False, 'true': False, 'notes': False, 'unspoken': False, 'familial': False, 'empathy': False, 'note': False, 'repetitively': False, 'bitchy': False, 'screenwriters': False, 'kristin': False, 'amundsen': False, 'hans': False, 'petter': False, 'moland': False, 'fabricate': False, 'series': False, 'contrivances': False, 'propel': False, 'forward': False, 'roving': False, 'hooligans': False, 'drunks': False, 'nosy': False, 'flat': False, 'tires': False, 'figure': False, 'schematic': False, 'convenient': False, 'narrative': False, 'reach': False, 'unveil': False, 'dark': False, 'past': False, 'simplistic': False, 'devices': False, 'trivialize': False, 'conflict': False, 'mainstays': False, 'wannabe': False, 'exists': False, 'purely': False, 'sake': False, 'weak': False, 'unimaginative': False, 'casting': False, 'thwarts': False, 'pivotal': False, 'role': False, 'were': False, 'stronger': False, 'actress': False, 'perhaps': False, 'coast': False, 'performances': False, 'moody': False, 'haunting': False, 'cinematography': False, 'rendering': False, 'pastoral': False, 'ghost': False, 'reference': False, 'certain': False, 'superior': False, 'indie': False, 'intentional': False, 'busy': False, 'using': False, 'furrowed': False, 'brow': False, 'convey': False, 'twitch': False, 'insouciance': False, 'paying': False, 'attention': False, 'maybe': False, 'doing': False, 'reveal': False, 'worthwhile': False, 'earlier': False, 'released': False, '2001': False, 'jonathan': False, 'nossiter': False, 'captivating': False, 'wonders': False, 'disturbed': False, 'parental': False, 'figures': False, 'bound': False, 'ceremonial': False, 'wedlock': False, 'differences': False, 'presented': False, 'significant': False, 'luminous': False, 'diva': False, 'preening': False, 'static': False, 'solid': False, 'performance': False, 'pathetic': False, 'drunk': False, 'emote': False, 'besides': False, 'catatonic': False, 'sorrow': False, 'genuine': False, 'ferocity': False, 'sexually': False, 'charged': False, 'frisson': False, 'during': False, 'understated': False, 'confrontations': False, 'suggest': False, 'gray': False, 'zone': False, 'complications': False, 'accompany': False, 'torn': False, 'romance': False, 'stifled': False, 'curiosity': False, 'thoroughly': False, 'explores': False, 'neurotic': False, 'territory': False, 'delving': False, 'americanization': False, 'greece': False, 'mysticism': False, 'illusion': False, 'deflect': False, 'pain': False, 'overloaded': False, 'willing': False, 'come': False, 'traditional': False, 'ambitious': False, 'sleepwalk': False, 'rhythms': False, 'timing': False, 'driven': False, 'stories': False, 'complexities': False, 'depressing': False, 'answer': False, 'lawrence': False, 'kasdan': False, 'trite': False, 'useful': False, 'grand': False, 'canyon': False, 'steve': False, 'martin': False, 'mogul': False, 'pronounces': False, 'riddles': False, 'answered': False, 'advice': False, 'heart': False, 'french': False, 'sees': False, 'parents': False, 'tim': False, 'roth': False, 'oops': False, 'vows': False, 'taught': False, 'musketeer': False, 'dude': False, 'used': False, 'fourteen': False, 'arrgh': False, 'swish': False, 'zzzzzzz': False, 'original': False, 'lacks': False, 'energy': False, 'next': False, 'hmmmm': False, 'justin': False, 'chambers': False, 'basically': False, 'uncharismatic': False, 'version': False, 'chris': False, 'o': False, 'donnell': False, 'range': False, 'mena': False, 'suvari': False, 'thora': False, 'birch': False, 'dungeons': False, 'dragons': False, 'miscast': False, 'deliveries': False, 'piss': False, 'poor': False, 'ms': False, 'fault': False, 'definitely': False, 'higher': False, 'semi': False, 'saving': False, 'grace': False, 'wise': False, 'irrepressible': False, 'once': False, 'thousand': False, 'god': False, 'beg': False, 'agent': False, 'marketplace': False, 'modern': False, 'day': False, 'roles': False, 'romantic': False, 'gunk': False, 'alright': False, 'yeah': False, 'yikes': False, 'notches': False, 'fellas': False, 'blares': False, 'ear': False, 'accentuate': False, 'annoy': False, 'important': False, 'behind': False, 'recognize': False, 'epic': False, 'fluffy': False, 'rehashed': False, 'cake': False, 'created': False, 'shrewd': False, 'advantage': False, 'kung': False, 'fu': False, 'phenomenon': False, 'test': False, 'dudes': False, 'keep': False, 'reading': False, 'editing': False, 'shoddy': False, 'banal': False, 'stilted': False, 'plentiful': False, 'top': False, 'horse': False, 'carriage': False, 'stand': False, 'opponent': False, 'scampering': False, 'cut': False, 'mouseketeer': False, 'rope': False, 'tower': False, 'jumping': False, 'chords': False, 'hanging': False, 'says': False, '14': False, 'shirt': False, 'strayed': False, 'championing': False, 'fun': False, 'stretches': False, 'atrocious': False, 'lake': False, 'reminded': False, 'school': False, 'cringe': False, 'musketeers': False, 'fat': False, 'raison': False, 'etre': False, 'numbers': False, 'hoping': False, 'packed': False, 'stuntwork': False, 'promoted': False, 'trailer': False, 'major': False, 'swashbuckling': False, 'beginning': False, 'finishes': False, 'juggling': False, 'ladders': False, 'ladder': False, 'definite': False, 'keeper': False, 'regurgitated': False, 'crap': False, 'tell': False, 'deneuve': False, 'placed': False, 'hullo': False, 'barely': False, 'ugh': False, 'small': False, 'annoyed': False, 'trash': False, 'gang': False, 'vow': False, 'stay': False, 'thank': False, 'outlaws': False, '5': False, 'crouching': False, 'tiger': False, 'hidden': False, 'matrix': False, 'replacement': False, 'killers': False, '6': False, 'romeo': False, 'die': False, 'shanghai': False, 'noon': False, 'remembered': False, 'dr': False, 'hannibal': False, 'lecter': False, 'michael': False, 'mann': False, 'forensics': False, 'thriller': False, 'manhunter': False, 'scottish': False, 'brian': False, 'cox': False, 'works': False, 'usually': False, 'schlock': False, 'halfway': False, 'goodnight': False, 'meaty': False, 'substantial': False, 'brilliant': False, 'check': False, 'dogged': False, 'inspector': False, 'opposite': False, 'frances': False, 'mcdormand': False, 'ken': False, 'loach': False, 'agenda': False, 'harrigan': False, 'disturbing': False, 'l': False, 'e': False, '47': False, 'picked': False, 'sundance': False, 'distributors': False, 'scared': False, 'budge': False, 'dares': False, 'speak': False, 'expresses': False, 'seeking': False, 'adolescents': False, 'pad': False, 'bothered': False, 'members': False, 'presentation': False, 'oddly': False, 'empathetic': False, 'light': False, 'tempered': False, 'robust': False, 'listens': False, 'opposed': False, 'friends': False, 'wire': False, 'act': False, 'confused': False, 'lives': False, 'pay': False, 'courtship': False, 'charming': False, 'temptations': False, 'grown': False, 'stands': False, 'island': False, 'expressway': False, 'slices': False, 'malls': False, 'class': False, 'homes': False, 'suburbia': False, 'filmmaker': False, 'cuesta': False, 'uses': False, 'transparent': False, 'metaphor': False, '15': False, 'protagonist': False, 'howie': False, 'franklin': False, 'dano': False, 'reveals': False, 'morbid': False, 'preoccupation': False, 'death': False, 'citing': False, 'deaths': False, 'alan': False, 'j': False, 'pakula': False, 'songwriter': False, 'harry': False, 'chapin': False, 'exit': False, '52': False, 'fascinated': False, 'feelings': False, 'projected': False, 'bright': False, 'move': False, 'force': False, 'complex': False, 'molesters': False, 'beast': False, 'ashamed': False, 'worked': False, 'ill': False, 'advised': False, 'foray': False, 'unnecessary': False, 'padding': False, 'miserable': False, 'bruce': False, 'altman': False, 'seat': False, 'collar': False, 'crime': False, 'degenerate': False, 'youngsters': False, 'kicks': False, 'robbing': False, 'houses': False, 'homoerotic': False, 'shenanigans': False, 'ass': False, 'terrio': False, 'billy': False, 'kay': False, 'handsome': False, 'artful': False, 'dodger': False, 'add': False, 'themes': False, 'suburban': False, 'ennui': False, 'needed': False, 'awkward': False, 'subplots': False, 'concurrently': False, 'relationship': False, 'evenly': False, 'paced': False, 'exceptionally': False, 'acted': False, 'sporting': False, 'baseball': False, 'cap': False, 'faded': False, 'marine': False, 'tattoo': False, 'bluff': False, 'bluster': False, 'quiet': False, 'glance': False, 'withdrawn': False, 'whose': False, 'dramatic': False, 'choices': False, 'broad': False, 'calling': False, 'haley': False, 'restraint': False, 'admirable': False, 'screenplay': False, 'material': False, 'reads': False, 'walt': False, 'whitman': False, 'poem': False, 'moment': False, 'precious': False, 'lingers': False, 'ecstatic': False, 'hearing': False, 'glenn': False, 'gould': False, 'performing': False, 'bach': False, 'goldberg': False, 'variations': False, 'involving': False, 'walter': False, 'masterson': False, 'jealous': False, 'newbie': False, 'thread': False, 'predictably': False, 'leads': False, 'observational': False, 'portrait': False, 'alienation': False, 'royally': False, 'screwed': False, 'terry': False, 'zwigoff': False, 'superb': False, 'confidence': False, 'ambivalent': False, 'typical': False, 'cinema': False, 'wrap': False, 'bullet': False, 'sparing': False, 'writers': False, 'philosophical': False, 'regard': False, 'countless': False, 'share': False, 'blockbuster': False, 'solved': False, 'obstacle': False, 'removed': False, 'often': False, 'extend': False, 'question': False, 'striving': False, 'realism': False, 'destroy': False, 'janeane': False, 'garofalo': False, 'couple': False, 'truth': False, 'cats': False, 'dogs': False, 'excruciating': False, 'matchmaker': False, 'books': False, 'plods': False, 'predestined': False, 'surprises': False, 'jumps': False, 'popular': False, 'political': False, 'satire': False, 'bandwagon': False, 'campaign': False, 'aide': False, 'massacusetts': False, 'senator': False, 'sanders': False, 'reelection': False, 'denis': False, 'leary': False, 'stereotypical': False, 'strategist': False, 'ethics': False, 'scandal': False, 'plagued': False, 'play': False, 'irish': False, 'roots': False, 'boston': False, 'roman': False, 'catholic': False, 'democrat': False, 'contingent': False, 'kennedy': False, 'family': False, 'orders': False, 'ireland': False, 'relatives': False, 'exploit': False, 'soon': False, 'learns': False, 'said': False, 'done': False, 'mantra': False, 'tiny': False, 'misses': False, 'bus': False, 'hotel': False, 'ends': False, 'smallest': False, 'trashiest': False, 'dog': False, 'luggage': False, 'roger': False, 'ebert': False, 'calls': False, 'meet': False, 'happens': False, 'unconventional': False, 'cinematic': False, 'walks': False, 'bathroom': False, 'nude': False, 'sean': False, 'david': False, 'hara': False, 'bathtub': False, 'points': False, 'guessing': False, 'water': False, 'hates': False, 'instant': False, 'saw': False, 'irishman': False, 'hate': False, 'awhile': False, 'succumb': False, 'charms': False, 'happily': False, 'superficial': False, 'detail': False, 'throw': False, 'turmoil': False, 'reconcile': False, 'tune': False, 'annual': False, 'matchmaking': False, 'festival': False, 'lonely': False, 'county': False, 'future': False, 'bliss': False, 'milo': False, 'shea': False, 'snyder': False, 'pops': False, 'onscreen': False, 'spew': False, 'souls': False, 'assured': False, 'match': False, 'utter': False, 'predictability': False, 'message': False, 'respectable': False, 'person': False, 'comedic': False, 'distinction': False, 'sell': False, 'script': False, 'excited': False, 'stays': False, 'stateside': False, 'yelling': False, 'phone': False, 'undoes': False, 'microphone': False, 'speech': False, 'known': False, 'flying': False, 'hong': False, 'kong': False, 'style': False, 'filmmaking': False, 'classics': False, 'nod': False, 'asia': False, 'france': False, 'lukewarm': False, 'dumas': False, 'asian': False, 'stunt': False, 'coordinator': False, 'xing': False, 'xiong': False, 'prior': False, 'attempts': False, 'choreography': False, 'laughable': False, 'van': False, 'damme': False, 'vehicle': False, 'team': False, 'dennis': False, 'rodman': False, 'simon': False, 'sez': False, 'thrown': False, 'air': False, 'result': False, 'tepid': False, 'adventure': False, 'rip': False, 'stinks': False, 'indiana': False, 'jones': False, 'simple': False, 'grandmother': False, 'adapted': False, 'artagnan': False, 'vengeful': False, 'son': False, 'slain': False, 'travels': False, 'paris': False, 'join': False, 'royal': False, 'meets': False, 'cunning': False, 'cardinal': False, 'richelieu': False, 'stephen': False, 'rea': False, 'overthrow': False, 'associate': False, 'febre': False, 'killer': False, 'disbanded': False, 'rounds': False, 'aramis': False, 'nick': False, 'moran': False, 'athos': False, 'jan': False, 'gregor': False, 'kremp': False, 'porthos': False, 'steven': False, 'spiers': False, 'wrongfully': False, 'imprisoned': False, 'leader': False, 'treville': False, 'prison': False, 'frisky': False, 'interest': False, 'chambermaid': False, 'francesca': False, 'footsy': False, 'coo': False, 'hunts': False, 'queen': False, 'captured': False, 'menancing': False, 'forcing': False, 'regroup': False, 'leading': False, 'charge': False, 'peter': False, 'hyams': False, 'wanted': False, 'blend': False, 'eastern': False, 'western': False, 'styles': False, 'disaster': False, 'reality': False, 'ones': False, 'jet': False, 'li': False, 'risk': False, 'ironically': False, 'swordplay': False, 'spread': False, 'carry': False, 'bulk': False, '30': False, 'minute': False, 'picture': False, 'weighs': False, 'monotonous': False, 'gene': False, 'quintano': False, 'prosaic': False, 'wedding': False, 'planner': False, 'mousy': False, 'artangnan': False, 'hyam': False, 'candles': False, 'torches': False, 'grime': False, 'filth': False, '17th': False, 'noted': False, 'standout': False, 'mortal': False, 'kombat': False, 'annihilation': False, 'reviewed': False, 'multiple': False, 'levels': False, 'rampant': False, 'usage': False, 'randian': False, 'subtext': False, 'pervades': False, 'occasionaly': False, 'ironic': False, 'depreciating': False, 'remark': False, 'tosses': False, 'clearly': False, 'marxist': False, 'imagery': False, 'kidding': False, 'seriousness': False, 'fair': False, '*': False, 'necessary': False, 'viewpoints': False, 'watcher': False, 'unfamiliar': False, 'marginally': False, 'fan': False, 'games': False, '1995': False, 'concerned': False, 'martial': False, 'arts': False, 'tournament': False, 'decide': False, 'fate': False, 'billion': False, 'inhabitants': False, 'mortals': False, 'theory': False, 'prevented': False, 'emperor': False, 'shao': False, 'khan': False, 'arriving': False, 'ready': False, 'assumed': False, 'stance': False, 'extraordinarily': False, 'myself': False, 'game': False, 'enjoyed': False, 'directors': False, 'knew': False, 'limitations': False, 'try': False, 'overachieve': False, 'accompanying': False, 'intersperesed': False, 'distracting': False, 'non': False, 'intrusive': False, 'bits': False, 'fluff': False, 'passing': False, 'smashing': False, 'success': False, 'box': False, 'office': False, 'picks': False, 'precisely': False, 'introductory': False, 'exposition': False, 'anyways': False, 'hell': False, 'silly': False, 'rule': False, 'winning': False, 'thereafter': False, 'approximately': False, '85': False, 'alternates': False, 'general': False, 'impression': False, 'producers': False, 'thought': False, 'formula': False, 'volumes': False, 'truly': False, 'sandra': False, 'hess': False, 'sonya': False, 'blade': False, 'execrable': False, 'convince': False, 'loved': False, 'johnny': False, 'greased': False, 'worst': False, 'mis': False, 'james': False, 'remar': False, 'raiden': False, 'thunder': False, 'christopher': False, 'lambert': False, 'japanese': False, 'revered': False, 'chinese': False, 'mystics': False, 'against': False, 'lister': False, 'jr': False, 'president': False, 'u': False, 'fifth': False, 'utility': False, 'totally': False, 'luxury': False, 'amused': False, 'awareness': False, 'introduced': False, 'meaningless': False, 'sidetracks': False, 'including': False, 'muddled': False, 'liu': False, 'kang': False, 'shou': False, 'seeks': False, 'nightwolf': False, 'litefoot': False, 'mystical': False, 'hallucination': False, 'jade': False, 'irina': False, 'pantaeva': False, 'reasons': False, 'unless': False, 'critiques': False, 'apply': False, 'worse': False, 'bridgette': False, 'convincing': False, 'looked': False, 'mimicing': False, 'movements': False, 'choreographer': False, 'knows': False, 'puts': False, 'believable': False, 'jax': False, 'earthquake': False, 'animality': False, 'bonus': False, 'similar': False, 'moves': False, 'mistakenly': False, 'hang': False, 'lamest': False, 'involved': False, 'mud': False, 'wrestling': False, 'lame': False, 'politically': False, 'incorrect': False, 'noticed': False, 'remarked': False, 'upon': False, 'emporer': False, 'perform': False, 'animalities': False, 'motaro': False, 'sheeva': False, 'lifelike': False, 'goro': False, 'enjoy': False, 'femme': False, 'la': False, 'nikita': False, 'backdraft': False, 'sliver': False, 'cindy': False, 'crawford': False, 'anne': False, 'parillaud': False, 'conspire': False, 'shattered': False, 'image': False, 'hooey': False, 'stallone': False, 'stone': False, 'specialist': False, 'poses': False, 'recurring': False, 'assassin': False, 'honeymooning': False, 'jamaica': False, 'believe': False, 'runs': False, 'painful': False, 'pedestrian': False, 'siouxsie': False, 'sioux': False, 'wig': False, 'emotionless': False, 'leather': False, 'clothing': False, 'seattle': False, 'moping': False, 'karen': False, 'endlessly': False, 'complicated': False, 'plots': False, 'helps': False, 'crisp': False, 'modicum': False, 'shakespearean': False, 'begin': False, 'saddled': False, 'leaden': False, 'zero': False, 'breaking': False, 'cardboard': False, 'confines': False, 'insist': False, 'couldn': False, 'huh': False, 'wonderful': False, 'interchange': False, 'charm': False, 'faster': False, 'learned': False, 'cereal': False, 'crybaby': False, 'imagines': False, 'stranger': False, 'sends': False, 'flowers': False, 'chromium': False, 'tough': False, 'nails': False, 'crack': False, 'killing': False, 'machine': False, 'shoots': False, 'mirrors': False, 'stock': False, 'interested': False, 'nest': False, 'egg': False, 'pave': False, 'paradise': False, 'put': False, 'parking': False, 'graham': False, 'greene': False, 'barbet': False, 'schroeder': False, 'reversal': False, 'fortune': False, 'co': False, 'produced': False, 'agonizingly': False, 'b': False, 'york': False, 'vampires': False, 'latest': False, 'opus': False, 'bent': False, 'outing': False, 'aptly': False, 'titled': False, 'suppose': False, 'went': False, 'prefixed': False, 'possessive': False, 'unashamed': False, 'punctuated': False, 'above': False, 'storyline': False, 'borders': False, 'idiotic': False, 'chaotic': False, 'dormant': False, 'martians': False, 'swirling': False, 'gases': False, 'awakened': False, 'meddling': False, 'possess': False, 'hapless': False, 'colonists': False, 'testy': False, 'marilyn': False, 'manson': False, 'lookalikes': False, 'pooh': False, 'bah': False, 'counsel': False, 'official': False, 'melanie': False, 'ballard': False, 'natasha': False, 'henstridge': False, 'sub': False, 'species': False, 'returnee': False, 'officer': False, 'incarcerated': False, 'felon': False, 'second': False, 'blonde': False, 'pulled': False, 'tightly': False, 'awkwardly': False, 'ponytail': False, 'ice': False, 'cube': False, 'appropriately': False, 'named': False, 'pam': False, 'grier': False, 'briefly': False, 'whom': False, 'wonder': False, 'host': False, 'extras': False, 'therefore': False, 'bird': False, 'shots': False, 'sprawling': False, 'metropolis': False, 'reddish': False, 'state': False, 'art': False, 'trademark': False, 'finding': False, 'department': False, 'lock': False, 'laughing': False, 'barrel': False, 'fare': False, 'dingy': False, 'interiors': False, 'cluttered': False, 'exteriors': False, 'inane': False, 'lots': False, 'scarred': False, 'crazed': False, 'aliens': False, 'weaponry': False, 'warfare': False, 'warning': False, 'spontaneously': False, 'stupidly': False, 'villains': False, 'border': False, 'conflicts': False, 'shootouts': False, 'minus': False, 'hissing': False, 'plissken': False, 'miss': False, 'dubbed': False, 'minimalist': False, 'soundtracks': False, 'graduated': False, 'effective': False, 'scoring': False, 'highlighting': False, 'screeching': False, 'guitar': False, 'fortunately': False, 'drowns': False, 'er': False, 'audible': False, 'priceless': False, 'proven': False, 'infertile': False, 'breeding': False, 'ground': False, 'stillborn': False, 'val': False, 'kilmer': False, 'disappointing': False, 'weekend': False, 'overshadowed': False, 'sequels': False, 'among': False, 'pie': False, 'rush': False, 'absent': False, 'references': False, 'set': False, 'perth': False, 'amboys': False, 'closest': False, 'neighbor': False, 'slap': False, 'upside': False, '1': False, 'keeps': False, 'miraculously': False, 'pretend': False, 'means': False, 'intelligent': False, 'grade': False, 'sci': False, 'fi': False, 'singularly': False, 'luck': False, 'starting': False, 'alicia': False, 'silverstone': False, 'beautiful': False, 'creatures': False, 'green': False, 'critic': False, 'large': False, 'choosing': False, 'strikes': False, 'crush': False, 'slow': False, 'moving': False, 'horrific': False, 'adaptation': False, 'clueless': False, 'mailed': False, 'saying': False, 'theater': False, 'expecting': False, 'preview': False, 'crazymadinlove': False, 'whiny': False, 'unlikable': False, 'wasn': False, 'yelled': False, 'f': False, '$&#': False, 'laugh': False, 'agreement': False, 'walked': False, 'babysitter': False, 'inner': False, 'compulsion': False, 'understand': False, 'rent': False, 'regret': False, 'paragraph': False, 'competition': False, 'criticizing': False, 'thin': False, 'shred': False, 'slower': False, 'glacier': False, 'writing': False, 'appeal': False, 'whatsoever': False, 'pointlessly': False, 'concluded': False, 'violent': False, 'plus': False, 'equals': False, 'spends': False, 'twenty': False, 'bubble': False, 'bath': False, 'joined': False, 'four': False, 'features': False, 'settle': False, 'friday': False, 'cocktail': False, 'automatically': False, 'nights': False, 'trods': False, 'discover': False, 'silent': False, 'object': False, 'male': False, 'fantasies': False, 'thinks': False, 'recapture': False, 'youth': False, 'boyfriend': False, 'lets': False, 'wild': False, 'spying': False, 'outside': False, 'prepubescent': False, 'keyhole': False, 'aged': False, 'counterpart': False, 'asked': False, '200': False, 'pound': False, 'silk': False, 'teddy': False, 'fanatasies': False, 'realm': False, 'pg': False, 'imagine': False, 'cinemax': False, 'staple': False, 'absolutely': False, 'pointed': False, 'classify': False, 'mix': False, 'various': False, 'encounters': False, 'third': False, 'space': False, 'odyssey': False, 'apollo': False, 'contact': False, 'hope': False, 'melange': False, 'considering': False, 'disastrous': False, 'results': False, 'sucks': False, 'rescue': False, 'astronauts': False, 'sent': False, '2020': False, 'unknown': False, 'aviators': False, 'visit': False, 'underwhelming': False, 'describe': False, 'uneven': False, 'promise': False, 'buzz': False, 'neutral': False, 'cherry': False, 'colored': False, 'nerdies': False, 'techie': False}\n"
     ]
    }
   ],
   "source": [
    "print((find_features(movie_reviews.words('neg/cv000_29416.txt'))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "featuresets = [(find_features(rev), category) for (rev, category) in documents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'plot': False,\n",
       "  ':': True,\n",
       "  'two': False,\n",
       "  'teen': False,\n",
       "  'couples': False,\n",
       "  'go': False,\n",
       "  'to': True,\n",
       "  'a': True,\n",
       "  'church': False,\n",
       "  'party': False,\n",
       "  ',': True,\n",
       "  'drink': False,\n",
       "  'and': True,\n",
       "  'then': False,\n",
       "  'drive': False,\n",
       "  '.': True,\n",
       "  'they': True,\n",
       "  'get': True,\n",
       "  'into': True,\n",
       "  'an': True,\n",
       "  'accident': False,\n",
       "  'one': False,\n",
       "  'of': True,\n",
       "  'the': True,\n",
       "  'guys': False,\n",
       "  'dies': False,\n",
       "  'but': True,\n",
       "  'his': True,\n",
       "  'girlfriend': False,\n",
       "  'continues': False,\n",
       "  'see': True,\n",
       "  'him': False,\n",
       "  'in': True,\n",
       "  'her': False,\n",
       "  'life': False,\n",
       "  'has': True,\n",
       "  'nightmares': False,\n",
       "  'what': True,\n",
       "  \"'\": True,\n",
       "  's': True,\n",
       "  'deal': False,\n",
       "  '?': False,\n",
       "  'watch': False,\n",
       "  'movie': True,\n",
       "  '\"': False,\n",
       "  'sorta': False,\n",
       "  'find': False,\n",
       "  'out': False,\n",
       "  'critique': False,\n",
       "  'mind': False,\n",
       "  '-': True,\n",
       "  'fuck': False,\n",
       "  'for': True,\n",
       "  'generation': False,\n",
       "  'that': True,\n",
       "  'touches': False,\n",
       "  'on': True,\n",
       "  'very': False,\n",
       "  'cool': False,\n",
       "  'idea': False,\n",
       "  'presents': False,\n",
       "  'it': True,\n",
       "  'bad': False,\n",
       "  'package': False,\n",
       "  'which': True,\n",
       "  'is': True,\n",
       "  'makes': False,\n",
       "  'this': True,\n",
       "  'review': True,\n",
       "  'even': False,\n",
       "  'harder': False,\n",
       "  'write': False,\n",
       "  'since': False,\n",
       "  'i': True,\n",
       "  'generally': False,\n",
       "  'applaud': False,\n",
       "  'films': False,\n",
       "  'attempt': False,\n",
       "  'break': True,\n",
       "  'mold': False,\n",
       "  'mess': False,\n",
       "  'with': True,\n",
       "  'your': True,\n",
       "  'head': True,\n",
       "  'such': False,\n",
       "  '(': True,\n",
       "  'lost': True,\n",
       "  'highway': False,\n",
       "  '&': False,\n",
       "  'memento': False,\n",
       "  ')': True,\n",
       "  'there': True,\n",
       "  'are': True,\n",
       "  'good': True,\n",
       "  'ways': False,\n",
       "  'making': False,\n",
       "  'all': True,\n",
       "  'types': False,\n",
       "  'these': False,\n",
       "  'folks': False,\n",
       "  'just': True,\n",
       "  'didn': True,\n",
       "  't': True,\n",
       "  'snag': False,\n",
       "  'correctly': False,\n",
       "  'seem': False,\n",
       "  'have': True,\n",
       "  'taken': False,\n",
       "  'pretty': False,\n",
       "  'neat': False,\n",
       "  'concept': False,\n",
       "  'executed': False,\n",
       "  'terribly': False,\n",
       "  'so': False,\n",
       "  'problems': False,\n",
       "  'well': True,\n",
       "  'its': False,\n",
       "  'main': False,\n",
       "  'problem': False,\n",
       "  'simply': False,\n",
       "  'too': True,\n",
       "  'jumbled': False,\n",
       "  'starts': False,\n",
       "  'off': False,\n",
       "  'normal': False,\n",
       "  'downshifts': False,\n",
       "  'fantasy': False,\n",
       "  'world': False,\n",
       "  'you': True,\n",
       "  'as': True,\n",
       "  'audience': False,\n",
       "  'member': False,\n",
       "  'no': True,\n",
       "  'going': True,\n",
       "  'dreams': False,\n",
       "  'characters': False,\n",
       "  'coming': False,\n",
       "  'back': False,\n",
       "  'from': True,\n",
       "  'dead': True,\n",
       "  'others': False,\n",
       "  'who': True,\n",
       "  'look': False,\n",
       "  'like': True,\n",
       "  'strange': False,\n",
       "  'apparitions': False,\n",
       "  'disappearances': False,\n",
       "  'looooot': False,\n",
       "  'chase': False,\n",
       "  'scenes': True,\n",
       "  'tons': False,\n",
       "  'weird': False,\n",
       "  'things': False,\n",
       "  'happen': False,\n",
       "  'most': True,\n",
       "  'not': True,\n",
       "  'explained': False,\n",
       "  'now': True,\n",
       "  'personally': False,\n",
       "  'don': False,\n",
       "  'trying': False,\n",
       "  'unravel': False,\n",
       "  'film': True,\n",
       "  'every': False,\n",
       "  'when': True,\n",
       "  'does': True,\n",
       "  'give': False,\n",
       "  'me': True,\n",
       "  'same': True,\n",
       "  'clue': False,\n",
       "  'over': False,\n",
       "  'again': True,\n",
       "  'kind': True,\n",
       "  'fed': False,\n",
       "  'up': False,\n",
       "  'after': False,\n",
       "  'while': False,\n",
       "  'biggest': False,\n",
       "  'obviously': False,\n",
       "  'got': False,\n",
       "  'big': False,\n",
       "  'secret': False,\n",
       "  'hide': False,\n",
       "  'seems': False,\n",
       "  'want': True,\n",
       "  'completely': False,\n",
       "  'until': False,\n",
       "  'final': False,\n",
       "  'five': False,\n",
       "  'minutes': False,\n",
       "  'do': True,\n",
       "  'make': False,\n",
       "  'entertaining': False,\n",
       "  'thrilling': False,\n",
       "  'or': True,\n",
       "  'engaging': False,\n",
       "  'meantime': False,\n",
       "  'really': False,\n",
       "  'sad': False,\n",
       "  'part': False,\n",
       "  'arrow': False,\n",
       "  'both': False,\n",
       "  'dig': False,\n",
       "  'flicks': False,\n",
       "  'we': True,\n",
       "  'actually': False,\n",
       "  'figured': False,\n",
       "  'by': False,\n",
       "  'half': True,\n",
       "  'way': False,\n",
       "  'point': False,\n",
       "  'strangeness': False,\n",
       "  'did': False,\n",
       "  'start': False,\n",
       "  'little': True,\n",
       "  'bit': False,\n",
       "  'sense': False,\n",
       "  'still': True,\n",
       "  'more': True,\n",
       "  'guess': False,\n",
       "  'bottom': False,\n",
       "  'line': False,\n",
       "  'movies': True,\n",
       "  'should': False,\n",
       "  'always': False,\n",
       "  'sure': False,\n",
       "  'before': False,\n",
       "  'given': False,\n",
       "  'password': False,\n",
       "  'enter': False,\n",
       "  'understanding': False,\n",
       "  'mean': False,\n",
       "  'showing': False,\n",
       "  'melissa': False,\n",
       "  'sagemiller': False,\n",
       "  'running': False,\n",
       "  'away': False,\n",
       "  'visions': False,\n",
       "  'about': True,\n",
       "  '20': False,\n",
       "  'throughout': False,\n",
       "  'plain': False,\n",
       "  'lazy': False,\n",
       "  '!': False,\n",
       "  'okay': False,\n",
       "  'people': False,\n",
       "  'chasing': False,\n",
       "  'know': False,\n",
       "  'need': True,\n",
       "  'how': False,\n",
       "  'giving': False,\n",
       "  'us': True,\n",
       "  'different': False,\n",
       "  'offering': False,\n",
       "  'further': False,\n",
       "  'insight': False,\n",
       "  'down': False,\n",
       "  'apparently': False,\n",
       "  'studio': False,\n",
       "  'took': False,\n",
       "  'director': False,\n",
       "  'chopped': False,\n",
       "  'themselves': False,\n",
       "  'shows': False,\n",
       "  'might': True,\n",
       "  've': False,\n",
       "  'been': True,\n",
       "  'decent': False,\n",
       "  'here': True,\n",
       "  'somewhere': False,\n",
       "  'suits': False,\n",
       "  'decided': False,\n",
       "  'turning': False,\n",
       "  'music': False,\n",
       "  'video': False,\n",
       "  'edge': False,\n",
       "  'would': True,\n",
       "  'actors': True,\n",
       "  'although': True,\n",
       "  'wes': False,\n",
       "  'bentley': False,\n",
       "  'seemed': False,\n",
       "  'be': True,\n",
       "  'playing': False,\n",
       "  'exact': False,\n",
       "  'character': False,\n",
       "  'he': True,\n",
       "  'american': False,\n",
       "  'beauty': False,\n",
       "  'only': False,\n",
       "  'new': False,\n",
       "  'neighborhood': False,\n",
       "  'my': False,\n",
       "  'kudos': False,\n",
       "  'holds': False,\n",
       "  'own': False,\n",
       "  'entire': False,\n",
       "  'feeling': False,\n",
       "  'unraveling': False,\n",
       "  'overall': False,\n",
       "  'doesn': False,\n",
       "  'stick': False,\n",
       "  'because': True,\n",
       "  'entertain': False,\n",
       "  'confusing': False,\n",
       "  'rarely': False,\n",
       "  'excites': False,\n",
       "  'feels': False,\n",
       "  'redundant': True,\n",
       "  'runtime': False,\n",
       "  'despite': False,\n",
       "  'ending': False,\n",
       "  'explanation': False,\n",
       "  'craziness': False,\n",
       "  'came': False,\n",
       "  'oh': False,\n",
       "  'horror': True,\n",
       "  'slasher': False,\n",
       "  'flick': False,\n",
       "  'packaged': False,\n",
       "  'someone': False,\n",
       "  'assuming': False,\n",
       "  'genre': True,\n",
       "  'hot': False,\n",
       "  'kids': False,\n",
       "  'also': False,\n",
       "  'wrapped': False,\n",
       "  'production': False,\n",
       "  'years': False,\n",
       "  'ago': False,\n",
       "  'sitting': False,\n",
       "  'shelves': False,\n",
       "  'ever': False,\n",
       "  'whatever': False,\n",
       "  'skip': False,\n",
       "  'where': False,\n",
       "  'joblo': False,\n",
       "  'nightmare': False,\n",
       "  'elm': False,\n",
       "  'street': False,\n",
       "  '3': False,\n",
       "  '7': False,\n",
       "  '/': False,\n",
       "  '10': False,\n",
       "  'blair': False,\n",
       "  'witch': False,\n",
       "  '2': False,\n",
       "  'crow': False,\n",
       "  '9': False,\n",
       "  'salvation': False,\n",
       "  '4': False,\n",
       "  'stir': False,\n",
       "  'echoes': False,\n",
       "  '8': False,\n",
       "  'happy': False,\n",
       "  'bastard': False,\n",
       "  'quick': False,\n",
       "  'damn': False,\n",
       "  'y2k': False,\n",
       "  'bug': False,\n",
       "  'starring': False,\n",
       "  'jamie': False,\n",
       "  'lee': False,\n",
       "  'curtis': False,\n",
       "  'another': False,\n",
       "  'baldwin': False,\n",
       "  'brother': False,\n",
       "  'william': False,\n",
       "  'time': True,\n",
       "  'story': False,\n",
       "  'regarding': False,\n",
       "  'crew': False,\n",
       "  'tugboat': False,\n",
       "  'comes': False,\n",
       "  'across': False,\n",
       "  'deserted': False,\n",
       "  'russian': False,\n",
       "  'tech': False,\n",
       "  'ship': False,\n",
       "  'kick': False,\n",
       "  'power': False,\n",
       "  'within': False,\n",
       "  'gore': False,\n",
       "  'bringing': False,\n",
       "  'few': False,\n",
       "  'action': True,\n",
       "  'sequences': False,\n",
       "  'virus': True,\n",
       "  'empty': False,\n",
       "  'flash': False,\n",
       "  'substance': False,\n",
       "  'why': False,\n",
       "  'was': False,\n",
       "  'middle': False,\n",
       "  'nowhere': False,\n",
       "  'origin': False,\n",
       "  'pink': False,\n",
       "  'flashy': False,\n",
       "  'thing': True,\n",
       "  'hit': True,\n",
       "  'mir': False,\n",
       "  'course': False,\n",
       "  'donald': False,\n",
       "  'sutherland': False,\n",
       "  'stumbling': False,\n",
       "  'around': False,\n",
       "  'drunkenly': False,\n",
       "  'hey': False,\n",
       "  'let': False,\n",
       "  'some': True,\n",
       "  'robots': False,\n",
       "  'acting': False,\n",
       "  'below': False,\n",
       "  'average': True,\n",
       "  'likes': False,\n",
       "  're': False,\n",
       "  'likely': False,\n",
       "  'work': False,\n",
       "  'halloween': False,\n",
       "  'h20': False,\n",
       "  'wasted': False,\n",
       "  'real': False,\n",
       "  'star': False,\n",
       "  'stan': False,\n",
       "  'winston': False,\n",
       "  'robot': False,\n",
       "  'design': False,\n",
       "  'schnazzy': False,\n",
       "  'cgi': True,\n",
       "  'occasional': False,\n",
       "  'shot': False,\n",
       "  'picking': False,\n",
       "  'brain': False,\n",
       "  'if': True,\n",
       "  'body': False,\n",
       "  'parts': False,\n",
       "  'turn': False,\n",
       "  'otherwise': False,\n",
       "  'much': True,\n",
       "  'sunken': False,\n",
       "  'jaded': False,\n",
       "  'viewer': False,\n",
       "  'thankful': False,\n",
       "  'invention': False,\n",
       "  'timex': False,\n",
       "  'indiglo': False,\n",
       "  'based': False,\n",
       "  'late': False,\n",
       "  '1960': False,\n",
       "  'television': False,\n",
       "  'show': False,\n",
       "  'name': False,\n",
       "  'mod': False,\n",
       "  'squad': False,\n",
       "  'tells': False,\n",
       "  'tale': False,\n",
       "  'three': False,\n",
       "  'reformed': False,\n",
       "  'criminals': False,\n",
       "  'under': False,\n",
       "  'employ': False,\n",
       "  'police': False,\n",
       "  'undercover': False,\n",
       "  'however': False,\n",
       "  'wrong': False,\n",
       "  'evidence': False,\n",
       "  'gets': False,\n",
       "  'stolen': False,\n",
       "  'immediately': False,\n",
       "  'suspicion': False,\n",
       "  'ads': False,\n",
       "  'cuts': False,\n",
       "  'claire': False,\n",
       "  'dane': False,\n",
       "  'nice': False,\n",
       "  'hair': False,\n",
       "  'cute': False,\n",
       "  'outfits': False,\n",
       "  'car': True,\n",
       "  'chases': False,\n",
       "  'stuff': False,\n",
       "  'blowing': False,\n",
       "  'sounds': False,\n",
       "  'first': False,\n",
       "  'fifteen': False,\n",
       "  'quickly': False,\n",
       "  'becomes': False,\n",
       "  'apparent': False,\n",
       "  'certainly': False,\n",
       "  'slick': False,\n",
       "  'looking': False,\n",
       "  'complete': False,\n",
       "  'costumes': False,\n",
       "  'isn': False,\n",
       "  'enough': False,\n",
       "  'best': False,\n",
       "  'described': False,\n",
       "  'cross': False,\n",
       "  'between': False,\n",
       "  'hour': False,\n",
       "  'long': False,\n",
       "  'cop': False,\n",
       "  'stretched': False,\n",
       "  'span': False,\n",
       "  'single': True,\n",
       "  'clich': False,\n",
       "  'matter': False,\n",
       "  'elements': False,\n",
       "  'recycled': False,\n",
       "  'everything': True,\n",
       "  'already': False,\n",
       "  'seen': False,\n",
       "  'nothing': True,\n",
       "  'spectacular': False,\n",
       "  'sometimes': False,\n",
       "  'bordering': False,\n",
       "  'wooden': False,\n",
       "  'danes': False,\n",
       "  'omar': False,\n",
       "  'epps': False,\n",
       "  'deliver': False,\n",
       "  'their': False,\n",
       "  'lines': False,\n",
       "  'bored': False,\n",
       "  'transfers': False,\n",
       "  'onto': False,\n",
       "  'escape': False,\n",
       "  'relatively': False,\n",
       "  'unscathed': False,\n",
       "  'giovanni': False,\n",
       "  'ribisi': False,\n",
       "  'plays': False,\n",
       "  'resident': False,\n",
       "  'crazy': False,\n",
       "  'man': True,\n",
       "  'ultimately': False,\n",
       "  'being': True,\n",
       "  'worth': False,\n",
       "  'watching': False,\n",
       "  'unfortunately': True,\n",
       "  'save': False,\n",
       "  'convoluted': False,\n",
       "  'apart': False,\n",
       "  'occupying': False,\n",
       "  'screen': False,\n",
       "  'young': False,\n",
       "  'cast': True,\n",
       "  'clothes': False,\n",
       "  'hip': False,\n",
       "  'soundtrack': False,\n",
       "  'appears': False,\n",
       "  'geared': False,\n",
       "  'towards': False,\n",
       "  'teenage': True,\n",
       "  'mindset': False,\n",
       "  'r': False,\n",
       "  'rating': False,\n",
       "  'content': False,\n",
       "  'justify': False,\n",
       "  'juvenile': False,\n",
       "  'older': False,\n",
       "  'information': False,\n",
       "  'literally': False,\n",
       "  'spoon': False,\n",
       "  'hard': False,\n",
       "  'instead': True,\n",
       "  'telling': False,\n",
       "  'dialogue': False,\n",
       "  'poorly': False,\n",
       "  'written': False,\n",
       "  'extremely': False,\n",
       "  'predictable': False,\n",
       "  'progresses': False,\n",
       "  'won': False,\n",
       "  'care': False,\n",
       "  'heroes': False,\n",
       "  'any': True,\n",
       "  'jeopardy': False,\n",
       "  'll': True,\n",
       "  'aren': True,\n",
       "  'basing': False,\n",
       "  'nobody': False,\n",
       "  'remembers': False,\n",
       "  'questionable': False,\n",
       "  'wisdom': False,\n",
       "  'especially': False,\n",
       "  'considers': False,\n",
       "  'target': False,\n",
       "  'fact': False,\n",
       "  'number': False,\n",
       "  'memorable': False,\n",
       "  'can': True,\n",
       "  'counted': False,\n",
       "  'hand': True,\n",
       "  'missing': False,\n",
       "  'finger': False,\n",
       "  'times': False,\n",
       "  'checked': False,\n",
       "  'six': False,\n",
       "  'clear': False,\n",
       "  'indication': False,\n",
       "  'them': True,\n",
       "  'than': True,\n",
       "  'cash': False,\n",
       "  'spending': False,\n",
       "  'dollar': False,\n",
       "  'judging': False,\n",
       "  'rash': False,\n",
       "  'awful': False,\n",
       "  'seeing': False,\n",
       "  'avoid': False,\n",
       "  'at': False,\n",
       "  'costs': False,\n",
       "  'quest': False,\n",
       "  'camelot': False,\n",
       "  'warner': False,\n",
       "  'bros': False,\n",
       "  'feature': False,\n",
       "  'length': False,\n",
       "  'fully': False,\n",
       "  'animated': False,\n",
       "  'steal': False,\n",
       "  'clout': False,\n",
       "  'disney': False,\n",
       "  'cartoon': False,\n",
       "  'empire': False,\n",
       "  'mouse': False,\n",
       "  'reason': True,\n",
       "  'worried': False,\n",
       "  'other': True,\n",
       "  'recent': False,\n",
       "  'challenger': False,\n",
       "  'throne': False,\n",
       "  'last': True,\n",
       "  'fall': False,\n",
       "  'promising': False,\n",
       "  'flawed': False,\n",
       "  '20th': False,\n",
       "  'century': False,\n",
       "  'fox': True,\n",
       "  'anastasia': False,\n",
       "  'hercules': False,\n",
       "  'lively': False,\n",
       "  'colorful': False,\n",
       "  'palate': False,\n",
       "  'had': False,\n",
       "  'beat': False,\n",
       "  'hands': False,\n",
       "  'crown': False,\n",
       "  '1997': False,\n",
       "  'piece': False,\n",
       "  'animation': False,\n",
       "  'year': False,\n",
       "  'contest': False,\n",
       "  'arrival': False,\n",
       "  'magic': False,\n",
       "  'kingdom': False,\n",
       "  'mediocre': False,\n",
       "  '--': True,\n",
       "  'd': True,\n",
       "  'pocahontas': False,\n",
       "  'those': False,\n",
       "  'keeping': False,\n",
       "  'score': False,\n",
       "  'nearly': False,\n",
       "  'dull': False,\n",
       "  'revolves': False,\n",
       "  'adventures': False,\n",
       "  'free': False,\n",
       "  'spirited': False,\n",
       "  'kayley': False,\n",
       "  'voiced': False,\n",
       "  'jessalyn': False,\n",
       "  'gilsig': False,\n",
       "  'early': False,\n",
       "  'daughter': False,\n",
       "  'belated': False,\n",
       "  'knight': False,\n",
       "  'king': False,\n",
       "  'arthur': False,\n",
       "  'round': False,\n",
       "  'table': False,\n",
       "  'dream': False,\n",
       "  'follow': False,\n",
       "  'father': False,\n",
       "  'footsteps': False,\n",
       "  'she': False,\n",
       "  'chance': False,\n",
       "  'evil': False,\n",
       "  'warlord': False,\n",
       "  'ruber': False,\n",
       "  'gary': False,\n",
       "  'oldman': False,\n",
       "  'ex': False,\n",
       "  'gone': False,\n",
       "  'steals': False,\n",
       "  'magical': False,\n",
       "  'sword': True,\n",
       "  'excalibur': False,\n",
       "  'accidentally': False,\n",
       "  'loses': False,\n",
       "  'dangerous': False,\n",
       "  'booby': False,\n",
       "  'trapped': False,\n",
       "  'forest': False,\n",
       "  'help': False,\n",
       "  'hunky': False,\n",
       "  'blind': False,\n",
       "  'timberland': False,\n",
       "  'dweller': False,\n",
       "  'garrett': False,\n",
       "  'carey': False,\n",
       "  'elwes': False,\n",
       "  'headed': False,\n",
       "  'dragon': False,\n",
       "  'eric': False,\n",
       "  'idle': False,\n",
       "  'rickles': False,\n",
       "  'arguing': False,\n",
       "  'itself': False,\n",
       "  'able': True,\n",
       "  'medieval': False,\n",
       "  'sexist': False,\n",
       "  'prove': False,\n",
       "  'fighter': False,\n",
       "  'side': False,\n",
       "  'pure': False,\n",
       "  'showmanship': False,\n",
       "  'essential': False,\n",
       "  'element': False,\n",
       "  'expected': False,\n",
       "  'climb': False,\n",
       "  'high': True,\n",
       "  'ranks': False,\n",
       "  'differentiates': False,\n",
       "  'something': True,\n",
       "  'saturday': False,\n",
       "  'morning': False,\n",
       "  'subpar': False,\n",
       "  'instantly': False,\n",
       "  'forgettable': False,\n",
       "  'songs': False,\n",
       "  'integrated': False,\n",
       "  'computerized': False,\n",
       "  'footage': False,\n",
       "  'compare': False,\n",
       "  'run': True,\n",
       "  'angry': False,\n",
       "  'ogre': False,\n",
       "  'herc': False,\n",
       "  'battle': False,\n",
       "  'hydra': False,\n",
       "  'rest': False,\n",
       "  'case': True,\n",
       "  'stink': False,\n",
       "  'none': False,\n",
       "  'remotely': False,\n",
       "  'interesting': False,\n",
       "  'race': False,\n",
       "  'bland': False,\n",
       "  'end': False,\n",
       "  'tie': False,\n",
       "  'win': False,\n",
       "  'comedy': False,\n",
       "  'shtick': False,\n",
       "  'awfully': False,\n",
       "  'cloying': False,\n",
       "  'least': False,\n",
       "  'signs': False,\n",
       "  'pulse': False,\n",
       "  'fans': False,\n",
       "  \"-'\": False,\n",
       "  '90s': False,\n",
       "  'tgif': False,\n",
       "  'will': False,\n",
       "  'thrilled': False,\n",
       "  'jaleel': False,\n",
       "  'urkel': False,\n",
       "  'white': False,\n",
       "  'bronson': False,\n",
       "  'balki': False,\n",
       "  'pinchot': False,\n",
       "  'sharing': False,\n",
       "  'nicely': False,\n",
       "  'realized': False,\n",
       "  'though': False,\n",
       "  'm': False,\n",
       "  'loss': False,\n",
       "  'recall': False,\n",
       "  'specific': False,\n",
       "  'providing': False,\n",
       "  'voice': False,\n",
       "  'talent': False,\n",
       "  'enthusiastic': False,\n",
       "  'paired': False,\n",
       "  'singers': False,\n",
       "  'sound': False,\n",
       "  'musical': False,\n",
       "  'moments': False,\n",
       "  'jane': False,\n",
       "  'seymour': False,\n",
       "  'celine': False,\n",
       "  'dion': False,\n",
       "  'must': False,\n",
       "  'strain': False,\n",
       "  'through': False,\n",
       "  'aside': False,\n",
       "  'children': False,\n",
       "  'probably': False,\n",
       "  'adults': False,\n",
       "  'grievous': False,\n",
       "  'error': False,\n",
       "  'lack': False,\n",
       "  'personality': False,\n",
       "  'learn': False,\n",
       "  'goes': False,\n",
       "  'synopsis': False,\n",
       "  'mentally': False,\n",
       "  'unstable': False,\n",
       "  'undergoing': False,\n",
       "  'psychotherapy': False,\n",
       "  'saves': False,\n",
       "  'boy': False,\n",
       "  'potentially': False,\n",
       "  'fatal': False,\n",
       "  'falls': False,\n",
       "  'love': False,\n",
       "  'mother': False,\n",
       "  'fledgling': False,\n",
       "  'restauranteur': False,\n",
       "  'unsuccessfully': False,\n",
       "  'attempting': False,\n",
       "  'gain': False,\n",
       "  'woman': False,\n",
       "  'favor': False,\n",
       "  'takes': False,\n",
       "  'pictures': False,\n",
       "  'kills': False,\n",
       "  'comments': False,\n",
       "  'stalked': False,\n",
       "  'yet': True,\n",
       "  'seemingly': False,\n",
       "  'endless': False,\n",
       "  'string': False,\n",
       "  'spurned': False,\n",
       "  'psychos': False,\n",
       "  'getting': True,\n",
       "  'revenge': False,\n",
       "  'type': False,\n",
       "  'stable': False,\n",
       "  'category': False,\n",
       "  '1990s': False,\n",
       "  'industry': False,\n",
       "  'theatrical': False,\n",
       "  'direct': False,\n",
       "  'proliferation': False,\n",
       "  'may': False,\n",
       "  'due': False,\n",
       "  'typically': False,\n",
       "  'inexpensive': False,\n",
       "  'produce': False,\n",
       "  'special': False,\n",
       "  'effects': False,\n",
       "  'stars': False,\n",
       "  'serve': False,\n",
       "  'vehicles': False,\n",
       "  'nudity': False,\n",
       "  'allowing': False,\n",
       "  'frequent': False,\n",
       "  'night': False,\n",
       "  'cable': False,\n",
       "  'wavers': False,\n",
       "  'slightly': False,\n",
       "  'norm': False,\n",
       "  'respect': False,\n",
       "  'psycho': False,\n",
       "  'never': False,\n",
       "  'affair': False,\n",
       "  ';': False,\n",
       "  'contrary': False,\n",
       "  'rejected': False,\n",
       "  'rather': False,\n",
       "  'lover': False,\n",
       "  'wife': False,\n",
       "  'husband': False,\n",
       "  'entry': False,\n",
       "  'doomed': False,\n",
       "  'collect': False,\n",
       "  'dust': False,\n",
       "  'viewed': False,\n",
       "  'midnight': False,\n",
       "  'provide': False,\n",
       "  'suspense': False,\n",
       "  'sets': False,\n",
       "  'interspersed': False,\n",
       "  'opening': False,\n",
       "  'credits': False,\n",
       "  'instance': False,\n",
       "  'serious': False,\n",
       "  'sounding': False,\n",
       "  'narrator': False,\n",
       "  'spouts': False,\n",
       "  'statistics': False,\n",
       "  'stalkers': False,\n",
       "  'ponders': False,\n",
       "  'cause': False,\n",
       "  'stalk': False,\n",
       "  'implicitly': False,\n",
       "  'implied': False,\n",
       "  'men': False,\n",
       "  'shown': False,\n",
       "  'snapshot': False,\n",
       "  'actor': False,\n",
       "  'jay': False,\n",
       "  'underwood': False,\n",
       "  'states': False,\n",
       "  'daryl': False,\n",
       "  'gleason': False,\n",
       "  'stalker': False,\n",
       "  'brooke': False,\n",
       "  'daniels': False,\n",
       "  'meant': False,\n",
       "  'called': False,\n",
       "  'guesswork': False,\n",
       "  'required': False,\n",
       "  'proceeds': False,\n",
       "  'begins': False,\n",
       "  'obvious': False,\n",
       "  'sequence': False,\n",
       "  'contrived': False,\n",
       "  'quite': False,\n",
       "  'brings': False,\n",
       "  'victim': False,\n",
       "  'together': False,\n",
       "  'obsesses': False,\n",
       "  'follows': False,\n",
       "  'tries': False,\n",
       "  'woo': True,\n",
       "  'plans': False,\n",
       "  'become': False,\n",
       "  'desperate': False,\n",
       "  'elaborate': False,\n",
       "  'include': False,\n",
       "  'cliche': False,\n",
       "  'murdered': False,\n",
       "  'pet': False,\n",
       "  'require': False,\n",
       "  'found': False,\n",
       "  'exception': False,\n",
       "  'cat': False,\n",
       "  'shower': False,\n",
       "  'events': False,\n",
       "  'lead': False,\n",
       "  'inevitable': False,\n",
       "  'showdown': False,\n",
       "  'survives': False,\n",
       "  'invariably': False,\n",
       "  'conclusion': False,\n",
       "  'turkey': False,\n",
       "  'uniformly': False,\n",
       "  'adequate': False,\n",
       "  'anything': True,\n",
       "  'home': True,\n",
       "  'either': False,\n",
       "  'turns': False,\n",
       "  'toward': False,\n",
       "  'melodrama': False,\n",
       "  'overdoes': False,\n",
       "  'words': False,\n",
       "  'manages': False,\n",
       "  'creepy': False,\n",
       "  'pass': False,\n",
       "  'demands': False,\n",
       "  'maryam': False,\n",
       "  'abo': False,\n",
       "  'close': False,\n",
       "  'played': False,\n",
       "  'bond': False,\n",
       "  'chick': False,\n",
       "  'living': False,\n",
       "  'daylights': False,\n",
       "  'equally': False,\n",
       "  'title': False,\n",
       "  'ditzy': False,\n",
       "  'strong': True,\n",
       "  'independent': False,\n",
       "  'business': False,\n",
       "  'owner': False,\n",
       "  'needs': False,\n",
       "  'proceed': False,\n",
       "  'example': False,\n",
       "  'suspicions': False,\n",
       "  'ensure': False,\n",
       "  'use': False,\n",
       "  'excuse': False,\n",
       "  'decides': False,\n",
       "  'return': False,\n",
       "  'toolbox': False,\n",
       "  'left': False,\n",
       "  'place': True,\n",
       "  ...},\n",
       " 'pos')"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "featuresets[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Naive Bayes Classifier with NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set that we'll train our classifier with\n",
    "training_set = featuresets[:1900]\n",
    "\n",
    "# set that we'll test against.\n",
    "testing_set = featuresets[1900:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = nltk.NaiveBayesClassifier.train(training_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier accuracy percent: 81.0\n"
     ]
    }
   ],
   "source": [
    "print(\"Classifier accuracy percent:\",(nltk.classify.accuracy(classifier, testing_set))*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "                   sucks = True              neg : pos    =     10.7 : 1.0\n",
      "                  justin = True              neg : pos    =      9.1 : 1.0\n",
      "                  annual = True              pos : neg    =      8.9 : 1.0\n",
      "                  turkey = True              neg : pos    =      8.5 : 1.0\n",
      "           unimaginative = True              neg : pos    =      8.4 : 1.0\n",
      "                 frances = True              pos : neg    =      8.3 : 1.0\n",
      "                 idiotic = True              neg : pos    =      7.3 : 1.0\n",
      "               atrocious = True              neg : pos    =      7.1 : 1.0\n",
      "                  shoddy = True              neg : pos    =      7.1 : 1.0\n",
      "                  regard = True              pos : neg    =      6.9 : 1.0\n",
      "                 kidding = True              neg : pos    =      6.4 : 1.0\n",
      "                    mena = True              neg : pos    =      6.4 : 1.0\n",
      "                  suvari = True              neg : pos    =      6.4 : 1.0\n",
      "                 singers = True              pos : neg    =      6.3 : 1.0\n",
      "                obstacle = True              pos : neg    =      6.3 : 1.0\n"
     ]
    }
   ],
   "source": [
    "classifier.show_most_informative_features(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "save_classifier = open(\"naivebayes.pickle\",\"wb\")\n",
    "pickle.dump(classifier, save_classifier)\n",
    "save_classifier.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier_f = open(\"naivebayes.pickle\", \"rb\")\n",
    "classifier = pickle.load(classifier_f)\n",
    "classifier_f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scikit-Learn Sklearn with NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.classify.scikitlearn import SklearnClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB,BernoulliNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MultinomialNB accuracy percent: 81.0\n",
      "BernoulliNB accuracy percent: 81.0\n"
     ]
    }
   ],
   "source": [
    "MNB_classifier = SklearnClassifier(MultinomialNB())\n",
    "MNB_classifier.train(training_set)\n",
    "print(\"MultinomialNB accuracy percent:\",nltk.classify.accuracy(MNB_classifier, testing_set)*100)\n",
    "\n",
    "BNB_classifier = SklearnClassifier(BernoulliNB())\n",
    "BNB_classifier.train(training_set)\n",
    "print(\"BernoulliNB accuracy percent:\",nltk.classify.accuracy(BNB_classifier, testing_set)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression,SGDClassifier\n",
    "from sklearn.svm import SVC, LinearSVC, NuSVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Naive Bayes Algo accuracy percent: 81.0\n",
      "Most Informative Features\n",
      "                   sucks = True              neg : pos    =     10.7 : 1.0\n",
      "                  justin = True              neg : pos    =      9.1 : 1.0\n",
      "                  annual = True              pos : neg    =      8.9 : 1.0\n",
      "                  turkey = True              neg : pos    =      8.5 : 1.0\n",
      "           unimaginative = True              neg : pos    =      8.4 : 1.0\n",
      "                 frances = True              pos : neg    =      8.3 : 1.0\n",
      "                 idiotic = True              neg : pos    =      7.3 : 1.0\n",
      "               atrocious = True              neg : pos    =      7.1 : 1.0\n",
      "                  shoddy = True              neg : pos    =      7.1 : 1.0\n",
      "                  regard = True              pos : neg    =      6.9 : 1.0\n",
      "                 kidding = True              neg : pos    =      6.4 : 1.0\n",
      "                    mena = True              neg : pos    =      6.4 : 1.0\n",
      "                  suvari = True              neg : pos    =      6.4 : 1.0\n",
      "                 singers = True              pos : neg    =      6.3 : 1.0\n",
      "                obstacle = True              pos : neg    =      6.3 : 1.0\n",
      "MNB_classifier accuracy percent: 81.0\n",
      "BernoulliNB_classifier accuracy percent: 81.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thamm\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression_classifier accuracy percent: 77.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thamm\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SGDClassifier_classifier accuracy percent: 78.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thamm\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVC_classifier accuracy percent: 76.0\n",
      "LinearSVC_classifier accuracy percent: 74.0\n",
      "NuSVC_classifier accuracy percent: 82.0\n"
     ]
    }
   ],
   "source": [
    "print(\"Original Naive Bayes Algo accuracy percent:\", (nltk.classify.accuracy(classifier, testing_set))*100)\n",
    "classifier.show_most_informative_features(15)\n",
    "\n",
    "MNB_classifier = SklearnClassifier(MultinomialNB())\n",
    "MNB_classifier.train(training_set)\n",
    "print(\"MNB_classifier accuracy percent:\", (nltk.classify.accuracy(MNB_classifier, testing_set))*100)\n",
    "\n",
    "BernoulliNB_classifier = SklearnClassifier(BernoulliNB())\n",
    "BernoulliNB_classifier.train(training_set)\n",
    "print(\"BernoulliNB_classifier accuracy percent:\", (nltk.classify.accuracy(BernoulliNB_classifier, testing_set))*100)\n",
    "\n",
    "LogisticRegression_classifier = SklearnClassifier(LogisticRegression())\n",
    "LogisticRegression_classifier.train(training_set)\n",
    "print(\"LogisticRegression_classifier accuracy percent:\", (nltk.classify.accuracy(LogisticRegression_classifier, testing_set))*100)\n",
    "\n",
    "SGDClassifier_classifier = SklearnClassifier(SGDClassifier())\n",
    "SGDClassifier_classifier.train(training_set)\n",
    "print(\"SGDClassifier_classifier accuracy percent:\", (nltk.classify.accuracy(SGDClassifier_classifier, testing_set))*100)\n",
    "\n",
    "SVC_classifier = SklearnClassifier(SVC())\n",
    "SVC_classifier.train(training_set)\n",
    "print(\"SVC_classifier accuracy percent:\", (nltk.classify.accuracy(SVC_classifier, testing_set))*100)\n",
    "\n",
    "LinearSVC_classifier = SklearnClassifier(LinearSVC())\n",
    "LinearSVC_classifier.train(training_set)\n",
    "print(\"LinearSVC_classifier accuracy percent:\", (nltk.classify.accuracy(LinearSVC_classifier, testing_set))*100)\n",
    "\n",
    "NuSVC_classifier = SklearnClassifier(NuSVC())\n",
    "NuSVC_classifier.train(training_set)\n",
    "print(\"NuSVC_classifier accuracy percent:\", (nltk.classify.accuracy(NuSVC_classifier, testing_set))*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combining Algorithms with NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.classify import ClassifierI\n",
    "from statistics import mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VoteClassifier(ClassifierI):\n",
    "    def __init__(self, *classifiers):\n",
    "        self._classifiers = classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classify(self, features):\n",
    "    votes = []\n",
    "    for c in self._classifiers:\n",
    "        v = c.classify(features)\n",
    "        votes.append(v)\n",
    "    return mode(votes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def confidence(self, features):\n",
    "    votes = []\n",
    "    for c in self._classifiers:\n",
    "        v = c.classify(features)\n",
    "        votes.append(v)\n",
    "\n",
    "    choice_votes = votes.count(mode(votes))\n",
    "    conf = choice_votes / len(votes)\n",
    "    return conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Naive Bayes Algo accuracy percent: 90.0\n",
      "Most Informative Features\n",
      "                   sucks = True              neg : pos    =     10.7 : 1.0\n",
      "                  justin = True              neg : pos    =      9.1 : 1.0\n",
      "                  annual = True              pos : neg    =      8.9 : 1.0\n",
      "                  turkey = True              neg : pos    =      8.5 : 1.0\n",
      "           unimaginative = True              neg : pos    =      8.4 : 1.0\n",
      "                 frances = True              pos : neg    =      8.3 : 1.0\n",
      "                 idiotic = True              neg : pos    =      7.3 : 1.0\n",
      "               atrocious = True              neg : pos    =      7.1 : 1.0\n",
      "                  shoddy = True              neg : pos    =      7.1 : 1.0\n",
      "                  regard = True              pos : neg    =      6.9 : 1.0\n",
      "                 kidding = True              neg : pos    =      6.4 : 1.0\n",
      "                    mena = True              neg : pos    =      6.4 : 1.0\n",
      "                  suvari = True              neg : pos    =      6.4 : 1.0\n",
      "                 singers = True              pos : neg    =      6.3 : 1.0\n",
      "                obstacle = True              pos : neg    =      6.3 : 1.0\n",
      "MNB_classifier accuracy percent: 84.0\n",
      "BernoulliNB_classifier accuracy percent: 78.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thamm\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression_classifier accuracy percent: 83.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thamm\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SGDClassifier_classifier accuracy percent: 82.0\n",
      "LinearSVC_classifier accuracy percent: 81.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thamm\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NuSVC_classifier accuracy percent: 82.0\n",
      "voted_classifier accuracy percent: 82.0\n",
      "Classification: neg Confidence %: 100.0\n",
      "Classification: pos Confidence %: 71.42857142857143\n",
      "Classification: neg Confidence %: 100.0\n",
      "Classification: neg Confidence %: 57.14285714285714\n",
      "Classification: neg Confidence %: 100.0\n",
      "Classification: neg Confidence %: 100.0\n"
     ]
    }
   ],
   "source": [
    "#Complete module\n",
    "\n",
    "import nltk\n",
    "import random\n",
    "from nltk.corpus import movie_reviews\n",
    "from nltk.classify.scikitlearn import SklearnClassifier\n",
    "import pickle\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB, BernoulliNB\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.svm import SVC, LinearSVC, NuSVC\n",
    "\n",
    "from nltk.classify import ClassifierI\n",
    "from statistics import mode\n",
    "\n",
    "\n",
    "class VoteClassifier(ClassifierI):\n",
    "    def __init__(self, *classifiers):\n",
    "        self._classifiers = classifiers\n",
    "\n",
    "    def classify(self, features):\n",
    "        votes = []\n",
    "        for c in self._classifiers:\n",
    "            v = c.classify(features)\n",
    "            votes.append(v)\n",
    "        return mode(votes)\n",
    "\n",
    "    def confidence(self, features):\n",
    "        votes = []\n",
    "        for c in self._classifiers:\n",
    "            v = c.classify(features)\n",
    "            votes.append(v)\n",
    "\n",
    "        choice_votes = votes.count(mode(votes))\n",
    "        conf = choice_votes / len(votes)\n",
    "        return conf\n",
    "\n",
    "documents = [(list(movie_reviews.words(fileid)), category)\n",
    "             for category in movie_reviews.categories()\n",
    "             for fileid in movie_reviews.fileids(category)]\n",
    "\n",
    "random.shuffle(documents)\n",
    "\n",
    "all_words = []\n",
    "\n",
    "for w in movie_reviews.words():\n",
    "    all_words.append(w.lower())\n",
    "\n",
    "all_words = nltk.FreqDist(all_words)\n",
    "\n",
    "word_features = list(all_words.keys())[:3000]\n",
    "\n",
    "def find_features(document):\n",
    "    words = set(document)\n",
    "    features = {}\n",
    "    for w in word_features:\n",
    "        features[w] = (w in words)\n",
    "\n",
    "    return features\n",
    "\n",
    "#print((find_features(movie_reviews.words('neg/cv000_29416.txt'))))\n",
    "\n",
    "featuresets = [(find_features(rev), category) for (rev, category) in documents]\n",
    "        \n",
    "training_set = featuresets[:1900]\n",
    "testing_set =  featuresets[1900:]\n",
    "\n",
    "#classifier = nltk.NaiveBayesClassifier.train(training_set)\n",
    "\n",
    "classifier_f = open(\"naivebayes.pickle\",\"rb\")\n",
    "classifier = pickle.load(classifier_f)\n",
    "classifier_f.close()\n",
    "\n",
    "print(\"Original Naive Bayes Algo accuracy percent:\", (nltk.classify.accuracy(classifier, testing_set))*100)\n",
    "classifier.show_most_informative_features(15)\n",
    "\n",
    "MNB_classifier = SklearnClassifier(MultinomialNB())\n",
    "MNB_classifier.train(training_set)\n",
    "print(\"MNB_classifier accuracy percent:\", (nltk.classify.accuracy(MNB_classifier, testing_set))*100)\n",
    "\n",
    "BernoulliNB_classifier = SklearnClassifier(BernoulliNB())\n",
    "BernoulliNB_classifier.train(training_set)\n",
    "print(\"BernoulliNB_classifier accuracy percent:\", (nltk.classify.accuracy(BernoulliNB_classifier, testing_set))*100)\n",
    "\n",
    "LogisticRegression_classifier = SklearnClassifier(LogisticRegression())\n",
    "LogisticRegression_classifier.train(training_set)\n",
    "print(\"LogisticRegression_classifier accuracy percent:\", (nltk.classify.accuracy(LogisticRegression_classifier, testing_set))*100)\n",
    "\n",
    "SGDClassifier_classifier = SklearnClassifier(SGDClassifier())\n",
    "SGDClassifier_classifier.train(training_set)\n",
    "print(\"SGDClassifier_classifier accuracy percent:\", (nltk.classify.accuracy(SGDClassifier_classifier, testing_set))*100)\n",
    "\n",
    "##SVC_classifier = SklearnClassifier(SVC())\n",
    "##SVC_classifier.train(training_set)\n",
    "##print(\"SVC_classifier accuracy percent:\", (nltk.classify.accuracy(SVC_classifier, testing_set))*100)\n",
    "\n",
    "LinearSVC_classifier = SklearnClassifier(LinearSVC())\n",
    "LinearSVC_classifier.train(training_set)\n",
    "print(\"LinearSVC_classifier accuracy percent:\", (nltk.classify.accuracy(LinearSVC_classifier, testing_set))*100)\n",
    "\n",
    "NuSVC_classifier = SklearnClassifier(NuSVC())\n",
    "NuSVC_classifier.train(training_set)\n",
    "print(\"NuSVC_classifier accuracy percent:\", (nltk.classify.accuracy(NuSVC_classifier, testing_set))*100)\n",
    "\n",
    "\n",
    "voted_classifier = VoteClassifier(classifier,\n",
    "                                  NuSVC_classifier,\n",
    "                                  LinearSVC_classifier,\n",
    "                                  SGDClassifier_classifier,\n",
    "                                  MNB_classifier,\n",
    "                                  BernoulliNB_classifier,\n",
    "                                  LogisticRegression_classifier)\n",
    "\n",
    "print(\"voted_classifier accuracy percent:\", (nltk.classify.accuracy(voted_classifier, testing_set))*100)\n",
    "\n",
    "print(\"Classification:\", voted_classifier.classify(testing_set[0][0]), \"Confidence %:\",voted_classifier.confidence(testing_set[0][0])*100)\n",
    "print(\"Classification:\", voted_classifier.classify(testing_set[1][0]), \"Confidence %:\",voted_classifier.confidence(testing_set[1][0])*100)\n",
    "print(\"Classification:\", voted_classifier.classify(testing_set[2][0]), \"Confidence %:\",voted_classifier.confidence(testing_set[2][0])*100)\n",
    "print(\"Classification:\", voted_classifier.classify(testing_set[3][0]), \"Confidence %:\",voted_classifier.confidence(testing_set[3][0])*100)\n",
    "print(\"Classification:\", voted_classifier.classify(testing_set[4][0]), \"Confidence %:\",voted_classifier.confidence(testing_set[4][0])*100)\n",
    "print(\"Classification:\", voted_classifier.classify(testing_set[5][0]), \"Confidence %:\",voted_classifier.confidence(testing_set[5][0])*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Improving Training Data for sentiment analysis with NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "short_pos = open(\"dataset/positive.txt\",\"r\").read()\n",
    "short_neg = open(\"dataset/negative.txt\",\"r\").read()\n",
    "\n",
    "documents = []\n",
    "\n",
    "for r in short_pos.split('\\n'):\n",
    "    documents.append( (r, \"pos\") )\n",
    "\n",
    "for r in short_neg.split('\\n'):\n",
    "    documents.append( (r, \"neg\") )\n",
    "\n",
    "\n",
    "all_words = []\n",
    "\n",
    "short_pos_words = word_tokenize(short_pos)\n",
    "short_neg_words = word_tokenize(short_neg)\n",
    "\n",
    "for w in short_pos_words:\n",
    "    all_words.append(w.lower())\n",
    "\n",
    "for w in short_neg_words:\n",
    "    all_words.append(w.lower())\n",
    "\n",
    "all_words = nltk.FreqDist(all_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_features = list(all_words.keys())[:5000]\n",
    "\n",
    "def find_features(document):\n",
    "    words = word_tokenize(document)\n",
    "    features = {}\n",
    "    for w in word_features:\n",
    "        features[w] = (w in words)\n",
    "\n",
    "    return features\n",
    "\t\n",
    "featuresets = [(find_features(rev), category) for (rev, category) in documents]\n",
    "random.shuffle(featuresets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original Naive Bayes Algo accuracy percent: 0\n",
      "Most Informative Features\n",
      "                   fails = True              neg : pos    =      8.4 : 1.0\n",
      "                    warm = True              pos : neg    =      8.3 : 1.0\n",
      "                   tries = True              neg : pos    =      7.7 : 1.0\n",
      "                   bland = True              neg : pos    =      7.7 : 1.0\n",
      "                     bad = True              neg : pos    =      7.2 : 1.0\n",
      "            performances = True              pos : neg    =      7.1 : 1.0\n",
      "                   title = True              neg : pos    =      6.4 : 1.0\n",
      "               beautiful = True              pos : neg    =      6.3 : 1.0\n",
      "                       s = True              pos : neg    =      6.3 : 1.0\n",
      "               hilarious = True              pos : neg    =      6.3 : 1.0\n",
      "                touching = True              pos : neg    =      6.3 : 1.0\n",
      "                     guy = True              neg : pos    =      5.7 : 1.0\n",
      "                 problem = True              neg : pos    =      5.7 : 1.0\n",
      "                    mess = True              neg : pos    =      5.7 : 1.0\n",
      "                portrait = True              pos : neg    =      5.6 : 1.0\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Sample sequence X is empty.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-71-734afe3a1e31>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     93\u001b[0m \u001b[0mMNB_classifier\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSklearnClassifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mMultinomialNB\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[0mMNB_classifier\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtraining_set\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 95\u001b[1;33m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"MNB_classifier accuracy percent:\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mnltk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclassify\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maccuracy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mMNB_classifier\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtesting_set\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;36m100\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     96\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     97\u001b[0m \u001b[0mBernoulliNB_classifier\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSklearnClassifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBernoulliNB\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\nltk\\classify\\util.py\u001b[0m in \u001b[0;36maccuracy\u001b[1;34m(classifier, gold)\u001b[0m\n\u001b[0;32m     90\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0maccuracy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclassifier\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgold\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 92\u001b[1;33m     \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclassifier\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclassify_many\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mfs\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mfs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mgold\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     93\u001b[0m     \u001b[0mcorrect\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0ml\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mr\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mr\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgold\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresults\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcorrect\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\nltk\\classify\\scikitlearn.py\u001b[0m in \u001b[0;36mclassify_many\u001b[1;34m(self, featuresets)\u001b[0m\n\u001b[0;32m     83\u001b[0m         \u001b[1;33m:\u001b[0m\u001b[0mrtype\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     84\u001b[0m         \"\"\"\n\u001b[1;32m---> 85\u001b[1;33m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_vectorizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeaturesets\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     86\u001b[0m         \u001b[0mclasses\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_encoder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     87\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mclasses\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_clf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\feature_extraction\\dict_vectorizer.py\u001b[0m in \u001b[0;36mtransform\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    290\u001b[0m         \"\"\"\n\u001b[0;32m    291\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msparse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 292\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfitting\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    293\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    294\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\feature_extraction\\dict_vectorizer.py\u001b[0m in \u001b[0;36m_transform\u001b[1;34m(self, X, fitting)\u001b[0m\n\u001b[0;32m    182\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    183\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindptr\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 184\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Sample sequence X is empty.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    185\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    186\u001b[0m         \u001b[0mindices\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrombuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindices\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mintc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Sample sequence X is empty."
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "import random\n",
    "from nltk.corpus import movie_reviews\n",
    "from nltk.classify.scikitlearn import SklearnClassifier\n",
    "import pickle\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB, BernoulliNB\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.svm import SVC, LinearSVC, NuSVC\n",
    "\n",
    "from nltk.classify import ClassifierI\n",
    "from statistics import mode\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "\n",
    "class VoteClassifier(ClassifierI):\n",
    "    def __init__(self, *classifiers):\n",
    "        self._classifiers = classifiers\n",
    "\n",
    "    def classify(self, features):\n",
    "        votes = []\n",
    "        for c in self._classifiers:\n",
    "            v = c.classify(features)\n",
    "            votes.append(v)\n",
    "        return mode(votes)\n",
    "\n",
    "    def confidence(self, features):\n",
    "        votes = []\n",
    "        for c in self._classifiers:\n",
    "            v = c.classify(features)\n",
    "            votes.append(v)\n",
    "\n",
    "        choice_votes = votes.count(mode(votes))\n",
    "        conf = choice_votes / len(votes)\n",
    "        return conf\n",
    "        \n",
    "short_pos = open(\"dataset/positive.txt\",\"r\").read()\n",
    "short_neg = open(\"dataset/negative.txt\",\"r\").read()\n",
    "\n",
    "documents = []\n",
    "\n",
    "for r in short_pos.split('\\n'):\n",
    "    documents.append( (r, \"pos\") )\n",
    "\n",
    "for r in short_neg.split('\\n'):\n",
    "    documents.append( (r, \"neg\") )\n",
    "\n",
    "\n",
    "all_words = []\n",
    "\n",
    "short_pos_words = word_tokenize(short_pos)\n",
    "short_neg_words = word_tokenize(short_neg)\n",
    "\n",
    "for w in short_pos_words:\n",
    "    all_words.append(w.lower())\n",
    "\n",
    "for w in short_neg_words:\n",
    "    all_words.append(w.lower())\n",
    "\n",
    "all_words = nltk.FreqDist(all_words)\n",
    "\n",
    "word_features = list(all_words.keys())[:5000]\n",
    "\n",
    "def find_features(document):\n",
    "    words = word_tokenize(document)\n",
    "    features = {}\n",
    "    for w in word_features:\n",
    "        features[w] = (w in words)\n",
    "\n",
    "    return features\n",
    "\n",
    "#print((find_features(movie_reviews.words('neg/cv000_29416.txt'))))\n",
    "\n",
    "featuresets = [(find_features(rev), category) for (rev, category) in documents]\n",
    "\n",
    "random.shuffle(featuresets)\n",
    "\n",
    "# positive data example:      \n",
    "training_set = featuresets[:10000]\n",
    "testing_set =  featuresets[10000:]\n",
    "\n",
    "##\n",
    "### negative data example:      \n",
    "##training_set = featuresets[100:]\n",
    "##testing_set =  featuresets[:100]\n",
    "\n",
    "\n",
    "classifier = nltk.NaiveBayesClassifier.train(training_set)\n",
    "print(\"Original Naive Bayes Algo accuracy percent:\", (nltk.classify.accuracy(classifier, testing_set))*100)\n",
    "classifier.show_most_informative_features(15)\n",
    "\n",
    "MNB_classifier = SklearnClassifier(MultinomialNB())\n",
    "MNB_classifier.train(training_set)\n",
    "print(\"MNB_classifier accuracy percent:\", (nltk.classify.accuracy(MNB_classifier, testing_set))*100)\n",
    "\n",
    "BernoulliNB_classifier = SklearnClassifier(BernoulliNB())\n",
    "BernoulliNB_classifier.train(training_set)\n",
    "print(\"BernoulliNB_classifier accuracy percent:\", (nltk.classify.accuracy(BernoulliNB_classifier, testing_set))*100)\n",
    "\n",
    "LogisticRegression_classifier = SklearnClassifier(LogisticRegression())\n",
    "LogisticRegression_classifier.train(training_set)\n",
    "print(\"LogisticRegression_classifier accuracy percent:\", (nltk.classify.accuracy(LogisticRegression_classifier, testing_set))*100)\n",
    "\n",
    "SGDClassifier_classifier = SklearnClassifier(SGDClassifier())\n",
    "SGDClassifier_classifier.train(training_set)\n",
    "print(\"SGDClassifier_classifier accuracy percent:\", (nltk.classify.accuracy(SGDClassifier_classifier, testing_set))*100)\n",
    "\n",
    "##SVC_classifier = SklearnClassifier(SVC())\n",
    "##SVC_classifier.train(training_set)\n",
    "##print(\"SVC_classifier accuracy percent:\", (nltk.classify.accuracy(SVC_classifier, testing_set))*100)\n",
    "\n",
    "LinearSVC_classifier = SklearnClassifier(LinearSVC())\n",
    "LinearSVC_classifier.train(training_set)\n",
    "print(\"LinearSVC_classifier accuracy percent:\", (nltk.classify.accuracy(LinearSVC_classifier, testing_set))*100)\n",
    "\n",
    "NuSVC_classifier = SklearnClassifier(NuSVC())\n",
    "NuSVC_classifier.train(training_set)\n",
    "print(\"NuSVC_classifier accuracy percent:\", (nltk.classify.accuracy(NuSVC_classifier, testing_set))*100)\n",
    "\n",
    "\n",
    "voted_classifier = VoteClassifier(\n",
    "                                  NuSVC_classifier,\n",
    "                                  LinearSVC_classifier,\n",
    "                                  MNB_classifier,\n",
    "                                  BernoulliNB_classifier,\n",
    "                                  LogisticRegression_classifier)\n",
    "\n",
    "print(\"voted_classifier accuracy percent:\", (nltk.classify.accuracy(voted_classifier, testing_set))*100)\n",
    "O"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating a module for Sentiment Analysis with NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2442\n",
      "Original Naive Bayes Algo accuracy percent: 0\n",
      "Most Informative Features\n",
      "                    warm = True              pos : neg    =      8.3 : 1.0\n",
      "                     bad = True              neg : pos    =      7.2 : 1.0\n",
      "                    cold = True              neg : pos    =      6.4 : 1.0\n",
      "               beautiful = True              pos : neg    =      6.3 : 1.0\n",
      "                       s = True              pos : neg    =      6.3 : 1.0\n",
      "               hilarious = True              pos : neg    =      6.3 : 1.0\n",
      "                touching = True              pos : neg    =      6.3 : 1.0\n",
      "                     guy = True              neg : pos    =      5.7 : 1.0\n",
      "                   quiet = True              pos : neg    =      5.6 : 1.0\n",
      "                  gentle = True              pos : neg    =      5.6 : 1.0\n",
      "                    fine = True              pos : neg    =      5.4 : 1.0\n",
      "                  stupid = True              neg : pos    =      5.0 : 1.0\n",
      "                straight = True              neg : pos    =      5.0 : 1.0\n",
      "                  worthy = True              pos : neg    =      5.0 : 1.0\n",
      "                    nice = True              pos : neg    =      5.0 : 1.0\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Sample sequence X is empty.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-73-582649ff2daf>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m    109\u001b[0m \u001b[0mMNB_classifier\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSklearnClassifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mMultinomialNB\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    110\u001b[0m \u001b[0mMNB_classifier\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtraining_set\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 111\u001b[1;33m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"MNB_classifier accuracy percent:\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mnltk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclassify\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maccuracy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mMNB_classifier\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtesting_set\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;36m100\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    112\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    113\u001b[0m \u001b[0msave_classifier\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"MNB_classifier5k.pickle\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m\"wb\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\nltk\\classify\\util.py\u001b[0m in \u001b[0;36maccuracy\u001b[1;34m(classifier, gold)\u001b[0m\n\u001b[0;32m     90\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0maccuracy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclassifier\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgold\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 92\u001b[1;33m     \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclassifier\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclassify_many\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mfs\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mfs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mgold\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     93\u001b[0m     \u001b[0mcorrect\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0ml\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mr\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mr\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgold\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresults\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcorrect\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\nltk\\classify\\scikitlearn.py\u001b[0m in \u001b[0;36mclassify_many\u001b[1;34m(self, featuresets)\u001b[0m\n\u001b[0;32m     83\u001b[0m         \u001b[1;33m:\u001b[0m\u001b[0mrtype\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     84\u001b[0m         \"\"\"\n\u001b[1;32m---> 85\u001b[1;33m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_vectorizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeaturesets\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     86\u001b[0m         \u001b[0mclasses\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_encoder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     87\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mclasses\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_clf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\feature_extraction\\dict_vectorizer.py\u001b[0m in \u001b[0;36mtransform\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    290\u001b[0m         \"\"\"\n\u001b[0;32m    291\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msparse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 292\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfitting\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    293\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    294\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\feature_extraction\\dict_vectorizer.py\u001b[0m in \u001b[0;36m_transform\u001b[1;34m(self, X, fitting)\u001b[0m\n\u001b[0;32m    182\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    183\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindptr\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 184\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Sample sequence X is empty.\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    185\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    186\u001b[0m         \u001b[0mindices\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrombuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindices\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mintc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Sample sequence X is empty."
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "import random\n",
    "#from nltk.corpus import movie_reviews\n",
    "from nltk.classify.scikitlearn import SklearnClassifier\n",
    "import pickle\n",
    "from sklearn.naive_bayes import MultinomialNB, BernoulliNB\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.svm import SVC, LinearSVC, NuSVC\n",
    "from nltk.classify import ClassifierI\n",
    "from statistics import mode\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "\n",
    "\n",
    "class VoteClassifier(ClassifierI):\n",
    "    def __init__(self, *classifiers):\n",
    "        self._classifiers = classifiers\n",
    "\n",
    "    def classify(self, features):\n",
    "        votes = []\n",
    "        for c in self._classifiers:\n",
    "            v = c.classify(features)\n",
    "            votes.append(v)\n",
    "        return mode(votes)\n",
    "\n",
    "    def confidence(self, features):\n",
    "        votes = []\n",
    "        for c in self._classifiers:\n",
    "            v = c.classify(features)\n",
    "            votes.append(v)\n",
    "\n",
    "        choice_votes = votes.count(mode(votes))\n",
    "        conf = choice_votes / len(votes)\n",
    "        return conf\n",
    "    \n",
    "short_pos = open(\"dataset/positive.txt\",\"r\").read()\n",
    "short_neg = open(\"dataset/negative.txt\",\"r\").read()\n",
    "\n",
    "# move this up here\n",
    "all_words = []\n",
    "documents = []\n",
    "\n",
    "\n",
    "#  j is adject, r is adverb, and v is verb\n",
    "#allowed_word_types = [\"J\",\"R\",\"V\"]\n",
    "allowed_word_types = [\"J\"]\n",
    "\n",
    "for p in short_pos.split('\\n'):\n",
    "    documents.append( (p, \"pos\") )\n",
    "    words = word_tokenize(p)\n",
    "    pos = nltk.pos_tag(words)\n",
    "    for w in pos:\n",
    "        if w[1][0] in allowed_word_types:\n",
    "            all_words.append(w[0].lower())\n",
    "\n",
    "    \n",
    "for p in short_neg.split('\\n'):\n",
    "    documents.append( (p, \"neg\") )\n",
    "    words = word_tokenize(p)\n",
    "    pos = nltk.pos_tag(words)\n",
    "    for w in pos:\n",
    "        if w[1][0] in allowed_word_types:\n",
    "            all_words.append(w[0].lower())\n",
    "\n",
    "\n",
    "\n",
    "save_documents = open(\"documents.pickle\",\"wb\")\n",
    "pickle.dump(documents, save_documents)\n",
    "save_documents.close()\n",
    "\n",
    "\n",
    "all_words = nltk.FreqDist(all_words)\n",
    "\n",
    "\n",
    "word_features = list(all_words.keys())[:5000]\n",
    "\n",
    "\n",
    "save_word_features = open(\"word_features5k.pickle\",\"wb\")\n",
    "pickle.dump(word_features, save_word_features)\n",
    "save_word_features.close()\n",
    "\n",
    "\n",
    "def find_features(document):\n",
    "    words = word_tokenize(document)\n",
    "    features = {}\n",
    "    for w in word_features:\n",
    "        features[w] = (w in words)\n",
    "\n",
    "    return features\n",
    "\n",
    "featuresets = [(find_features(rev), category) for (rev, category) in documents]\n",
    "\n",
    "random.shuffle(featuresets)\n",
    "print(len(featuresets))\n",
    "\n",
    "testing_set = featuresets[10000:]\n",
    "training_set = featuresets[:10000]\n",
    "\n",
    "\n",
    "classifier = nltk.NaiveBayesClassifier.train(training_set)\n",
    "print(\"Original Naive Bayes Algo accuracy percent:\", (nltk.classify.accuracy(classifier, testing_set))*100)\n",
    "classifier.show_most_informative_features(15)\n",
    "\n",
    "###############\n",
    "save_classifier = open(\"originalnaivebayes5k.pickle\",\"wb\")\n",
    "pickle.dump(classifier, save_classifier)\n",
    "save_classifier.close()\n",
    "\n",
    "MNB_classifier = SklearnClassifier(MultinomialNB())\n",
    "MNB_classifier.train(training_set)\n",
    "print(\"MNB_classifier accuracy percent:\", (nltk.classify.accuracy(MNB_classifier, testing_set))*100)\n",
    "\n",
    "save_classifier = open(\"MNB_classifier5k.pickle\",\"wb\")\n",
    "pickle.dump(MNB_classifier, save_classifier)\n",
    "save_classifier.close()\n",
    "\n",
    "BernoulliNB_classifier = SklearnClassifier(BernoulliNB())\n",
    "BernoulliNB_classifier.train(training_set)\n",
    "print(\"BernoulliNB_classifier accuracy percent:\", (nltk.classify.accuracy(BernoulliNB_classifier, testing_set))*100)\n",
    "\n",
    "save_classifier = open(\"BernoulliNB_classifier5k.pickle\",\"wb\")\n",
    "pickle.dump(BernoulliNB_classifier, save_classifier)\n",
    "save_classifier.close()\n",
    "\n",
    "LogisticRegression_classifier = SklearnClassifier(LogisticRegression())\n",
    "LogisticRegression_classifier.train(training_set)\n",
    "print(\"LogisticRegression_classifier accuracy percent:\", (nltk.classify.accuracy(LogisticRegression_classifier, testing_set))*100)\n",
    "\n",
    "save_classifier = open(\"LogisticRegression_classifier5k.pickle\",\"wb\")\n",
    "pickle.dump(LogisticRegression_classifier, save_classifier)\n",
    "save_classifier.close()\n",
    "\n",
    "\n",
    "LinearSVC_classifier = SklearnClassifier(LinearSVC())\n",
    "LinearSVC_classifier.train(training_set)\n",
    "print(\"LinearSVC_classifier accuracy percent:\", (nltk.classify.accuracy(LinearSVC_classifier, testing_set))*100)\n",
    "\n",
    "save_classifier = open(\"LinearSVC_classifier5k.pickle\",\"wb\")\n",
    "pickle.dump(LinearSVC_classifier, save_classifier)\n",
    "save_classifier.close()\n",
    "\n",
    "\n",
    "##NuSVC_classifier = SklearnClassifier(NuSVC())\n",
    "##NuSVC_classifier.train(training_set)\n",
    "##print(\"NuSVC_classifier accuracy percent:\", (nltk.classify.accuracy(NuSVC_classifier, testing_set))*100)\n",
    "\n",
    "\n",
    "SGDC_classifier = SklearnClassifier(SGDClassifier())\n",
    "SGDC_classifier.train(training_set)\n",
    "print(\"SGDClassifier accuracy percent:\",nltk.classify.accuracy(SGDC_classifier, testing_set)*100)\n",
    "\n",
    "save_classifier = open(\"SGDC_classifier5k.pickle\",\"wb\")\n",
    "pickle.dump(SGDC_classifier, save_classifier)\n",
    "save_classifier.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Now, you just need to run this one time. You can always run it again if you wanted, but now, you are ready to create the sentiment analysis module. Here's the file that we're going to call sentiment_mod.py\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'featuresets.pickle'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-74-2c2f98a5a52f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     60\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     61\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 62\u001b[1;33m \u001b[0mfeaturesets_f\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"featuresets.pickle\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"rb\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     63\u001b[0m \u001b[0mfeaturesets\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeaturesets_f\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     64\u001b[0m \u001b[0mfeaturesets_f\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'featuresets.pickle'"
     ]
    }
   ],
   "source": [
    "\n",
    "#File: sentiment_mod.py\n",
    "\n",
    "import nltk\n",
    "import random\n",
    "#from nltk.corpus import movie_reviews\n",
    "from nltk.classify.scikitlearn import SklearnClassifier\n",
    "import pickle\n",
    "from sklearn.naive_bayes import MultinomialNB, BernoulliNB\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.svm import SVC, LinearSVC, NuSVC\n",
    "from nltk.classify import ClassifierI\n",
    "from statistics import mode\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "\n",
    "\n",
    "class VoteClassifier(ClassifierI):\n",
    "    def __init__(self, *classifiers):\n",
    "        self._classifiers = classifiers\n",
    "\n",
    "    def classify(self, features):\n",
    "        votes = []\n",
    "        for c in self._classifiers:\n",
    "            v = c.classify(features)\n",
    "            votes.append(v)\n",
    "        return mode(votes)\n",
    "\n",
    "    def confidence(self, features):\n",
    "        votes = []\n",
    "        for c in self._classifiers:\n",
    "            v = c.classify(features)\n",
    "            votes.append(v)\n",
    "\n",
    "        choice_votes = votes.count(mode(votes))\n",
    "        conf = choice_votes / len(votes)\n",
    "        return conf\n",
    "\n",
    "\n",
    "documents_f = open(\"documents.pickle\", \"rb\")\n",
    "documents = pickle.load(documents_f)\n",
    "documents_f.close()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "word_features5k_f = open(\"word_features5k.pickle\", \"rb\")\n",
    "word_features = pickle.load(word_features5k_f)\n",
    "word_features5k_f.close()\n",
    "\n",
    "\n",
    "def find_features(document):\n",
    "    words = word_tokenize(document)\n",
    "    features = {}\n",
    "    for w in word_features:\n",
    "        features[w] = (w in words)\n",
    "\n",
    "    return features\n",
    "\n",
    "\n",
    "\n",
    "featuresets_f = open(\"featuresets.pickle\", \"rb\")\n",
    "featuresets = pickle.load(featuresets_f)\n",
    "featuresets_f.close()\n",
    "\n",
    "random.shuffle(featuresets)\n",
    "print(len(featuresets))\n",
    "\n",
    "testing_set = featuresets[10000:]\n",
    "training_set = featuresets[:10000]\n",
    "\n",
    "\n",
    "\n",
    "open_file = open(\"originalnaivebayes5k.pickle\", \"rb\")\n",
    "classifier = pickle.load(open_file)\n",
    "open_file.close()\n",
    "\n",
    "\n",
    "open_file = open(\"MNB_classifier5k.pickle\", \"rb\")\n",
    "MNB_classifier = pickle.load(open_file)\n",
    "open_file.close()\n",
    "\n",
    "\n",
    "\n",
    "open_file = open(\"BernoulliNB_classifier5k.pickle\", \"rb\")\n",
    "BernoulliNB_classifier = pickle.load(open_file)\n",
    "open_file.close()\n",
    "\n",
    "\n",
    "open_file = open(\"LogisticRegression_classifier5k.pickle\", \"rb\")\n",
    "LogisticRegression_classifier = pickle.load(open_file)\n",
    "open_file.close()\n",
    "\n",
    "\n",
    "open_file = open(\"LinearSVC_classifier5k.pickle\", \"rb\")\n",
    "LinearSVC_classifier = pickle.load(open_file)\n",
    "open_file.close()\n",
    "\n",
    "\n",
    "open_file = open(\"SGDC_classifier5k.pickle\", \"rb\")\n",
    "SGDC_classifier = pickle.load(open_file)\n",
    "open_file.close()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "voted_classifier = VoteClassifier(\n",
    "                                  classifier,\n",
    "                                  LinearSVC_classifier,\n",
    "                                  MNB_classifier,\n",
    "                                  BernoulliNB_classifier,\n",
    "                                  LogisticRegression_classifier)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def sentiment(text):\n",
    "    feats = find_features(text)\n",
    "    return voted_classifier.classify(feats),voted_classifier.confidence(feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentiment_mod as s\n",
    "\n",
    "print(s.sentiment(\"This movie was awesome! The acting was great, plot was wonderful, and there were pythons...so yea!\"))\n",
    "print(s.sentiment(\"This movie was utter junk. There were absolutely 0 pythons. I don't see what the point was at all. Horrible movie, 0/10\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Twitter Sentiment Analysis with NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tweepy import Stream\n",
    "from tweepy import OAuthHandler\n",
    "from tweepy.streaming import StreamListener\n",
    "\n",
    "\n",
    "#consumer key, consumer secret, access token, access secret.\n",
    "ckey=\"fsdfasdfsafsffa\"\n",
    "csecret=\"asdfsadfsadfsadf\"\n",
    "atoken=\"asdf-aassdfs\"\n",
    "asecret=\"asdfsadfsdafsdafs\"\n",
    "\n",
    "class listener(StreamListener):\n",
    "\n",
    "    def on_data(self, data):\n",
    "        print(data)\n",
    "        return(True)\n",
    "\n",
    "    def on_error(self, status):\n",
    "        print status\n",
    "\n",
    "auth = OAuthHandler(ckey, csecret)\n",
    "auth.set_access_token(atoken, asecret)\n",
    "\n",
    "twitterStream = Stream(auth, listener())\n",
    "twitterStream.filter(track=[\"car\"])\n",
    "\n",
    "tweet = all_data[\"text\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tweepy import Stream\n",
    "from tweepy import OAuthHandler\n",
    "from tweepy.streaming import StreamListener\n",
    "import json\n",
    "import sentiment_mod as s\n",
    "\n",
    "#consumer key, consumer secret, access token, access secret.\n",
    "ckey=\"asdfsafsafsaf\"\n",
    "csecret=\"asdfasdfsadfsa\"\n",
    "atoken=\"asdfsadfsafsaf-asdfsaf\"\n",
    "asecret=\"asdfsadfsadfsadfsadfsad\"\n",
    "\n",
    "from twitterapistuff import *\n",
    "\n",
    "class listener(StreamListener):\n",
    "\n",
    "    def on_data(self, data):\n",
    "\t\tall_data = json.loads(data)\n",
    "\n",
    "\t\ttweet = all_data[\"text\"]\n",
    "\t\tsentiment_value, confidence = s.sentiment(tweet)\n",
    "\t\tprint(tweet, sentiment_value, confidence)\n",
    "\n",
    "\t\tif confidence*100 >= 80:\n",
    "\t\t\toutput = open(\"twitter-out.txt\",\"a\")\n",
    "\t\t\toutput.write(sentiment_value)\n",
    "\t\t\toutput.write('\\n')\n",
    "\t\t\toutput.close()\n",
    "\n",
    "\t\treturn True\n",
    "\n",
    "    def on_error(self, status):\n",
    "        print(status)\n",
    "\n",
    "auth = OAuthHandler(ckey, csecret)\n",
    "auth.set_access_token(atoken, asecret)\n",
    "\n",
    "twitterStream = Stream(auth, listener())\n",
    "twitterStream.filter(track=[\"happy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Graphing Live Twitter Sentiment Analysis with NLTK with NLTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.animation as animation\n",
    "from matplotlib import style\n",
    "import time\n",
    "\n",
    "style.use(\"ggplot\")\n",
    "\n",
    "fig = plt.figure()\n",
    "ax1 = fig.add_subplot(1,1,1)\n",
    "\n",
    "def animate(i):\n",
    "    pullData = open(\"twitter-out.txt\",\"r\").read()\n",
    "    lines = pullData.split('\\n')\n",
    "\n",
    "    xar = []\n",
    "    yar = []\n",
    "\n",
    "    x = 0\n",
    "    y = 0\n",
    "\n",
    "    for l in lines[-200:]:\n",
    "        x += 1\n",
    "        if \"pos\" in l:\n",
    "            y += 1\n",
    "        elif \"neg\" in l:\n",
    "            y -= 1\n",
    "\n",
    "        xar.append(x)\n",
    "        yar.append(y)\n",
    "        \n",
    "    ax1.clear()\n",
    "    ax1.plot(xar,yar)\n",
    "ani = animation.FuncAnimation(fig, animate, interval=1000)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}