chapter_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "89d3af54",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import *\n",
    "import string\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "from collections import defaultdict\n",
    "import re\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import wikipedia as wp\n",
    "from nltk.util import ngrams"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7acb01e1",
   "metadata": {},
   "source": [
    "__◑ Read in some text from a corpus, tokenize it, and print the list of all wh-word types that occur. (wh-words in English are used in questions, relative clauses and exclamations: who, which, what, and so on.) Print them in order. Are any words duplicated in this list, because of the presence of case distinctions or punctuation?__\n",
    "\n",
    "Specifying only startwith WH gets unrelated words such as \"whisper\". In my case they are not duplicated due to case or punctuation becuase the text was pre-processed before extracting WH. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "84b163fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "emma = gutenberg.raw(\"austen-emma.txt\")\n",
    "emma_tokens = nltk.word_tokenize(emma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "351e7689",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'what',\n",
       " 'whatever',\n",
       " 'wheat',\n",
       " 'when',\n",
       " 'whence',\n",
       " 'whenever',\n",
       " 'where',\n",
       " 'wheres',\n",
       " 'wherever',\n",
       " 'whether',\n",
       " 'which',\n",
       " 'whichever',\n",
       " 'while',\n",
       " 'whim',\n",
       " 'whims',\n",
       " 'whirling',\n",
       " 'whisper',\n",
       " 'whispered',\n",
       " 'whispering',\n",
       " 'whispers',\n",
       " 'whist',\n",
       " 'whist-club',\n",
       " 'whist-players',\n",
       " 'white',\n",
       " 'whiten',\n",
       " 'who',\n",
       " 'whoever',\n",
       " 'whole',\n",
       " 'whole-length',\n",
       " 'whole-lengths',\n",
       " 'wholesome',\n",
       " 'wholesome.',\n",
       " 'wholesomeness',\n",
       " 'wholly',\n",
       " 'whom',\n",
       " 'whose',\n",
       " 'why'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wh = [tok.lower() for tok in emma_tokens if tok.lower().startswith(\"wh\")]\n",
    "set(wh)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ac91bf0",
   "metadata": {},
   "source": [
    "__◑ Create a file consisting of words and (made up) frequencies, where each line consists of a word, the space character, and a positive integer, e.g. fuzzy 53. Read the file into a Python list using open(filename).readlines(). Next, break each line into its two fields using split(), and convert the number into an integer using int(). The result should be a list of the form: [['fuzzy', 53], ...].__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "72e13451",
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = []\n",
    "with open(\"freqs.txt\") as f:\n",
    "    for line in f.readlines():\n",
    "        lines.append(line.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9f13423f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['fuzzy', '54'],\n",
       " ['wug', '101'],\n",
       " ['nlp', '12'],\n",
       " ['machine', '4'],\n",
       " ['learning', '8'],\n",
       " ['corpus', '90']]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2945e3c2",
   "metadata": {},
   "source": [
    "__◑ Write code to access a favorite webpage and extract some text from it. For example, access a weather site and extract the forecast top temperature for your town or city today.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "63a01c39",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_url(url):\n",
    "    import requests\n",
    "    from bs4 import BeautifulSoup\n",
    "    from unicodedata import normalize\n",
    "    raw = requests.get(url).text\n",
    "    clean1 = BeautifulSoup(raw)\n",
    "    final = \" \".join([s for s in clean1.stripped_strings])\n",
    "    return normalize(\"NFKC\", final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "fbe21d17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The Project Gutenberg eBook of The Odyssey, by Homer The Project Gutenberg eBook of The Odyssey, by '"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "odyssey = get_url(\"https://www.gutenberg.org/files/3160/3160-h/3160-h.htm\")\n",
    "odyssey[:100]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5ba6486",
   "metadata": {},
   "source": [
    "__◑ Write a function unknown() that takes a URL as its argument, and returns a list of unknown words that occur on that webpage. In order to do this, extract all substrings consisting of lowercase letters (using re.findall()) and remove any items from this set that occur in the Words Corpus (nltk.corpus.words). Try to categorize these words manually and discuss your findings.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "5cf9f49f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "    if type(text) == str:\n",
    "        text = nltk.word_tokenize(text)\n",
    "        text = [w.lower() for w in text if w.lower() not in stopwords.words(\"english\") and w.lower() not in string.punctuation]\n",
    "        \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "72b9bec8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def unknown(url):\n",
    "    text = get_url(url)\n",
    "    corpus = [w.lower() for w in words.words()]\n",
    "    unknown = [w for w in preprocess(text) if w not in corpus]\n",
    "    return unknown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7407456",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "04c0737d",
   "metadata": {},
   "source": [
    "__◑ Examine the results of processing the URL http://news.bbc.co.uk/ using the regular expressions suggested above. You will see that there is still a fair amount of non-textual data there, particularly Javascript commands. You may also find that sentence breaks have not been properly preserved. Define further regular expressions that improve the extraction of text from this web page.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "2f567d49",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Home - BBC News Homepage Accessibility links Skip to content Accessibility Help BBC Account Notifications Home News Sport Weather iPlayer Sounds CBBC CBeebies Food Bitesize Arts Taster Local Three Men'"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_url(\"http://news.bbc.co.uk/\")[:200] # firts 200 char, seems to be quite clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "cfe91f2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ity Help Parental Guidance Contact the BBC Get Personalised Newsletters Copyright © 2022 BBC. The BBC is not responsible for the content of external sites. Read about our approach to external linking.'"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_url(\"http://news.bbc.co.uk/\")[-200:] # last 200 char, also seems to be quite clean"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75f943ae",
   "metadata": {},
   "source": [
    "__◑ Are you able to write a regular expression to tokenize text in such a way that the word _don't_ is tokenized into _do_ and _n't_? Explain why this regular expression won't work: «n't|\\w+».__\n",
    "\n",
    "It does word due to the greedy quantifier \\w+. Making the quantifier lazy (\\w+?) will not solve the problem, since it will match as less as possible, matching individual characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "94339c52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I',\n",
       " 'do',\n",
       " \"n't\",\n",
       " 'want',\n",
       " 'to',\n",
       " 'go',\n",
       " 'I',\n",
       " 'wo',\n",
       " \"n't\",\n",
       " 'go',\n",
       " 'Quotes',\n",
       " 'citation',\n",
       " '2021']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'n\\'t|\\w+(?=n\\'t)|\\w+(?!n\\'t)', \"I don't want to go! I won't go. 'Quotes' (citation, 2021)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aaabbcae",
   "metadata": {},
   "source": [
    "__◑ Try to write code to convert text into hAck3r, using regular expressions and substitution, where e → 3, i → 1, o → 0, l → |, s → 5, . → 5w33t!, ate → 8. Normalize the text to lowercase before converting it. Add more substitutions of your own. Now try to map s to two different values: $ for word-initial s, and 5 for word-internal s.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "8e0fc9b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate(word):\n",
    "    org = ['ate', 'e', 'i', 'o', 'l', '\\.']\n",
    "    sub = ['8', '3', '1', '0', '|', '5w33t!']\n",
    "    for i in range(len(org)):\n",
    "        word = re.sub(org[i], sub[i], word.lower())\n",
    "        word = re.sub(\"^s\", \"$\", word.lower())\n",
    "        word = re.sub(\"(?<!^)s\", \"5\", word.lower())\n",
    "    print(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "49ca7aa6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "$tr1ng w1th 5 1nt3rna| - 5a|m0n\n"
     ]
    }
   ],
   "source": [
    "translate(\"String with s internal - salmon\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "030db72d",
   "metadata": {},
   "source": [
    "__◑ Pig Latin is a simple transformation of English text. Each word of the text is converted as follows: move any consonant (or consonant cluster) that appears at the start of the word to the end, then append ay, e.g. string → ingstray, idle → idleay. http://en.wikipedia.org/wiki/Pig_Latin__\n",
    "\n",
    "Write a function to convert a word to Pig Latin.\n",
    "\n",
    "Write code that converts text, instead of individual words.\n",
    "\n",
    "Extend it further to preserve capitalization, to keep qu together (i.e. so that quiet becomes ietquay), and to detect when y is used as a consonant (e.g. yellow) vs a vowel (e.g. style)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "6572708b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pig_latin(tokens):\n",
    "    \n",
    "    if type(tokens) == str:\n",
    "        tokens = nltk.word_tokenize(tokens.lower())\n",
    "    new_word = []\n",
    "\n",
    "    for word in tokens:\n",
    "        if re.search(\"^[^a-z]\", word.lower()):\n",
    "            new_word.append(word)\n",
    "        elif re.search(\"^[aoiue]\", word.lower()):\n",
    "            new_word.append(word + \"ay\")\n",
    "        elif re.search(\"^[^aoiue]\", word.lower()):\n",
    "            d = re.findall(\"(^[^aouie]+)([aoiue])\", word)\n",
    "            start = word.index(d[0][1])\n",
    "            new_word.append(word[start:] + d[0][0] + \"ay\")\n",
    "        \n",
    "    return new_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "5c138c17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['okenstay', 'anday', 'omesay', 'oremay', 'ordsway', 'usplay', '700']"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pig_latin(\"tokens and some more words plus 700\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7995d356",
   "metadata": {},
   "source": [
    "__◑ Download some text from a language that has vowel harmony (e.g. Hungarian), extract the vowel sequences of words, and create a vowel bigram table.__\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "0a29cb8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "turk_text = get_url(\"https://www.bbc.com/turkce/haberler-turkiye-60593781\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "b9983b23",
   "metadata": {},
   "outputs": [],
   "source": [
    "vowels_in_word = []\n",
    "for word in turk_text.split():\n",
    "    vowels_in_word.append(re.findall(\"[aeıioöuü]\", word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "e4de2930",
   "metadata": {},
   "outputs": [],
   "source": [
    "bi_dict = defaultdict(int)\n",
    "for vow_seq in vowels_in_word:\n",
    "    for bi in nltk.bigrams(vow_seq):\n",
    "        bi_dict[bi] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "a3ef610a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bi_dict) # all possible vow bigrams in the tex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "1cd340b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.zeros((3,3))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a281a207",
   "metadata": {},
   "source": [
    "__◑ Python's random module includes a function choice() which randomly chooses an item from a sequence, e.g. choice(\"aehh \") will produce one of four possible characters, with the letter h being twice as frequent as the others. Write a generator expression that produces a sequence of 500 randomly chosen letters drawn from the string \"aehh \", and put this expression inside a call to the ''.join() function, to concatenate them into one long string. You should get a result that looks like uncontrolled sneezing or maniacal laughter: he  haha ee  heheeh eha. Use split() and join() again to normalize the whitespace in this string.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "e16c6cb2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'aehhhhhheaehheehhhaeehhhhehehehhahhehaaahehheehhae'"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\".join([random.choice(\"aehh\") for _ in range(50)]) #sneezing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "5a64f6a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ah h hh haeeh eehhh h hh  h h hhheheehaheehh hehea'"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\".join([random.choice(\"aehh \") for _ in range(50)]) #some whitespace"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ee067bf",
   "metadata": {},
   "source": [
    "__◑ Consider the numeric expressions in the following sentence from the MedLine Corpus: The corresponding free cortisol fractions in these sera were 4.53 +/- 0.15% and 8.16 +/- 0.23%, respectively. Should we say that the numeric expression 4.53 +/- 0.15% is three words? Or should we say that it's a single compound word? Or should we say that it is actually nine words, since it's read \"four point five three, plus or minus zero point fifteen percent\"? Or should we say that it's not a \"real\" word at all, since it wouldn't appear in any dictionary? Discuss these different possibilities. Can you think of application domains that motivate at least two of these answers?__\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e99cf86e",
   "metadata": {},
   "source": [
    "__◑ Readability measures are used to score the reading difficulty of a text, for the purposes of selecting texts of appropriate difficulty for language learners. Let us define μw to be the average number of letters per word, and μs to be the average number of words per sentence, in a given text. The Automated Readability Index (ARI) of the text is defined to be: 4.71 μw + 0.5 μs - 21.43. Compute the ARI score for various sections of the Brown Corpus, including section f (lore) and j (learned). Make use of the fact that nltk.corpus.brown.words() produces a sequence of words, while nltk.corpus.brown.sents() produces a sequence of sentences.__\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "96b0c7b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.08 adventure           \n",
      "10.99 belles_lettres      \n",
      "9.47 editorial           \n",
      "4.91 fiction             \n",
      "12.08 government          \n",
      "8.92 hobbies             \n",
      "7.89 humor               \n",
      "11.93 learned             \n",
      "10.25 lore                \n",
      "3.83 mystery             \n",
      "10.18 news                \n",
      "10.20 religion            \n",
      "10.77 reviews             \n",
      "4.35 romance             \n",
      "4.98 science_fiction     \n"
     ]
    }
   ],
   "source": [
    "def ari(words, sents):\n",
    "    s = sum([len(s) for s in sents]) / len(sents)\n",
    "    w = sum([len(w) for w in words]) / len(words)\n",
    "    ari = (4.71 * w) + (0.5 * s) - 21.43\n",
    "    return ari\n",
    "\n",
    "for cat in brown.categories():\n",
    "    ari_score = ari(brown.words(categories=cat), brown.sents(categories=cat))\n",
    "    print(\"{:.2f} {:<20s}\".format(ari_score, cat))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efe61164",
   "metadata": {},
   "source": [
    "__◑ Use the Porter Stemmer to normalize some tokenized text, calling the stemmer on each word. Do the same thing with the Lancaster Stemmer and see if you observe any differences.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "f7c8c168",
   "metadata": {},
   "outputs": [],
   "source": [
    "porter = nltk.PorterStemmer()\n",
    "lancaster = nltk.LancasterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "52e6f439",
   "metadata": {},
   "outputs": [],
   "source": [
    "news = brown.words(categories=\"news\")\n",
    "news_porter = [porter.stem(w) for w in news]\n",
    "news_lanc = [lancaster.stem(w) for w in news]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "7f75e047",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the fulton counti grand juri said friday an investig of atlanta' recent primari elect produc `` no evid '' that\n",
      "the fulton county grand jury said friday an investig of atlanta's rec prim elect produc `` no evid '' that\n",
      "The Fulton County Grand Jury said Friday an investigation of Atlanta's recent primary election produced `` no evidence '' that\n"
     ]
    }
   ],
   "source": [
    "print(*news_porter[:20])\n",
    "print(*news_lanc[:20])\n",
    "print(*news[:20])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb139109",
   "metadata": {},
   "source": [
    "__◑ Define the variable saying to contain the list ['After', 'all', 'is', 'said', 'and', 'done', ',', 'more',\n",
    "'is', 'said', 'than', 'done', '.'].__\n",
    "\n",
    "__Process this list using a for loop, and store the length of each word in a new list lengths. Hint: begin by assigning the empty list to lengths, using lengths = []. Then each time through the loop, use append() to add another length value to the list. Now do the same thing using a list comprehension.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "20861b0b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5, 3, 2, 4, 3, 4, 1, 4, 2, 4, 4, 4, 1]"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list1 = ['After', 'all', 'is', 'said', 'and', 'done', ',', 'more', 'is', 'said', 'than', 'done', '.']\n",
    "length_list1 = [len(w) for w in list1]\n",
    "length_list1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "346fcb77",
   "metadata": {},
   "source": [
    "__◑ Define a variable silly to contain the string: 'newly formed bland ideas are inexpressible in an infuriating\n",
    "way'. (This happens to be the legitimate interpretation that bilingual English-Spanish speakers can assign to Chomsky's famous nonsense phrase, colorless green ideas sleep furiously according to Wikipedia). Now write code to perform the following tasks:__\n",
    "\n",
    "Split silly into a list of strings, one per word, using Python's split() operation, and save this to a variable called bland.\n",
    "\n",
    "Extract the second letter of each word in silly and join them into a string, to get 'eoldrnnnna'.\n",
    "\n",
    "Combine the words in bland back into a single string, using join(). Make sure the words in the resulting string are separated with whitespace.\n",
    "\n",
    "Print the words of silly in alphabetical order, one per line."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "a4fc9f5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['an',\n",
       " 'are',\n",
       " 'bland',\n",
       " 'formed',\n",
       " 'ideas',\n",
       " 'in',\n",
       " 'inexpressible',\n",
       " 'infuriating',\n",
       " 'newly',\n",
       " 'way']"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "silly = \"newly formed bland ideas are inexpressible in an infuriating way\"\n",
    "split_silly = silly.split()  # split\n",
    "second_letters = \"\".join([w[1] for w in split_silly])  # extract 2nd letter\n",
    "joined_silly = \" \".join(split_silly)  # join again\n",
    "sorted(split_silly)  #sort alpahbetically"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da6fb4d0",
   "metadata": {},
   "source": [
    "__◑ The index() function can be used to look up items in sequences. For example, 'inexpressible'.index('e') tells us the index of the first position of the letter e.__\n",
    "\n",
    "What happens when you look up a substring, e.g. 'inexpressible'.index('re')?\n",
    "\n",
    "Define a variable words containing a list of words. Now use words.index() to look up the position of an individual word.\n",
    "\n",
    "Define a variable silly as in the exercise above. Use the index() function in combination with list slicing to build a list phrase consisting of all the words up to (but not including) in in silly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "id": "4415d613",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'r'"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"inexpressible\"[\"inexpressible\".index('re')]  # gives the beginning index of \"re\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "95b3f0cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'said'"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list1[list1.index(\"said\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "0572a52c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'newly formed bland ideas are '"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "silly[:silly.index(\"in\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b39ac955",
   "metadata": {},
   "source": [
    "__◑ Write code to convert nationality adjectives like Canadian and Australian to their corresponding nouns Canada and Australia (see http://en.wikipedia.org/wiki/List_of_adjectival_forms_of_place_names).__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8399647",
   "metadata": {},
   "outputs": [],
   "source": [
    "def nationality(adj):\n",
    "    \"\"\"\n",
    "    Converts nationality adjectives like Canadian and Australian to their corresponding nouns Canada and Australia\n",
    "    \"\"\"\n",
    "    html = wp.page(\"List_of_adjectival_and_demonymic_forms_of_place_names\").html().encode(\"UTF-8\")\n",
    "    for i in range(1, 13):\n",
    "        try: \n",
    "            df = pd.read_html(html)[i]\n",
    "        except IndexError:\n",
    "            break\n",
    "            \n",
    "    df.Adjective\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01d23673",
   "metadata": {},
   "source": [
    "__◑ Read the LanguageLog post on phrases of the form as best as p can and as best p can, where p is a pronoun. Investigate this phenomenon with the help of a corpus and the findall() method for searching tokenized text described in 3.5. http://itre.cis.upenn.edu/~myl/languagelog/archives/002733.html__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "id": "c05bd359",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens_brown = brown.words()\n",
    "fd = nltk.FreqDist(match for match in re.findall(r'as best (?:as |)(?:he|she|you|I|we|they) can', \" \".join(tokens_brown)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "id": "e5046b33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['best they can']"
      ]
     },
     "execution_count": 270,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the above fd did not give any results, so I will try wihtout the first \"as\"\n",
    "re.findall(r'best (?:as )?(?:he|she|you|I|we|they) can', \" \".join(tokens_brown))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "id": "f15a3160",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['best we can', 'best I can']"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# another corpus, still no matches with \"as\"\n",
    "re.findall(r'(?:as )?best (?:as )?(?:he|she|you|I|we|they) can', \" \".join(gutenberg.words()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d82a05d0",
   "metadata": {},
   "source": [
    "__◑ Study the lolcat version of the book of Genesis, accessible as nltk.corpus.genesis.words('lolcat.txt'), and the rules for converting text into lolspeak at http://www.lolcatbible.com/index.php?title=How_to_speak_lolcat. Define regular expressions to convert English words into corresponding lolspeak words.__\n",
    "\n",
    "I will skip this one, as the url does not open and I couldn't find lolcat rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "id": "6f542fc6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Oh', 'hai', '.', 'In', 'teh', 'beginnin', 'Ceiling', ...]"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.corpus.genesis.words('lolcat.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c20def9f",
   "metadata": {},
   "source": [
    "__◑ Read about the re.sub() function for string substitution using regular expressions, using help(re.sub) and by consulting the further readings for this chapter. Use re.sub in writing code to remove HTML tags from an HTML file, and to normalize whitespace.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "id": "bb14607c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def html_clean(html):\n",
    "    html = re.sub(r'<\\/?\\w+?>', \"\", html) #remove html markups\n",
    "    html = re.sub(r'(\\s|\\n)+', \" \", html) #remove extra whitespaces\n",
    "    return html"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd124c6f",
   "metadata": {},
   "source": [
    "__★ An interesting challenge for tokenization is words that have been split across a line-break. E.g. if long-term is split, then we have the string long-\\nterm.__\n",
    "\n",
    "Write a regular expression that identifies words that are hyphenated at a line-break. The expression will need to include the \\n character.\n",
    "\n",
    "Use re.sub() to remove the \\n character from these words.\n",
    "\n",
    "How might you identify words that should not remain hyphenated once the newline is removed, e.g. 'encyclo-\\npedia'?x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "b259cff8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "encyclo-\n",
      "pedia\n",
      "or it can be a self-\n",
      "esteem\n",
      "maybe long-\n",
      "term or something incompre-\n",
      "hensible\n"
     ]
    }
   ],
   "source": [
    "line_break = \"\"\n",
    "with open(\"linbreaks.txt\", \"r\") as f:\n",
    "    for line in f.readlines():\n",
    "        line_break += line\n",
    "print(line_break)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "265eaff5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'encyclo-pedia or it can be a self-esteem maybe long-term or something incompre-hensible'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "line_break = re.sub(\"(?<=-)\\n\", \"\", line_break)\n",
    "line_break = re.sub(\"\\n(?=[a-zA-Z])\", \" \", line_break)\n",
    "line_break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2e2c8ecf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "encyclopedia or it can be a self-esteem maybe long-term or something incomprehensible\n"
     ]
    }
   ],
   "source": [
    "after_join = []\n",
    "for w in line_break.split():\n",
    "    if w.replace(\"-\",\"\") in nltk.corpus.words.words():\n",
    "        after_join.append(w.replace(\"-\",\"\"))\n",
    "    else:\n",
    "        after_join.append(w)\n",
    "\n",
    "print(\" \".join(after_join))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f21b8c5",
   "metadata": {},
   "source": [
    "__★ Read the Wikipedia entry on Soundex. Implement this algorithm in Python.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "10ea06f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "mappings = {let:num for num, let in zip(\"01230120022455012623010202\", \"abcdefghijklmnopqrstuvwxyz\")}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "b0704f99",
   "metadata": {},
   "outputs": [],
   "source": [
    "def soundex(word):\n",
    "    \n",
    "    soundex = \"\"\n",
    "    soundex += word[0].upper()\n",
    "    word = word.lower()\n",
    "    \n",
    "    for i in range(1, len(word)):\n",
    "        if mappings[word[i]] != \"0\": soundex += mappings[word[i]] \n",
    "    \n",
    "    if len(soundex) < 4:\n",
    "        soundex += (4 - len(soundex)) * \"0\"\n",
    "    elif len(soundex) > 3:\n",
    "        soundex = soundex[:4]\n",
    "    return soundex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "098afc1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "S500 D120 R263 R163\n"
     ]
    }
   ],
   "source": [
    "print(soundex(\"SIM\"), soundex(\"Davis\"), soundex(\"Richard\"), soundex(\"Rupert\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dcebe19e",
   "metadata": {},
   "source": [
    "__★ Obtain raw texts from two or more genres and compute their respective reading difficulty scores as in the earlier exercise on reading difficulty. E.g. compare ABC Rural News and ABC Science News (nltk.corpus.abc). Use Punkt to perform sentence segmentation.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "5955679b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12.35       rural.txt\n",
      "12.53     science.txt\n"
     ]
    }
   ],
   "source": [
    "for file in abc.fileids():\n",
    "    print(\"{:.2f} {:>15s}\".format(ari(abc.words(file), abc.sents(file)), file))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d2c23b6",
   "metadata": {},
   "source": [
    "__★ Use WordNet to create a semantic index for a text collection. Extend the concordance search program in 3.6, indexing each word using the offset of its first synset, e.g. wn.synsets('dog')[0].offset (and optionally the offset of some of its ancestors in the hypernym hierarchy).__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "4acfcdd2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r king ! DENNIS : Listen , strange women lying in ponds distributing swords is no (WordNet Offset: 751944)\n",
      " beat a very brave retreat . ROBIN : All lies ! MINSTREL : [ singing ] Bravest of (WordNet Offset: 6756831)\n",
      "       Nay . Nay . Come . Come . You may lie here . Oh , but you are wounded !    (WordNet Offset: 6756831)\n",
      "doctors immediately ! No , no , please ! Lie down . [ clap clap ] PIGLET : Well   (WordNet Offset: 6756831)\n",
      "ere is much danger , for beyond the cave lies the Gorge of Eternal Peril , which  (WordNet Offset: 6756831)\n",
      "   you . Oh ... TIM : To the north there lies a cave -- the cave of Caerbannog -- (WordNet Offset: 6756831)\n",
      "h it and lived ! Bones of full fifty men lie strewn about its lair . So , brave k (WordNet Offset: 6756831)\n",
      "not stop our fight ' til each one of you lies dead , and the Holy Grail returns t (WordNet Offset: 6756831)\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import wordnet as wn\n",
    "class IndexedText():\n",
    "    def __init__(self, stemmer, text):\n",
    "        self._text = text\n",
    "        self._stemmer = stemmer\n",
    "        self._index = nltk.Index((self._stem(word), i)\n",
    "            for (i, word) in enumerate(text))\n",
    "\n",
    "    def concordance(self, word, width=40):\n",
    "        key = self._stem(word)\n",
    "        wc = int(width/4) # words of context\n",
    "        for i in self._index[key]:\n",
    "            lcontext = ' '.join(self._text[i-wc:i])\n",
    "            rcontext = ' '.join(self._text[i:i+wc])\n",
    "            offset = '(WordNet Offset: ' + str(wn.synsets(self._text[i])[0].offset()) + ')'\n",
    "            ldisplay = '%*s' % (width, lcontext[-width:])\n",
    "            rdisplay = '%-*s' % (width, rcontext[:width])\n",
    "            print(ldisplay, rdisplay, offset)\n",
    "                \n",
    "    def _stem(self, word):\n",
    "        return self._stemmer.stem(word).lower()\n",
    "\n",
    "porter = nltk.PorterStemmer()\n",
    "grail = nltk.corpus.webtext.words('grail.txt')\n",
    "text = IndexedText(porter, grail)\n",
    "text.concordance('lie')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f84d6645",
   "metadata": {},
   "source": [
    "__★ With the help of a multilingual corpus such as the Universal Declaration of Human Rights Corpus (nltk.corpus.udhr), and NLTK's frequency distribution and rank correlation functionality (nltk.FreqDist, nltk.spearman_correlation), develop a system that guesses the language of a previously unseen text. For simplicity, work with a single character encoding and just a few languages.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "05dada44",
   "metadata": {},
   "outputs": [],
   "source": [
    "def guessLanguage(text):\n",
    "    # tokens = nltk.wordpunct_tokenize(text)\n",
    "    # text = nltk.Text(tokens)\n",
    "    text_trichar = ngrams(text, 6)\n",
    "    freq_char_source = nltk.FreqDist(text_trichar)\n",
    "    best_guess = ('', 0)\n",
    "    best_intersection = []\n",
    "    for lang in nltk.corpus.udhr.fileids():\n",
    "        if (lang.endswith('-Latin1')):\n",
    "            lang_trichar = ngrams(udhr.raw(lang).replace(\"\\n\", \"\"), 6)\n",
    "            freq_char_cand = nltk.FreqDist(lang_trichar)\n",
    "            intersection = list(set(freq_char_source.keys()) & set(freq_char_cand.keys()))\n",
    "            tuples_source = []\n",
    "            tuples_cand = []\n",
    "            for char in intersection:\n",
    "                tuples_source.append((char, freq_char_source[char]))\n",
    "                tuples_cand.append((char, freq_char_cand[char]))\n",
    "            spearman = nltk.spearman_correlation(tuples_source, tuples_cand)\n",
    "            if ((best_guess[1] == 0 and spearman != 0.0) or (spearman != 0.0 and spearman > best_guess[1])):\n",
    "                best_guess = (lang[:-7], spearman)\n",
    "    return best_guess[0];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "b60ae08f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Achehnese'"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "guessLanguage(\"You will definitely gues that the text is written in English\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d99eb079",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abff4dfa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5994800b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bea187c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}