NLP_BasicTask.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spell Correction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Non-word spelling mistake correction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from autocorrect import spell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "caesar\n",
      "message\n",
      "service\n",
      "the\n"
     ]
    }
   ],
   "source": [
    "print (spell('caaaar'))\n",
    "print (spell('mussage'))\n",
    "print (spell('survice'))\n",
    "print (spell('hte'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "car\n",
      "car\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "print(re.sub('a+','a','caaar'))\n",
    "print (spell(re.sub('a+','a','caaar')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentence Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Tokenization is a way to split text into tokens.',\n",
       " 'These tokens could be paragraphs, sentences, or individual words.']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "word_data = \"Tokenization is a way to split text into tokens. These tokens could be paragraphs, sentences, or individual words.\"\n",
    "nltk_tokens = nltk.sent_tokenize(word_data)\n",
    "display(nltk_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word Tokenizer (Unigrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The',\n",
       " 'best',\n",
       " 'performance',\n",
       " 'can',\n",
       " 'bring',\n",
       " 'in',\n",
       " 'sky',\n",
       " 'high',\n",
       " 'success',\n",
       " '.']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "word_data = \"The best performance can bring in sky high success.\"\n",
    "nltk_tokens = nltk.word_tokenize(word_data)\n",
    "display(nltk_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('The', 'best'),\n",
       " ('best', 'performance'),\n",
       " ('performance', 'can'),\n",
       " ('can', 'bring'),\n",
       " ('bring', 'in'),\n",
       " ('in', 'sky'),\n",
       " ('sky', 'high'),\n",
       " ('high', 'success'),\n",
       " ('success', '.')]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "word_data = \"The best performance can bring in sky high success.\"\n",
    "nltk_tokens = nltk.word_tokenize(word_data)\n",
    "\n",
    "display(list(nltk.bigrams(nltk_tokens)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Trigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('The', 'best', 'performance'),\n",
       " ('best', 'performance', 'can'),\n",
       " ('performance', 'can', 'bring'),\n",
       " ('can', 'bring', 'in'),\n",
       " ('bring', 'in', 'sky'),\n",
       " ('in', 'sky', 'high'),\n",
       " ('sky', 'high', 'success'),\n",
       " ('high', 'success', '.')]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "word_data = \"The best performance can bring in sky high success.\"\n",
    "nltk_tokens = nltk.word_tokenize(word_data)\n",
    "\n",
    "display(list(nltk.trigrams(nltk_tokens)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NGrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Four Grams\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('The', 'best', 'performance', 'can'),\n",
       " ('best', 'performance', 'can', 'bring'),\n",
       " ('performance', 'can', 'bring', 'in'),\n",
       " ('can', 'bring', 'in', 'sky'),\n",
       " ('bring', 'in', 'sky', 'high'),\n",
       " ('in', 'sky', 'high', 'success'),\n",
       " ('sky', 'high', 'success', '.')]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Five Grams\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('The', 'best', 'performance', 'can', 'bring'),\n",
       " ('best', 'performance', 'can', 'bring', 'in'),\n",
       " ('performance', 'can', 'bring', 'in', 'sky'),\n",
       " ('can', 'bring', 'in', 'sky', 'high'),\n",
       " ('bring', 'in', 'sky', 'high', 'success'),\n",
       " ('in', 'sky', 'high', 'success', '.')]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "word_data = \"The best performance can bring in sky high success.\"\n",
    "nltk_tokens = nltk.word_tokenize(word_data)\n",
    "\n",
    "print('Four Grams')\n",
    "display(list(nltk.ngrams(nltk_tokens,4)))\n",
    "\n",
    "print('\\nFive Grams')\n",
    "display(list(nltk.ngrams(nltk_tokens,5)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pos Tagging\n",
    "\n",
    "\n",
    "It is the process of marking up a word in a text (corpus) as corresponding to a particular part of speech, based on both its definition and its context—i.e., its relationship with adjacent and related words in a phrase, sentence, or paragraph.\n",
    "\n",
    "use nltk.help.upenn_tagset() to see the possible tags \n",
    "\n",
    "* IN  : preposition or conjunction, subordinating\n",
    "* JJ  : adjective or numeral, ordinal\n",
    "* JJR : adjective, comparative\n",
    "* JJS : adjective, superlative\n",
    "* NN  : noun, common, singular or mass\n",
    "* NNP : noun, proper, singular\n",
    "* NNPS: noun, proper, plural\n",
    "* NNS : noun, common, plural\n",
    "* RB  : adverb\n",
    "* RBR : adverb, comparative\n",
    "* RBS : adverb, superlative\n",
    "* VB  : verb, base form\n",
    "* VBD : verb, past tense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('My', 'PRP$'), ('name', 'NN'), ('is', 'VBZ'), ('Krishna', 'NNP')]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.pos_tag(nltk.word_tokenize('My name is Krishna'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Natural', 'JJ'),\n",
       " ('language', 'NN'),\n",
       " ('processing', 'NN'),\n",
       " ('is', 'VBZ'),\n",
       " ('the', 'DT'),\n",
       " ('ability', 'NN'),\n",
       " ('of', 'IN'),\n",
       " ('a', 'DT'),\n",
       " ('computer', 'NN'),\n",
       " ('program', 'NN'),\n",
       " ('to', 'TO'),\n",
       " ('understand', 'VB'),\n",
       " ('human', 'JJ'),\n",
       " ('language', 'NN'),\n",
       " ('as', 'IN'),\n",
       " ('it', 'PRP'),\n",
       " ('is', 'VBZ'),\n",
       " ('spoken', 'VBN'),\n",
       " ('.', '.')]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "s='Natural language processing is the ability of a computer program to understand human language as it is spoken.'\n",
    "pos=nltk.pos_tag(nltk.word_tokenize(s))\n",
    "\n",
    "display(pos)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NER Tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Tommorrow', 'NN'),\n",
       " ('I', 'PRP'),\n",
       " ('will', 'MD'),\n",
       " ('be', 'VB'),\n",
       " ('at', 'IN'),\n",
       " ('Mumbai', 'NNP'),\n",
       " ('.', '.')]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(S\n",
      "  Tommorrow/NN\n",
      "  I/PRP\n",
      "  will/MD\n",
      "  be/VB\n",
      "  at/IN\n",
      "  (ORGANIZATION Mumbai/NNP)\n",
      "  ./.)\n",
      "(ORGANIZATION Mumbai/NNP)\n"
     ]
    }
   ],
   "source": [
    "sentence = 'Tommorrow I will be at Mumbai.'\n",
    "tokenized = nltk.word_tokenize(sentence)\n",
    "tagged = nltk.pos_tag(tokenized)\n",
    "display(tagged)\n",
    "\n",
    "namedEnt = nltk.ne_chunk(tagged)\n",
    "for subtree in namedEnt.subtrees():\n",
    "    print(subtree)\n",
    "\n",
    "namedEnt.draw()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "paint\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem import PorterStemmer\n",
    "stemmer = PorterStemmer() \n",
    "print(stemmer.stem('painting'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "increas\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem import PorterStemmer \n",
    "stemmer = PorterStemmer() \n",
    "print(stemmer.stem('increases'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lemmitization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "increase\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem import WordNetLemmatizer\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "print(lemmatizer.lemmatize('increases'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "playing\n",
      "play\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem import WordNetLemmatizer\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "print(lemmatizer.lemmatize('playing')) # default pos is noun\n",
    "print(lemmatizer.lemmatize('playing', pos=\"v\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stop word removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\"]\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stpwrd=(stopwords.words('english'))\n",
    "print(stpwrd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['i', 'love', 'chocolates']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['love', 'chocolates']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence=\"I Love Chocolates\"\n",
    "tokens=sentence.lower().split(' ')\n",
    "print(tokens)\n",
    "\n",
    "for token in tokens:\n",
    "    if token in stpwrd:\n",
    "        tokens.remove(token)\n",
    "tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "documents=['I Love India','India sport']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(stop_words='english')\n",
    "X = vectorizer.fit_transform(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>india</th>\n",
       "      <th>love</th>\n",
       "      <th>sport</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.579739</td>\n",
       "      <td>0.814802</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.579739</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.814802</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      india      love     sport\n",
       "0  0.579739  0.814802  0.000000\n",
       "1  0.579739  0.000000  0.814802"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.DataFrame(X.toarray(),columns=vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Synonyms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a symptom of some physical hurt or disorder\n",
      "['the patient developed severe pain and distension']\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import wordnet\n",
    "syn = wordnet.synsets(\"pain\")\n",
    "print(syn[0].definition())\n",
    "print(syn[0].examples())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ail',\n",
       " 'anguish',\n",
       " 'annoyance',\n",
       " 'bother',\n",
       " 'botheration',\n",
       " 'hurt',\n",
       " 'hurting',\n",
       " 'infliction',\n",
       " 'nuisance',\n",
       " 'pain',\n",
       " 'pain_in_the_ass',\n",
       " 'pain_in_the_neck',\n",
       " 'pain_sensation',\n",
       " 'painful_sensation',\n",
       " 'painfulness',\n",
       " 'trouble'}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from nltk.corpus import wordnet \n",
    "synonyms = []\n",
    "for syn in wordnet.synsets('pain'):\n",
    "    for lemma in syn.lemmas():\n",
    "        synonyms.append(lemma.name())\n",
    "display(set(synonyms))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Antonyms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'big', 'large'}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from nltk.corpus import wordnet\n",
    "antonyms = []\n",
    "for syn in wordnet.synsets(\"small\"):\n",
    "    for l in syn.lemmas():\n",
    "        if l.antonyms():\n",
    "            antonyms.append(l.antonyms()[0].name())\n",
    "display(set(antonyms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "230px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}