Search_Engine.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uk69gSInIAdc"
   },
   "outputs": [],
   "source": [
    "pip install bs4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gQER2T81JC6D"
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.getcwd()\n",
    "os.chdir(\"HTML\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import nltk\n",
    "import codecs\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "def Sort_Tuple(tup,ind):\n",
    "    tup.sort(key = lambda x: x[ind])  \n",
    "    return tup  \n",
    "\n",
    "List_all_rank = []\n",
    "List_rank = []\n",
    "def lesk(query, html_content,ind):\n",
    "    Text1 = html_content.lower()\n",
    "    words = nltk.word_tokenize(Text1)\n",
    "    \n",
    "    \n",
    "    stop_words = stopwords.words(\"english\")\n",
    "    stop_words += ['can', 'will', 'use', 'one', 'using', 'used', 'also', 'see', 'first', 'like']\n",
    "    stop_words += ['page', 'get', 'new', 'two', 'site', 'blog', 'many', 'may' ,\"don't\", 'dont', 'way']\n",
    "    stop_words += ['last', 'best', 'able', 'even', 'next', 'last', 'let', \"none\", 'every', 'three']\n",
    "    stop_words += ['lot', 'well', 'chart', 'much', 'based', 'important', 'posts', 'reads', 'least']\n",
    "    stop_words += ['still', 'follow', 'called', 'and','this', 'that', 'there', 'as','the', 'is']\n",
    "    stop_words += ['/', '=', '.', ',', '.','|']\n",
    "\n",
    "    filtered_words = []\n",
    "    for word in words:\n",
    "        if word not in stop_words:\n",
    "            filtered_words.append(word)\n",
    "            \n",
    "    filtered_query = []\n",
    "    for word in query.split(\" \"):\n",
    "        if word not in stop_words:\n",
    "            filtered_query.append(word)\n",
    "    \n",
    "    \n",
    "    \n",
    "    count = 0;  \n",
    "    if(len(query.split()) > 1):\n",
    "\n",
    "        for i in range(0, len(filtered_query)):\n",
    "\n",
    "            for j in range(0, len(filtered_words)):  \n",
    "                if(WordNetLemmatizer().lemmatize(filtered_query[i], pos=\"v\") == WordNetLemmatizer().lemmatize(filtered_words[j], pos=\"v\")):  \n",
    "                    count = count + 1;\n",
    "\n",
    "\n",
    "        l = []\n",
    "        l.insert(0,ind)\n",
    "        l.append(count)\n",
    "        y2 = tuple(l)\n",
    "        List_rank.append(y2)\n",
    "    \n",
    "    else:\n",
    "        \n",
    "        word = query\n",
    "        len_syn = len(wordnet.synsets(word))\n",
    "\n",
    "        de = [] # words that found in definition and examples\n",
    "\n",
    "        for x in range(len_syn):\n",
    "\n",
    "            for i in filtered_words:\n",
    "                synset = wordnet.synsets(word)[x]\n",
    "                if i in synset.definition().lower():\n",
    "                    if i in de:\n",
    "                        pass\n",
    "                    else:\n",
    "                        de.append(i)\n",
    "\n",
    "                if len(synset.examples()) == 0:\n",
    "                    pass\n",
    "                else:\n",
    "                    if i in synset.examples()[0].lower() and word in synset.examples()[0].lower():\n",
    "                        if i in de:\n",
    "                            pass\n",
    "                        else:\n",
    "                            de.append(i)\n",
    "\n",
    "            l = []\n",
    "            l.insert(0,x)\n",
    "            r = len(de)\n",
    "            l.append(r)\n",
    "            y2 = tuple(l)\n",
    "            List_rank.append(y2)\n",
    "        Sorted_list_rank = Sort_Tuple(List_rank,1)\n",
    "        l = list(Sorted_list_rank[-1])\n",
    "        l.insert(0,ind)\n",
    "        y2 = tuple(l)                           \n",
    "        List_all_rank.append(y2)\n",
    "\n",
    "def main(query):\n",
    "    page_title = {}\n",
    "    page_name = {}\n",
    "    cw = os.getcwd()\n",
    "    # Get .html files\n",
    "    for indx, f_name in enumerate(os.listdir(cw)):\n",
    "        if f_name.endswith('.html'):\n",
    "            file = codecs.open(f_name, \"r\",\"utf-8\")\n",
    "            document = BeautifulSoup(file.read(), \"html.parser\")\n",
    "            page_title[indx] = document.title.text\n",
    "            page_name[indx] = f_name\n",
    "            for script in document([\"script\", \"style\"]):\n",
    "                script.decompose() # remove tags\n",
    "\n",
    "            strips = list(document.stripped_strings)\n",
    "            clean_html = ' '.join(strips)\n",
    "            lesk(query, clean_html, indx)\n",
    "    if(len(query.split()) > 1):\n",
    "        \n",
    "        s_li = Sort_Tuple(List_rank,1)\n",
    "        for x in range(len(List_rank),0,-1):\n",
    "            print(\"Page Number\",s_li[x-1][0]+1 ,\", With Rank\",s_li[x-1][1])\n",
    "            print(\"Title:\",page_title[s_li[x-1][0]])\n",
    "            print(\"File name:\",page_name[s_li[x-1][0]])\n",
    "            print(\"------------------------------------------\\n\")\n",
    "    else:\n",
    "        s_li = Sort_Tuple(List_all_rank,2)\n",
    "        for x in range(len(List_all_rank),0,-1):\n",
    "            print(\"Page Number\",s_li[x-1][0]+1, \", Most Synset Num\",s_li[x-1][1],\", With Rank\",s_li[x-1][2])\n",
    "            print(\"Title:\",page_title[s_li[x-1][0]])\n",
    "            print(\"File name:\",page_name[s_li[x-1][0]])\n",
    "            print(\"------------------------------------------\\n\")\n",
    "    \n",
    "#while True:\n",
    "#os.chdir(\"HTML\")\n",
    "query = input(\"Enter query to search (q to exit): \")\n",
    "if(query == 'q'):\n",
    "    #break\n",
    "    exit\n",
    "else:\n",
    "    if(len(query.split()) > 1):\n",
    "        main(query)\n",
    "    else:\n",
    "        len_syn = len(wordnet.synsets(query)) # if not synsets for word exit \n",
    "        if(len_syn == 0):\n",
    "            print(\"this word not found in Dict\")\n",
    "            #break\n",
    "            exit\n",
    "        else:\n",
    "            main(query)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Search Engine.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}