Web_Scraping_&_Natural_Language_Processing_of_web_articles.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e0c9a9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Topic: Data Extraction and NLP\n",
    "# Objective: To extract textual data of articles from the given URLs and perform text analysis. \n",
    "\n",
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "import string\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "\n",
    "\n",
    "######### DATA EXTRACTION #########\n",
    "\n",
    "input = pd.read_excel(\"Data/input.xlsx\",index_col=0)\n",
    "headers = {'User-Agent': '109.0.1518.61'}\n",
    "\n",
    "for url_id in input.index:\n",
    "    url = input.URL.loc[url_id]\n",
    "    print(url_id,url)\n",
    "    \n",
    "    # Web Scrapping\n",
    "    req = requests.get(url,headers=headers)\n",
    "    soup = BeautifulSoup(req.content, \"html.parser\")\n",
    "    \n",
    "    # Write data in a file\n",
    "    file = open(\"Data/TextFiles/\"+str(url_id)+\".txt\",\"w\",encoding=\"utf-8\")\n",
    "    file.write(soup.title.text)\n",
    "    # print(url_id, soup.title.text) # Un-comment to see current URL_ID\n",
    "\n",
    "    for data in soup.find_all(\"p\"):\n",
    "        file.write(' '+data.get_text())  # Added space to separate 2 paragraphs\n",
    "    file.close()\n",
    "\n",
    "\n",
    "######### DATA ANALYSIS #########\n",
    "\n",
    "output_df = pd.read_excel('Output Data Structure.xlsx', index_col=0)\n",
    "print(output_df.shape)\n",
    "output_df.sample()\n",
    "\n",
    "# Making a list with texts from all the text files (from URLs)\n",
    "texts = []\n",
    "for i in output_df.index:\n",
    "    with open('TextFiles/'+str(i)+'.txt',encoding=\"utf-8\",errors=\"ignore\") as f:\n",
    "        contents = f.read()\n",
    "    texts.append(contents)\n",
    "len(texts)\n",
    "\n",
    "# Load text data from txt files into dataframe\n",
    "texts_df = pd.DataFrame(output_df.loc[:,'URL'])\n",
    "texts_df.loc[:,'texts'] = texts\n",
    "texts_df.sample()\n",
    "\n",
    "stop_words_eng = set(stopwords.words('english'))\n",
    "\n",
    "\n",
    "######### 1.Sentimental Analysis #########\n",
    "\n",
    "##### 1.1 Cleaning using Stop Words #####\n",
    "column_names = ['POSITIVE SCORE', 'NEGATIVE SCORE', 'POLARITY SCORE', 'SUBJECTIVITY SCORE', \n",
    "                'AVG SENTENCE LENGTH', 'PERCENTAGE OF COMPLEX WORDS', 'FOG INDEX',\n",
    "                'AVG NUMBER OF WORDS PER SENTENCE', 'COMPLEX WORD COUNT', 'WORD COUNT',\n",
    "                'SYLLABLE PER WORD', 'PERSONAL PRONOUNS', 'AVG WORD LENGTH']\n",
    "\n",
    "# For 'Page not found' URLs\n",
    "no_of_pages_not_found = 0\n",
    "pages_not_found = []\n",
    "for i in texts_df.index:\n",
    "    \n",
    "    if ('Page not found' in texts_df.texts.loc[i]) or (len(texts_df.texts.loc[i])==0):\n",
    "        for column in column_names:\n",
    "            output_df.at[i,column] = 0\n",
    "        no_of_pages_not_found += 1\n",
    "        pages_not_found.append(i)\n",
    "        continue\n",
    "    \n",
    "# Convert text into list of tokens using nltk tokenize module \n",
    "    word_tokens = word_tokenize(texts_df.texts.loc[i])\n",
    "# Remove the stop words\n",
    "    filtered_words = [w for w in word_tokens if w.lower() not in stop_words_eng]\n",
    "# Remove the punctuations\n",
    "    clean_words = [w for w in filtered_words if w not in string.punctuation]\n",
    "    \n",
    "##### 1.2 Creating dictionary of Positive and Negative words #####\n",
    "    p_words_df = pd.read_csv('MasterDictionary/positive-words.txt',header=None,names=['words'])\n",
    "    n_words_df = pd.read_csv('MasterDictionary/negative-words.txt',header=None,names=['words'])\n",
    "\n",
    "    p_words = list(p_words_df.words)\n",
    "    n_words = list(n_words_df.words)\n",
    "    \n",
    "##### 1.3 Extracting Derived variables #####\n",
    "# 1.POSITIVE SCORE\n",
    "# 2.NEGATIVE SCORE\n",
    "# 3.POLARITY SCORE\n",
    "    positive_score, negative_score = 0, 0\n",
    "    for word in filtered_words:\n",
    "        if word in p_words:\n",
    "            positive_score += 1\n",
    "        elif word in n_words:\n",
    "            negative_score += 1\n",
    "\n",
    "    polarity_score = (positive_score-negative_score)/(positive_score+negative_score+0.000001)\n",
    "    \n",
    "# 4.SUBJECTIVITY SCORE\n",
    "    total_words = len(filtered_words)\n",
    "    subjectivity_score = (positive_score+negative_score)/(total_words+0.000001)\n",
    "\n",
    "\n",
    "######### Analysis of Readability #########\n",
    "\n",
    "# 5.WORD COUNT\n",
    "    no_of_words = len(clean_words)\n",
    "\n",
    "# 6.AVG SENTENCE LENGTH\n",
    "    sentence_tokens = sent_tokenize(texts_df.texts.iloc[0])\n",
    "    no_of_sentences = len(sentence_tokens)\n",
    "    avg_sentence_length = no_of_words/no_of_sentences\n",
    "    \n",
    "# 7.COMPLEX WORD COUNT\n",
    "    vowels = ['a','e','i','o','u']\n",
    "    no_of_complex_words = 0\n",
    "    total_syllables = 0\n",
    "    for word in clean_words:\n",
    "\n",
    "        vowel_count = 0\n",
    "        for v in vowels:\n",
    "            vowel_count += word.count(v)\n",
    "\n",
    "        syllable_count = vowel_count\n",
    "        if word[-2:] in [\"es\",\"ed\"]:\n",
    "            syllable_count -= 1\n",
    "\n",
    "        total_syllables += syllable_count\n",
    "        if syllable_count > 2:\n",
    "            no_of_complex_words += 1\n",
    "\n",
    "# 8.PERCENTAGE OF COMPLEX WORDS\n",
    "    percentage_of_complex_words = 100*no_of_complex_words/no_of_words\n",
    "\n",
    "# 9.FOG INDEX\n",
    "    fog_index = 0.4 * (avg_sentence_length + percentage_of_complex_words)\n",
    "\n",
    "# 10.AVG NUMBER OF WORDS PER SENTENCE\n",
    "    avg_no_of_words_per_sentence = avg_sentence_length\n",
    "        \n",
    "# 11.SYLLABLE PER WORD\n",
    "    syllables_per_word = total_syllables/no_of_words\n",
    "    \n",
    "# 12.PERSONAL PRONOUNS\n",
    "    no_of_pronouns = 0\n",
    "    personal_pronouns = ['i','me','mine','you','yours','he','him','his','she','her','hers','it','we','us','ours','they','them','theirs']\n",
    "    for word in word_tokens:\n",
    "        if word in personal_pronouns:\n",
    "            no_of_pronouns += 1\n",
    "            \n",
    "# 13.AVG WORD LENGTH\n",
    "    total_letters = 0\n",
    "    for word in clean_words:\n",
    "        total_letters += len(word)\n",
    "\n",
    "    avg_word_length = total_letters/no_of_words\n",
    "    \n",
    "    variable_names = [positive_score, negative_score, polarity_score, \n",
    "        subjectivity_score, avg_sentence_length, \n",
    "        percentage_of_complex_words, fog_index,\n",
    "        avg_no_of_words_per_sentence, no_of_complex_words, no_of_words,  \n",
    "        syllables_per_word, no_of_pronouns, avg_word_length]\n",
    "    \n",
    "    for (column,variable) in zip(column_names,variable_names):\n",
    "        output_df.at[i,column] = variable\n",
    "        output_df = output_df.round(2)\n",
    "        \n",
    "output_df.to_excel('Data/output.xlsx')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}