amazon_reviews_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7b626edc",
   "metadata": {},
   "source": [
    "#### \n",
    "### <font color=\"GREEN\">Introduction to Web Scraping</font>\n",
    "# <font color=\"RED\">Amazon Reviews Scraping</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5c876637",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import packages\n",
    "import requests\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d3fa2952",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Header to set the requests as a browser requests\n",
    "headers = {\n",
    "    'authority': 'www.amazon.com',\n",
    "    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',\n",
    "    'accept-language': 'en-US,en;q=0.9,bn;q=0.8',\n",
    "    'sec-ch-ua': '\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"102\", \"Google Chrome\";v=\"102\"',\n",
    "    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f19e72fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# URL of The amazon Review page\n",
    "reviews_url = 'https://www.amazon.com/Legendary-Whitetails-Journeyman-Jacket-Tarmac/product-reviews/B013KW38RQ/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "51563925",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define Page No\n",
    "len_page = 4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf616af8",
   "metadata": {},
   "source": [
    "### <font color=\"red\">Functions</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "75b0d65d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extra Data as Html object from amazon Review page\n",
    "def reviewsHtml(url, len_page):\n",
    "    \n",
    "    # Empty List define to store all pages html data\n",
    "    soups = []\n",
    "    \n",
    "    # Loop for gather all 3000 reviews from 300 pages via range\n",
    "    for page_no in range(1, len_page + 1):\n",
    "        \n",
    "        # parameter set as page no to the requests body\n",
    "        params = {\n",
    "            'ie': 'UTF8',\n",
    "            'reviewerType': 'all_reviews',\n",
    "            'filterByStar': 'critical',\n",
    "            'pageNumber': page_no,\n",
    "        }\n",
    "        \n",
    "        # Request make for each page\n",
    "        response = requests.get(url, headers=headers)\n",
    "        \n",
    "        # Save Html object by using BeautifulSoup4 and lxml parser\n",
    "        soup = BeautifulSoup(response.text, 'lxml')\n",
    "        \n",
    "        # Add single Html page data in master soups list\n",
    "        soups.append(soup)\n",
    "        \n",
    "    return soups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8dd5f9c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Grab Reviews name, description, date, stars, title from HTML\n",
    "def getReviews(html_data):\n",
    "\n",
    "    # Create Empty list to Hold all data\n",
    "    data_dicts = []\n",
    "    \n",
    "    # Select all Reviews BOX html using css selector\n",
    "    boxes = html_data.select('div[data-hook=\"review\"]')\n",
    "    \n",
    "    # Iterate all Reviews BOX \n",
    "    for box in boxes:\n",
    "        \n",
    "        # Select Name using css selector and cleaning text using strip()\n",
    "        # If Value is empty define value with 'N/A' for all.\n",
    "        try:\n",
    "            name = box.select_one('[class=\"a-profile-name\"]').text.strip()\n",
    "        except Exception as e:\n",
    "            name = 'N/A'\n",
    "\n",
    "        try:\n",
    "            stars = box.select_one('[data-hook=\"review-star-rating\"]').text.strip().split(' out')[0]\n",
    "        except Exception as e:\n",
    "            stars = 'N/A'   \n",
    "\n",
    "        try:\n",
    "            title = box.select_one('[data-hook=\"review-title\"]').text.strip()\n",
    "        except Exception as e:\n",
    "            title = 'N/A'\n",
    "\n",
    "        try:\n",
    "            # Convert date str to dd/mm/yyy format\n",
    "            datetime_str = box.select_one('[data-hook=\"review-date\"]').text.strip().split(' on ')[-1]\n",
    "            date = datetime.strptime(datetime_str, '%B %d, %Y').strftime(\"%d/%m/%Y\")\n",
    "        except Exception as e:\n",
    "            date = 'N/A'\n",
    "\n",
    "        try:\n",
    "            description = box.select_one('[data-hook=\"review-body\"]').text.strip()\n",
    "        except Exception as e:\n",
    "            description = 'N/A'\n",
    "\n",
    "        # create Dictionary with al review data \n",
    "        data_dict = {\n",
    "            'Name' : name,\n",
    "            'Stars' : stars,\n",
    "            'Title' : title,\n",
    "            'Date' : date,\n",
    "            'Description' : description\n",
    "        }\n",
    "\n",
    "        # Add Dictionary in master empty List\n",
    "        data_dicts.append(data_dict)\n",
    "    \n",
    "    return data_dicts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30cbf1c5",
   "metadata": {},
   "source": [
    "### <font color=\"red\">Data Process</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "83731672",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Grab all HTML\n",
    "html_datas = reviewsHtml(reviews_url, len_page)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "545bb6ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Empty List to Hold all reviews data\n",
    "reviews = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a26391c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Iterate all Html page \n",
    "for html_data in html_datas:\n",
    "    \n",
    "    # Grab review data\n",
    "    review = getReviews(html_data)\n",
    "    \n",
    "    # add review data in reviews empty list\n",
    "    reviews += review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "25fd4ec7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a dataframe with reviews Data\n",
    "df_reviews = pd.DataFrame(reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "394b9257",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Stars</th>\n",
       "      <th>Title</th>\n",
       "      <th>Date</th>\n",
       "      <th>Description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jeff</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Rugged, good look and nice fit, but should you...</td>\n",
       "      <td>02/10/2018</td>\n",
       "      <td>The short answer to if you should go down from...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M. Burke</td>\n",
       "      <td>5.0</td>\n",
       "      <td>My Favorite Jacket</td>\n",
       "      <td>19/04/2023</td>\n",
       "      <td>I went through the most recent winter with thi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Jake</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Decent shirt jacket</td>\n",
       "      <td>27/04/2023</td>\n",
       "      <td>I’ve been wearing this regularly for about a m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>joe j</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Not bad. Its a good light jacket of sorts</td>\n",
       "      <td>05/04/2023</td>\n",
       "      <td>Its not really a jacket and not really a overs...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Jorge Risco</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Muy buen material, abrigados y queda con todo</td>\n",
       "      <td>24/05/2023</td>\n",
       "      <td>Muy buen material, para mi que vivo en el sur ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Christopher Baca</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Gentle Pockets</td>\n",
       "      <td>01/05/2023</td>\n",
       "      <td>The pocket lining is a very gentle polyester-e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Dalton J.</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Fits great just a little lose around the chest</td>\n",
       "      <td>26/05/2023</td>\n",
       "      <td>Worth the money</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Dante Sparda</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Looks just like the picture</td>\n",
       "      <td>29/05/2023</td>\n",
       "      <td>light weight, comfortable and looks good. keep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Dave Warrick</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Absolutely love this jacket!</td>\n",
       "      <td>21/04/2023</td>\n",
       "      <td>After initially seeing this jacket and falling...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Megan&amp;Jordan</td>\n",
       "      <td>5.0</td>\n",
       "      <td>fits true to size</td>\n",
       "      <td>31/05/2023</td>\n",
       "      <td>this jacket was very comfy and warm perfect fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Jeff</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Rugged, good look and nice fit, but should you...</td>\n",
       "      <td>02/10/2018</td>\n",
       "      <td>The short answer to if you should go down from...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>M. Burke</td>\n",
       "      <td>5.0</td>\n",
       "      <td>My Favorite Jacket</td>\n",
       "      <td>19/04/2023</td>\n",
       "      <td>I went through the most recent winter with thi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Jake</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Decent shirt jacket</td>\n",
       "      <td>27/04/2023</td>\n",
       "      <td>I’ve been wearing this regularly for about a m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>joe j</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Not bad. Its a good light jacket of sorts</td>\n",
       "      <td>05/04/2023</td>\n",
       "      <td>Its not really a jacket and not really a overs...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Jorge Risco</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Muy buen material, abrigados y queda con todo</td>\n",
       "      <td>24/05/2023</td>\n",
       "      <td>Muy buen material, para mi que vivo en el sur ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Christopher Baca</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Gentle Pockets</td>\n",
       "      <td>01/05/2023</td>\n",
       "      <td>The pocket lining is a very gentle polyester-e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Dalton J.</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Fits great just a little lose around the chest</td>\n",
       "      <td>26/05/2023</td>\n",
       "      <td>Worth the money</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Dante Sparda</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Looks just like the picture</td>\n",
       "      <td>29/05/2023</td>\n",
       "      <td>light weight, comfortable and looks good. keep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Dave Warrick</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Absolutely love this jacket!</td>\n",
       "      <td>21/04/2023</td>\n",
       "      <td>After initially seeing this jacket and falling...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Megan&amp;Jordan</td>\n",
       "      <td>5.0</td>\n",
       "      <td>fits true to size</td>\n",
       "      <td>31/05/2023</td>\n",
       "      <td>this jacket was very comfy and warm perfect fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Jeff</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Rugged, good look and nice fit, but should you...</td>\n",
       "      <td>02/10/2018</td>\n",
       "      <td>The short answer to if you should go down from...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>M. Burke</td>\n",
       "      <td>5.0</td>\n",
       "      <td>My Favorite Jacket</td>\n",
       "      <td>19/04/2023</td>\n",
       "      <td>I went through the most recent winter with thi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Jake</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Decent shirt jacket</td>\n",
       "      <td>27/04/2023</td>\n",
       "      <td>I’ve been wearing this regularly for about a m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>joe j</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Not bad. Its a good light jacket of sorts</td>\n",
       "      <td>05/04/2023</td>\n",
       "      <td>Its not really a jacket and not really a overs...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Jorge Risco</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Muy buen material, abrigados y queda con todo</td>\n",
       "      <td>24/05/2023</td>\n",
       "      <td>Muy buen material, para mi que vivo en el sur ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Christopher Baca</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Gentle Pockets</td>\n",
       "      <td>01/05/2023</td>\n",
       "      <td>The pocket lining is a very gentle polyester-e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Dalton J.</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Fits great just a little lose around the chest</td>\n",
       "      <td>26/05/2023</td>\n",
       "      <td>Worth the money</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Dante Sparda</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Looks just like the picture</td>\n",
       "      <td>29/05/2023</td>\n",
       "      <td>light weight, comfortable and looks good. keep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Dave Warrick</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Absolutely love this jacket!</td>\n",
       "      <td>21/04/2023</td>\n",
       "      <td>After initially seeing this jacket and falling...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Megan&amp;Jordan</td>\n",
       "      <td>5.0</td>\n",
       "      <td>fits true to size</td>\n",
       "      <td>31/05/2023</td>\n",
       "      <td>this jacket was very comfy and warm perfect fo...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Name Stars                                              Title  \\\n",
       "0               Jeff   4.0  Rugged, good look and nice fit, but should you...   \n",
       "1           M. Burke   5.0                                 My Favorite Jacket   \n",
       "2               Jake   4.0                                Decent shirt jacket   \n",
       "3              joe j   4.0          Not bad. Its a good light jacket of sorts   \n",
       "4        Jorge Risco   5.0      Muy buen material, abrigados y queda con todo   \n",
       "5   Christopher Baca   4.0                                     Gentle Pockets   \n",
       "6          Dalton J.   4.0     Fits great just a little lose around the chest   \n",
       "7       Dante Sparda   5.0                        Looks just like the picture   \n",
       "8       Dave Warrick   5.0                       Absolutely love this jacket!   \n",
       "9       Megan&Jordan   5.0                                  fits true to size   \n",
       "10              Jeff   4.0  Rugged, good look and nice fit, but should you...   \n",
       "11          M. Burke   5.0                                 My Favorite Jacket   \n",
       "12              Jake   4.0                                Decent shirt jacket   \n",
       "13             joe j   4.0          Not bad. Its a good light jacket of sorts   \n",
       "14       Jorge Risco   5.0      Muy buen material, abrigados y queda con todo   \n",
       "15  Christopher Baca   4.0                                     Gentle Pockets   \n",
       "16         Dalton J.   4.0     Fits great just a little lose around the chest   \n",
       "17      Dante Sparda   5.0                        Looks just like the picture   \n",
       "18      Dave Warrick   5.0                       Absolutely love this jacket!   \n",
       "19      Megan&Jordan   5.0                                  fits true to size   \n",
       "20              Jeff   4.0  Rugged, good look and nice fit, but should you...   \n",
       "21          M. Burke   5.0                                 My Favorite Jacket   \n",
       "22              Jake   4.0                                Decent shirt jacket   \n",
       "23             joe j   4.0          Not bad. Its a good light jacket of sorts   \n",
       "24       Jorge Risco   5.0      Muy buen material, abrigados y queda con todo   \n",
       "25  Christopher Baca   4.0                                     Gentle Pockets   \n",
       "26         Dalton J.   4.0     Fits great just a little lose around the chest   \n",
       "27      Dante Sparda   5.0                        Looks just like the picture   \n",
       "28      Dave Warrick   5.0                       Absolutely love this jacket!   \n",
       "29      Megan&Jordan   5.0                                  fits true to size   \n",
       "\n",
       "          Date                                        Description  \n",
       "0   02/10/2018  The short answer to if you should go down from...  \n",
       "1   19/04/2023  I went through the most recent winter with thi...  \n",
       "2   27/04/2023  I’ve been wearing this regularly for about a m...  \n",
       "3   05/04/2023  Its not really a jacket and not really a overs...  \n",
       "4   24/05/2023  Muy buen material, para mi que vivo en el sur ...  \n",
       "5   01/05/2023  The pocket lining is a very gentle polyester-e...  \n",
       "6   26/05/2023                                    Worth the money  \n",
       "7   29/05/2023  light weight, comfortable and looks good. keep...  \n",
       "8   21/04/2023  After initially seeing this jacket and falling...  \n",
       "9   31/05/2023  this jacket was very comfy and warm perfect fo...  \n",
       "10  02/10/2018  The short answer to if you should go down from...  \n",
       "11  19/04/2023  I went through the most recent winter with thi...  \n",
       "12  27/04/2023  I’ve been wearing this regularly for about a m...  \n",
       "13  05/04/2023  Its not really a jacket and not really a overs...  \n",
       "14  24/05/2023  Muy buen material, para mi que vivo en el sur ...  \n",
       "15  01/05/2023  The pocket lining is a very gentle polyester-e...  \n",
       "16  26/05/2023                                    Worth the money  \n",
       "17  29/05/2023  light weight, comfortable and looks good. keep...  \n",
       "18  21/04/2023  After initially seeing this jacket and falling...  \n",
       "19  31/05/2023  this jacket was very comfy and warm perfect fo...  \n",
       "20  02/10/2018  The short answer to if you should go down from...  \n",
       "21  19/04/2023  I went through the most recent winter with thi...  \n",
       "22  27/04/2023  I’ve been wearing this regularly for about a m...  \n",
       "23  05/04/2023  Its not really a jacket and not really a overs...  \n",
       "24  24/05/2023  Muy buen material, para mi que vivo en el sur ...  \n",
       "25  01/05/2023  The pocket lining is a very gentle polyester-e...  \n",
       "26  26/05/2023                                    Worth the money  \n",
       "27  29/05/2023  light weight, comfortable and looks good. keep...  \n",
       "28  21/04/2023  After initially seeing this jacket and falling...  \n",
       "29  31/05/2023  this jacket was very comfy and warm perfect fo...  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "53a4b504",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save data\n",
    "df_reviews.to_csv('reviews.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4da15509",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}