extract_googlemap_reviews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bb05a9d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.chrome.service import Service as ChromeService\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "import pandas as pd\n",
    "import time\n",
    "import re\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f1e888a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_language(driver):\n",
    "    # Click the \"菜單\" button\n",
    "    menu_button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, \"//button[contains(@aria-label, '菜單')]\")))\n",
    "    menu_button.click()\n",
    "\n",
    "    # Click the \"語言\" button\n",
    "    language_button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, \"//button[contains(@jsaction, 'settings.languages')]\")))\n",
    "    language_button.click()\n",
    "\n",
    "    # Switch to English\n",
    "    english_button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, \"//a[contains(text(), '‪English (United States)‬')]\")))\n",
    "    english_button.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8c8a449c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_webpage(driver):\n",
    "    # Find the \"Reviews\" button and click on it\n",
    "    reviews_button= WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, \"//button[contains(@aria-label, 'Reviews')]\")))\n",
    "    reviews_button.click()\n",
    "\n",
    "    # Scroll down the webpage\n",
    "    scroll = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, \"/html/body/div[3]/div[9]/div[9]/div/div/div[1]/div[2]/div/div[1]/div/div/div[3]\")))\n",
    "    for i in range(2):\n",
    "        scroll.send_keys(Keys.END)\n",
    "        time.sleep(1)\n",
    "\n",
    "    try:\n",
    "        # Wait for the \"More\" button to be present\n",
    "        more_buttons = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, \"//button[contains(text(), 'More')]\")))\n",
    "        # Click on all \"More\" button\n",
    "        for button in more_buttons:\n",
    "            button.click()\n",
    "    except TimeoutException:\n",
    "        pass "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf5e76b9",
   "metadata": {},
   "source": [
    "### Sample the relevant data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "aa3e2036",
   "metadata": {},
   "outputs": [],
   "source": [
    "google_info = pd.read_csv(\"googlemaps_info.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afb627d8",
   "metadata": {},
   "source": [
    "### Sample based on general location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6c50491c",
   "metadata": {},
   "outputs": [],
   "source": [
    "boundaries = {\"North\":22.3, \"South\":22.27, \"East\":114.14, \"West\":114.11} # Western District"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ccc6db1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_row = []\n",
    "for index in range(len(google_info)):\n",
    "    current_coor = google_info[\"coordinates\"][index]\n",
    "    current_coor = current_coor.replace(\"'\", \"\\\"\")\n",
    "    convert_coor = json.loads(current_coor)\n",
    "    \n",
    "    lat = convert_coor[\"lat\"]\n",
    "    long = convert_coor[\"lng\"]\n",
    "\n",
    "    if boundaries[\"South\"] <= lat <= boundaries[\"North\"] and boundaries[\"West\"] <= long <= boundaries[\"East\"]:\n",
    "        pass\n",
    "    else:\n",
    "        drop_row.append(index)\n",
    "western_places = google_info.drop(drop_row).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "38c7968a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>location_id</th>\n",
       "      <th>location</th>\n",
       "      <th>rating</th>\n",
       "      <th>num_rating</th>\n",
       "      <th>price_level</th>\n",
       "      <th>url</th>\n",
       "      <th>types</th>\n",
       "      <th>coordinates</th>\n",
       "      <th>top5reviews</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_33</td>\n",
       "      <td>starbucks  香港大學綜合大樓地下2號舖</td>\n",
       "      <td>3.8</td>\n",
       "      <td>93</td>\n",
       "      <td>2</td>\n",
       "      <td>https://maps.google.com/?cid=17336784663903124052</td>\n",
       "      <td>['cafe', 'food', 'store', 'point_of_interest',...</td>\n",
       "      <td>{'lat': 22.2831435, 'lng': 114.1358022}</td>\n",
       "      <td>[{'author_name': 'Coey', 'author_url': 'https:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_48</td>\n",
       "      <td>starbucks The Henry*</td>\n",
       "      <td>3.9</td>\n",
       "      <td>145</td>\n",
       "      <td>2</td>\n",
       "      <td>https://maps.google.com/?cid=7986681234539092847</td>\n",
       "      <td>['cafe', 'store', 'food', 'point_of_interest',...</td>\n",
       "      <td>{'lat': 22.2873098, 'lng': 114.1385283}</td>\n",
       "      <td>[{'author_name': 'Jack Poon', 'author_url': 'h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_93</td>\n",
       "      <td>starbucks Imperial Kennedy</td>\n",
       "      <td>3.7</td>\n",
       "      <td>220</td>\n",
       "      <td>2</td>\n",
       "      <td>https://maps.google.com/?cid=7983266056879862977</td>\n",
       "      <td>['cafe', 'food', 'point_of_interest', 'store',...</td>\n",
       "      <td>{'lat': 22.2825974, 'lng': 114.1287074}</td>\n",
       "      <td>[{'author_name': 'Mugi Yamamoto', 'author_url'...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  location_id                    location  rating  num_rating  price_level   \n",
       "0       id_33    starbucks  香港大學綜合大樓地下2號舖     3.8          93            2  \\\n",
       "1       id_48        starbucks The Henry*     3.9         145            2   \n",
       "2       id_93  starbucks Imperial Kennedy     3.7         220            2   \n",
       "\n",
       "                                                 url   \n",
       "0  https://maps.google.com/?cid=17336784663903124052  \\\n",
       "1   https://maps.google.com/?cid=7986681234539092847   \n",
       "2   https://maps.google.com/?cid=7983266056879862977   \n",
       "\n",
       "                                               types   \n",
       "0  ['cafe', 'food', 'store', 'point_of_interest',...  \\\n",
       "1  ['cafe', 'store', 'food', 'point_of_interest',...   \n",
       "2  ['cafe', 'food', 'point_of_interest', 'store',...   \n",
       "\n",
       "                               coordinates   \n",
       "0  {'lat': 22.2831435, 'lng': 114.1358022}  \\\n",
       "1  {'lat': 22.2873098, 'lng': 114.1385283}   \n",
       "2  {'lat': 22.2825974, 'lng': 114.1287074}   \n",
       "\n",
       "                                         top5reviews  \n",
       "0  [{'author_name': 'Coey', 'author_url': 'https:...  \n",
       "1  [{'author_name': 'Jack Poon', 'author_url': 'h...  \n",
       "2  [{'author_name': 'Mugi Yamamoto', 'author_url'...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "western_places"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba2327d2",
   "metadata": {},
   "source": [
    "### Extract the reviews from googlemap webpage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e4083a5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extract for - starbucks  香港大學綜合大樓地下2號舖\n",
      "Extract for - starbucks The Henry*\n",
      "Extract for - starbucks Imperial Kennedy\n"
     ]
    }
   ],
   "source": [
    "url_link = western_places[\"url\"].tolist()\n",
    "\n",
    "allreview_list = []\n",
    "location_id_list = []\n",
    "for link in range(len(url_link)):\n",
    "    driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "\n",
    "    url = url_link[link]\n",
    "    print(f'Extract for - {western_places[\"location\"][link]}')\n",
    "\n",
    "    driver.get(url)\n",
    "    \n",
    "    change_language(driver)\n",
    "    prepare_webpage(driver)\n",
    "    \n",
    "    # Get the element that contains review data\n",
    "    elements = driver.find_elements_by_xpath('//*[@id=\"QA0Szd\"]/div/div/div[1]/div[2]/div/div[1]/div/div/div[3]/div[9]')\n",
    "\n",
    "    for review in elements:\n",
    "        user_list = review.find_elements_by_class_name(\"WEBjve\")\n",
    "        rating_list = review.find_elements_by_class_name(\"kvMYJc\")\n",
    "        ilya_list = review.find_elements_by_class_name(\"rsqaWe\")\n",
    "        review_list = review.find_elements_by_class_name(\"wiI7pd\")\n",
    "\n",
    "    # Extract the data\n",
    "    for i in range(len(rating_list)):\n",
    "        allreview_list.append(json.dumps({\"author_name\":user_list[i].get_attribute(\"aria-label\").split(\" \", 2)[2],\n",
    "                            \"rating\":rating_list[i].get_attribute(\"aria-label\"), \n",
    "                            \"review_date\":ilya_list[i].text,\n",
    "                            \"review\":review_list[i].text}))\n",
    "        location_id_list.append(western_places[\"location_id\"][link])\n",
    "\n",
    "    \n",
    "    # Close the browser window\n",
    "    driver.quit()\n",
    "    \n",
    "\n",
    "data_dict = {\"location_id\":location_id_list, \"moreReview\":allreview_list}\n",
    "data_df = pd.DataFrame(data = data_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c23e7ccd",
   "metadata": {},
   "source": [
    "### Save the updated data into new csv file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4b11332d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df.to_csv(\"googlemaps_review.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4028bb8a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4dd2961",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67bcc095",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b0e4b7b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "979fb775",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1777ca0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ad9a943",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fdbd201",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}