scrapping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Web Scrapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import warnings\n",
    "import time\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# to ignore all warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinkCollection:\n",
    "    \n",
    "    def __init__(self, url:str) -> None:\n",
    "        self.__url_link = url\n",
    "        self.__collected_links = None\n",
    "    \n",
    "    def collect_links(self):\n",
    "        start = time.time()\n",
    "        self.__collected_links = []\n",
    "        url = self.__url_link\n",
    "        prev_len = 0\n",
    "        integer = str(1)\n",
    "        while True:\n",
    "            # modifying the link for next page\n",
    "            end = url.split(\"-\")[-1]\n",
    "            end = end.split(\".\")\n",
    "            end[0] = integer\n",
    "            end = \".\".join(end)\n",
    "            url = url.split(\"-\")[:-1]\n",
    "            url.append(end)\n",
    "            url = \"-\".join(url)\n",
    "            # let me know \n",
    "            # if int(integer) % 50 == 0:\n",
    "            #     print(f\"Total collected links: {len(self.__collected_links)}\")\n",
    "            \n",
    "            # parse the data\n",
    "            # print(integer)\n",
    "            \n",
    "            html_data = requests.get(url).text\n",
    "            soup = BeautifulSoup(html_data, 'lxml')\n",
    "            html_data = soup.find_all(\"a\", class_=\"_7ac32433\", href=True)\n",
    "            if len(html_data) == 0:\n",
    "                end = time.time()\n",
    "                print(f\"Total Time: {end-start} sec\")\n",
    "                return\n",
    "            # collect the links\n",
    "            for link in soup.find_all(\"a\", class_=\"_7ac32433\"):\n",
    "                self.__collected_links.append(\"\".join([\"zameen.com\", link['href']]))\n",
    "            integer = int(integer)\n",
    "            integer += 1\n",
    "            integer = str(integer)\n",
    "    \n",
    "    def return_links(self):\n",
    "        return self.__collected_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://www.zameen.com/all_locations/Islamabad-3-1-9.html\"\n",
    "html_data = requests.get(url).text\n",
    "soup = BeautifulSoup(html_data, 'lxml')\n",
    "html_data = soup.find_all(\"ul\", class_=\"line-list\")\n",
    "urls_list = []\n",
    "urls_list_city = []\n",
    "for li in html_data:\n",
    "    a_tag = li.find_all(\"a\")\n",
    "    for link in a_tag:\n",
    "        urls_list_city.append(link.text)\n",
    "        urls_list.append(link['href'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_urls = {}\n",
    "counter = 0\n",
    "for url, url_city in zip(urls_list, urls_list_city):\n",
    "    lc = LinkCollection(url)\n",
    "    print(f\"Links for the city: {url_city}\")\n",
    "    lc.collect_links()\n",
    "    all_urls[url_city] = lc.return_links()\n",
    "    print(f\"Total collected links for {url_city}: {len(all_urls[url_city])}\")\n",
    "    counter += len(all_urls[url_city])\n",
    "    print(f\"Total collected links: {counter}\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"isb_house_buy.json\", \"wb\") as f:\n",
    "    pickle.dump(all_urls, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "isb_urls = {}\n",
    "with open(\"isb_house_buy.json\", \"rb\") as f:\n",
    "    isb_urls = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_urls = []\n",
    "for key in isb_urls:\n",
    "    all_urls.extend(isb_urls[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(all_urls)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collecting Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# urls_list = pd.read_csv(filepath_or_buffer=\"Lahore_House_Buy_Propery_Links.csv\")\n",
    "# urls_list = urls_list[\"0\"].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collect_data(url:str):\n",
    "    data = {}\n",
    "    html_data = requests.get(url).text\n",
    "    soup = BeautifulSoup(html_data, 'html.parser')\n",
    "    property_name = soup.find(\"h1\", class_ = \"_64bb5b3b\")\n",
    "    data['property_name'] = property_name.get_text()\n",
    "    details_metadata = soup.find_all(\"span\", class_=\"_3af7fa95\")\n",
    "    details_data = soup.find_all(\"span\", class_=\"_812aa185\")\n",
    "    for detail_meta, data_ in zip(details_metadata,details_data):\n",
    "        data[detail_meta.get_text()] = data_.get_text()\n",
    "    description = soup.find(\"span\", class_=\"_2a806e1e\").get_text()\n",
    "    data['description'] = description\n",
    "    \n",
    "    amenities = soup.find_all(\"li\", class_=\"_579bdb8a\")\n",
    "    for amenity in amenities:\n",
    "        amenity_metadata = amenity.find(\"div\", class_=\"a152770c\").text\n",
    "        # print(amenity_metadata)\n",
    "        amenity_data = amenity.find_all(\"span\", class_=\"_17984a2c\")\n",
    "        data[amenity_metadata] = list()\n",
    "        for single_amenity_data in amenity_data:\n",
    "            # print(single_amenity_data.text)\n",
    "            data[amenity_metadata].append(single_amenity_data.text)\n",
    "        # print()\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "useless_links = []\n",
    "whole_data = {}\n",
    "counter = 0\n",
    "start = time.time()\n",
    "for url in all_urls[200:300]:\n",
    "    url = \" https://\" + url\n",
    "    try:\n",
    "        url_data = collect_data(url)\n",
    "    except AttributeError:\n",
    "        useless_links.append(url)\n",
    "        continue\n",
    "    whole_data[url] = url_data\n",
    "    counter += 1\n",
    "    if counter % 100 == 0:\n",
    "        end = time.time()\n",
    "        print(f\"Data of {counter} links has been scrapped.\")\n",
    "        print(f\"No. of useless links: {len(useless_links)}\")\n",
    "        print(f\"Total Time: {end-start}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(whole_data).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df1 = pd.read_csv(\"Islamabad_House_Buy.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"Islamabad_House_Buy.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df, df1], axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_excel(\"Data/Rawalpindi_House_Buy.xlsx\", index=None)\n",
    "df.to_csv(\"Data/Rawalpindi_House_Buy.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"Islamabad_House_Buy.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"Rwp_House_Buy.csv\")\n",
    "df1 = pd.read_csv(\"Rwp_House_Buy_1.csv\")\n",
    "df2 = pd.read_csv(\"Rwp_House_Buy_2.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df, df1, df2], axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.duplicated().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>property_name</th>\n",
       "      <th>Type</th>\n",
       "      <th>Price</th>\n",
       "      <th>Location</th>\n",
       "      <th>Bath(s)</th>\n",
       "      <th>Area</th>\n",
       "      <th>Purpose</th>\n",
       "      <th>Bedroom(s)</th>\n",
       "      <th>Added</th>\n",
       "      <th>...</th>\n",
       "      <th>Rooms</th>\n",
       "      <th>Community Features</th>\n",
       "      <th>Nearby Locations and Other Facilities</th>\n",
       "      <th>Other Facilities\\r\\n</th>\n",
       "      <th>Business and Communication</th>\n",
       "      <th>Healthcare Recreational</th>\n",
       "      <th>Plot Features</th>\n",
       "      <th>Initial Amount</th>\n",
       "      <th>Monthly Installment</th>\n",
       "      <th>Remaining Installments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1953</th>\n",
       "      <td>https://zameen.com/Property/bahria_town_rawal...</td>\n",
       "      <td>Brand New Designer House For Sale Bharia Town ...</td>\n",
       "      <td>House</td>\n",
       "      <td>PKR5.5 Crore</td>\n",
       "      <td>Bahria Town Rawalpindi, Rawalpindi, Punjab</td>\n",
       "      <td>6</td>\n",
       "      <td>10 Marla</td>\n",
       "      <td>For Sale</td>\n",
       "      <td>5</td>\n",
       "      <td>2 weeks ago</td>\n",
       "      <td>...</td>\n",
       "      <td>['Bedrooms: 5', 'Bathrooms: 6', 'Servant Quart...</td>\n",
       "      <td>['Community Lawn or Garden', 'Community Swimmi...</td>\n",
       "      <td>['Nearby Schools', 'Nearby Hospitals', 'Nearby...</td>\n",
       "      <td>['Maintenance Staff', 'Security Staff', 'Facil...</td>\n",
       "      <td>['Broadband Internet Access', 'Satellite or Ca...</td>\n",
       "      <td>['Sauna', 'Jacuzzi']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9514</th>\n",
       "      <td>https://zameen.com/Property/rawalpindi_railwa...</td>\n",
       "      <td>House Is Available For Sale</td>\n",
       "      <td>House</td>\n",
       "      <td>PKR3.5 Crore</td>\n",
       "      <td>Railway Scheme 7, Rawalpindi, Punjab</td>\n",
       "      <td>6</td>\n",
       "      <td>8 Marla</td>\n",
       "      <td>For Sale</td>\n",
       "      <td>8</td>\n",
       "      <td>3 years ago</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343</th>\n",
       "      <td>https://zameen.com/Property/bahria_town_rawal...</td>\n",
       "      <td>Beautiful Designer House Single unit For Sale ...</td>\n",
       "      <td>House</td>\n",
       "      <td>PKR10.75 Crore</td>\n",
       "      <td>Bahria Town Rawalpindi, Rawalpindi, Punjab</td>\n",
       "      <td>6</td>\n",
       "      <td>1 Kanal</td>\n",
       "      <td>For Sale</td>\n",
       "      <td>5</td>\n",
       "      <td>7 hours ago</td>\n",
       "      <td>...</td>\n",
       "      <td>['Bedrooms: 5', 'Bathrooms: 6', 'Servant Quart...</td>\n",
       "      <td>['Community Lawn or Garden', 'Community Swimmi...</td>\n",
       "      <td>['Nearby Schools', 'Nearby Hospitals', 'Nearby...</td>\n",
       "      <td>['Maintenance Staff', 'Security Staff', 'Facil...</td>\n",
       "      <td>['Broadband Internet Access', 'Satellite or Ca...</td>\n",
       "      <td>['Jacuzzi']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1318</th>\n",
       "      <td>https://zameen.com/Property/bahria_town_rawal...</td>\n",
       "      <td>14 Marla Corner House Brand New Designer Avail...</td>\n",
       "      <td>House</td>\n",
       "      <td>PKR7.5 Crore</td>\n",
       "      <td>Bahria Town Rawalpindi, Rawalpindi, Punjab</td>\n",
       "      <td>6</td>\n",
       "      <td>14 Marla</td>\n",
       "      <td>For Sale</td>\n",
       "      <td>5</td>\n",
       "      <td>2 days ago</td>\n",
       "      <td>...</td>\n",
       "      <td>['Bedrooms: 5', 'Bathrooms: 6', 'Servant Quart...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Nearby Schools', 'Nearby Hospitals', 'Nearby...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4431</th>\n",
       "      <td>https://zameen.com/Property/chaklala_scheme_c...</td>\n",
       "      <td>Beautiful Brand New Single Story 4.5 Marla Hou...</td>\n",
       "      <td>House</td>\n",
       "      <td>PKR1.3 Crore</td>\n",
       "      <td>Chaklala Scheme, Rawalpindi, Punjab</td>\n",
       "      <td>2</td>\n",
       "      <td>4 Marla</td>\n",
       "      <td>For Sale</td>\n",
       "      <td>2</td>\n",
       "      <td>2 weeks ago</td>\n",
       "      <td>...</td>\n",
       "      <td>['Bedrooms', 'Bathrooms', 'Servant Quarters', ...</td>\n",
       "      <td>['Community Lawn or Garden', 'Community Swimmi...</td>\n",
       "      <td>['Nearby Schools', 'Nearby Hospitals', 'Nearby...</td>\n",
       "      <td>['Maintenance Staff', 'Security Staff', 'Facil...</td>\n",
       "      <td>['Broadband Internet Access', 'Satellite or Ca...</td>\n",
       "      <td>['Lawn or Garden', 'Swimming Pool', 'Sauna', '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  index  \\\n",
       "1953   https://zameen.com/Property/bahria_town_rawal...   \n",
       "9514   https://zameen.com/Property/rawalpindi_railwa...   \n",
       "343    https://zameen.com/Property/bahria_town_rawal...   \n",
       "1318   https://zameen.com/Property/bahria_town_rawal...   \n",
       "4431   https://zameen.com/Property/chaklala_scheme_c...   \n",
       "\n",
       "                                          property_name   Type  \\\n",
       "1953  Brand New Designer House For Sale Bharia Town ...  House   \n",
       "9514                        House Is Available For Sale  House   \n",
       "343   Beautiful Designer House Single unit For Sale ...  House   \n",
       "1318  14 Marla Corner House Brand New Designer Avail...  House   \n",
       "4431  Beautiful Brand New Single Story 4.5 Marla Hou...  House   \n",
       "\n",
       "               Price                                    Location Bath(s)  \\\n",
       "1953    PKR5.5 Crore  Bahria Town Rawalpindi, Rawalpindi, Punjab       6   \n",
       "9514    PKR3.5 Crore        Railway Scheme 7, Rawalpindi, Punjab       6   \n",
       "343   PKR10.75 Crore  Bahria Town Rawalpindi, Rawalpindi, Punjab       6   \n",
       "1318    PKR7.5 Crore  Bahria Town Rawalpindi, Rawalpindi, Punjab       6   \n",
       "4431    PKR1.3 Crore         Chaklala Scheme, Rawalpindi, Punjab       2   \n",
       "\n",
       "          Area   Purpose Bedroom(s)        Added  ...  \\\n",
       "1953  10 Marla  For Sale          5  2 weeks ago  ...   \n",
       "9514   8 Marla  For Sale          8  3 years ago  ...   \n",
       "343    1 Kanal  For Sale          5  7 hours ago  ...   \n",
       "1318  14 Marla  For Sale          5   2 days ago  ...   \n",
       "4431   4 Marla  For Sale          2  2 weeks ago  ...   \n",
       "\n",
       "                                                  Rooms  \\\n",
       "1953  ['Bedrooms: 5', 'Bathrooms: 6', 'Servant Quart...   \n",
       "9514                                                NaN   \n",
       "343   ['Bedrooms: 5', 'Bathrooms: 6', 'Servant Quart...   \n",
       "1318  ['Bedrooms: 5', 'Bathrooms: 6', 'Servant Quart...   \n",
       "4431  ['Bedrooms', 'Bathrooms', 'Servant Quarters', ...   \n",
       "\n",
       "                                     Community Features  \\\n",
       "1953  ['Community Lawn or Garden', 'Community Swimmi...   \n",
       "9514                                                NaN   \n",
       "343   ['Community Lawn or Garden', 'Community Swimmi...   \n",
       "1318                                                NaN   \n",
       "4431  ['Community Lawn or Garden', 'Community Swimmi...   \n",
       "\n",
       "                  Nearby Locations and Other Facilities  \\\n",
       "1953  ['Nearby Schools', 'Nearby Hospitals', 'Nearby...   \n",
       "9514                                                NaN   \n",
       "343   ['Nearby Schools', 'Nearby Hospitals', 'Nearby...   \n",
       "1318  ['Nearby Schools', 'Nearby Hospitals', 'Nearby...   \n",
       "4431  ['Nearby Schools', 'Nearby Hospitals', 'Nearby...   \n",
       "\n",
       "                                   Other Facilities\\r\\n  \\\n",
       "1953  ['Maintenance Staff', 'Security Staff', 'Facil...   \n",
       "9514                                                NaN   \n",
       "343   ['Maintenance Staff', 'Security Staff', 'Facil...   \n",
       "1318                                                NaN   \n",
       "4431  ['Maintenance Staff', 'Security Staff', 'Facil...   \n",
       "\n",
       "                             Business and Communication  \\\n",
       "1953  ['Broadband Internet Access', 'Satellite or Ca...   \n",
       "9514                                                NaN   \n",
       "343   ['Broadband Internet Access', 'Satellite or Ca...   \n",
       "1318                                                NaN   \n",
       "4431  ['Broadband Internet Access', 'Satellite or Ca...   \n",
       "\n",
       "                                Healthcare Recreational Plot Features  \\\n",
       "1953                               ['Sauna', 'Jacuzzi']           NaN   \n",
       "9514                                                NaN           NaN   \n",
       "343                                         ['Jacuzzi']           NaN   \n",
       "1318                                                NaN           NaN   \n",
       "4431  ['Lawn or Garden', 'Swimming Pool', 'Sauna', '...           NaN   \n",
       "\n",
       "     Initial Amount Monthly Installment Remaining Installments  \n",
       "1953            NaN                 NaN                    NaN  \n",
       "9514            NaN                 NaN                    NaN  \n",
       "343             NaN                 NaN                    NaN  \n",
       "1318            NaN                 NaN                    NaN  \n",
       "4431            NaN                 NaN                    NaN  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mypython",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}