Workshop 4 - Poprawność zachowania.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# robots.txt"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "User-agent: *\n",
    "\n",
    "#Sorting parameters\n",
    "Disallow: /*order=*\n",
    "Disallow: /*galleryView=*\n",
    "Disallow: /*sort=*\n",
    "\n",
    "#Other comments:\n",
    "Disallow: *?sr=8*\n",
    "Disallow: /f-pbfvb/*\n",
    "Disallow: *?be=*\n",
    "Disallow: *?geoTag=*\n",
    "Disallow: /api/*\n",
    "Disallow: /306629115/MX_*\n",
    "Disallow: /bolt-2dot0-frontend$*\n",
    "Disallow: /7162/Gumtree_*\n",
    "Disallow: /169054071/Gumtree_*\n",
    "Disallow: /306629115/iBazar_MX*\n",
    "Disallow: /306629115/AR_*\n",
    "\n",
    "#Sitemaps\n",
    "Sitemap: https://www.gumtree.pl/vip_index.xml\n",
    "Sitemap: https://www.gumtree.pl/sitemap_index.xml\n",
    "Sitemap: https://www.gumtree.pl/sitemap_loccat.xml\n",
    "Sitemap: https://www.gumtree.pl/sitemap_loccatatt.xml\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scraper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scrapy\n",
    "import scrapy.crawler as crawler\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "from scrapy.crawler import CrawlerProcess\n",
    "\n",
    "class GumtreeApartmentsSpider(scrapy.Spider):\n",
    "    name = 'gumtreeapartmentsspider'\n",
    "    start_urls = [\n",
    "        'https://www.gumtree.pl/s-mieszkania-i-domy-sprzedam-i-kupie/mazowieckie/page-'+str(i)+'/v1c9073l3200001p'+str(i)  for i in range(2,4)\n",
    "        ]\n",
    "    start_urls.append(\n",
    "        'https://www.gumtree.pl/s-mieszkania-i-domy-sprzedam-i-kupie/mazowieckie/v1c9073l3200001p1'\n",
    "    )\n",
    "    found_apartments = []\n",
    "   \n",
    "    custom_settings = {\n",
    "        'DOWNLOAD_DELAY': '4.0', #obejście Fail2Ban\n",
    "        'ROBOTSTXT_OBEY' : True\n",
    "    }\n",
    "\n",
    "    top_url = 'https://www.gumtree.pl'\n",
    "    def parse(self, response):\n",
    "        self.logger.info('Got successful response from {}'.format(response.url))\n",
    "        soup = BeautifulSoup(response.body, 'lxml')\n",
    "        link_tabs = soup.findAll(\"div\", {\"class\": \"result-link\"})\n",
    "        item_urls = []\n",
    "        next_urls = []\n",
    "        for tab in link_tabs:\n",
    "            hrefs = tab.findAll(\"a\", {\"class\": \"href-link\"})\n",
    "            for h in hrefs:\n",
    "                item_urls.append(self.top_url + h[\"href\"]) #dopisuje 'https://www.gumtree.pl', bo otrzymany adres jest względny, przygotowuje listę wszystkich linków\n",
    "            \n",
    "        for item_url in item_urls:\n",
    "            yield scrapy.Request(item_url, self.parse_item)\n",
    "        \n",
    "    def parse_item(self, response): #item_url - odwiedzanie strony, #self.parse_item - przetworzenie przy pomocy funkcji\n",
    "        #Courtesy of Mr Sebastian Muraszewski\n",
    "        found_apartments = []\n",
    "        soup = BeautifulSoup(response.text, 'html.parser') \n",
    "        apartments = soup.find('div', {'class': 'vip-header-and-details'})\n",
    "        apartment_details = dict()\n",
    "        apartment_details['Nazwa ogłoszenia'] = apartments.find('span', class_ = 'myAdTitle').text\n",
    "        apartment_details['Cena'] = apartments.find('span', class_ = 'amount').text.replace(\"\\xa0\",\" \") #replace w celu usunięcia twardej spacji\n",
    "        container = soup.find('ul', class_ = 'selMenu') #zebranie informacji z ramki do kontenera\n",
    "        \n",
    "        nazwy = container.findAll('span', class_ = 'name')\n",
    "        szczegoly = container.findAll('span', class_ = 'value')\n",
    "        apartment_details.update({name.text: value.text.strip() for name, value in zip(nazwy, szczegoly)}) #dodawanie elementów z ramki do listy i usunięcie whitespace'ów\n",
    "        \n",
    "        with open('plik.csv', 'a') as f:\n",
    "            v = apartment_details.values()\n",
    "            f.write('\\t'.join(v))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "process = CrawlerProcess({\n",
    "    'USER_AGENT': 'Testing MyCrawler (michal.korzycki@gmail.com)'\n",
    "})\n",
    "process.crawl(GumtreeApartmentsSpider)\n",
    "process.start()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}