test_crawler.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scrapy\n",
    "import logging\n",
    "import re\n",
    "import pandas as pd\n",
    "from datetime import date\n",
    "from scrapy.crawler import CrawlerProcess\n",
    "from scrapy.utils.project import get_project_settings\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PatientSpider(scrapy.Spider):\n",
    "    name = \"patient_check\"\n",
    "    #start_urls = ['https://ncov.moh.gov.vn/web/guest/trang-chu']\n",
    "    start_urls = []\n",
    "    rows = []\n",
    "    out = 'MOH_PatientTable_' + date.today().strftime(\"%d_%m\") + '.csv'\n",
    "\n",
    "    def __init__(self):\n",
    "        n = input('Bạn muốn lấy bệnh nhân tới trang thứ mấy (100 bệnh nhân/trang)? ')\n",
    "        n_pages=int(n)\n",
    "        self.start_urls = ['https://ncov.moh.gov.vn/web/guest/trang-chu?p_p_id=corona_trangchu_top_CoronaTrangchuTopPortlet_INSTANCE_RrVAbIFIPL7v&p_p_lifecycle=0&p_p_state=normal&p_p_mode=view&_corona_trangchu_top_CoronaTrangchuTopPortlet_INSTANCE_RrVAbIFIPL7v_delta=100&_corona_trangchu_top_CoronaTrangchuTopPortlet_INSTANCE_RrVAbIFIPL7v_resetCur=false&_corona_trangchu_top_CoronaTrangchuTopPortlet_INSTANCE_RrVAbIFIPL7v_cur='+str(i) for i in range(1,n_pages)]    \n",
    "        logging.getLogger('scrapy').propagate = False\n",
    "\n",
    "    def close(self, reason):\n",
    "        df_out = pd.DataFrame(self.rows)[::-1]\n",
    "        #df_out['Patient'] = df_out['Patient'].str[2:].apply(lambda x: re.sub(r'^[0-9]','',x))#.astype(int)\n",
    "        df_out['Patient'] = df_out['Patient'].apply(lambda x: re.sub(r'^[0-9]+','',x), convert_dtype=True)#.astype(int)\n",
    "        df_out['Patient'] = df_out['Patient'].str[2:].replace({',':'', ' ':'', 'B':'','N':''}, regex=True)\n",
    "        # fix bugs\n",
    "        df_out.loc[df_out['Patient'] == '1372)', 'Patient'] = '1372'\n",
    "        df_out.loc[df_out['Patient'] == '92859285', 'Patient'] = '9285'\n",
    "        df_out['Patient'] = pd.to_numeric(df_out['Patient'])\n",
    "        df_out = df_out.sort_values('Patient', ascending = False)\n",
    "        #print(df_out.dtypes)\n",
    "        df_out.sort_values(by=['Patient'],inplace=True,ascending=True)\n",
    "        df_out.to_csv(self.out,index=False)\n",
    "        print(\"Hoàn tất. Dữ liệu đã được xuất ra file \" + self.out)\n",
    "\n",
    "    def parse(self, response):\n",
    "        for patient in response.xpath('//*[@class=\"table table-striped table-covid19\"]//tbody/tr'):\n",
    "            p_dict={'Patient': patient.xpath('td[1]//text()').extract_first(),#.rstrip(),\n",
    "                    'Age': patient.xpath('td[2]//text()').extract_first(),\n",
    "                    'Location': patient.xpath('td[3]//text()').extract_first(),\n",
    "                    'Status': patient.xpath('td[4]//text()').extract_first(),\n",
    "                    'Nationality': patient.xpath('td[5]//text()').extract_first()}\n",
    "            if p_dict['Age']:\n",
    "                self.rows += [p_dict]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-06-15 08:14:34 [scrapy.utils.log] INFO: Scrapy 2.5.0 started (bot: scrapybot)\n",
      "2021-06-15 08:14:34 [scrapy.utils.log] INFO: Versions: lxml 4.3.2.0, libxml2 2.9.9, cssselect 1.1.0, parsel 1.6.0, w3lib 1.22.0, Twisted 21.2.0, Python 3.7.3 (default, Mar 27 2019, 16:54:48) - [Clang 4.0.1 (tags/RELEASE_401/final)], pyOpenSSL 19.0.0 (OpenSSL 1.1.1f  31 Mar 2020), cryptography 2.6.1, Platform Darwin-18.7.0-x86_64-i386-64bit\n",
      "2021-06-15 08:14:34 [scrapy.utils.log] DEBUG: Using reactor: twisted.internet.selectreactor.SelectReactor\n",
      "2021-06-15 08:14:34 [scrapy.crawler] INFO: Overridden settings:\n",
      "{}\n",
      "2021-06-15 08:14:34 [scrapy.extensions.telnet] INFO: Telnet Password: a6feb4023728c51d\n",
      "2021-06-15 08:14:34 [scrapy.middleware] INFO: Enabled extensions:\n",
      "['scrapy.extensions.corestats.CoreStats',\n",
      " 'scrapy.extensions.telnet.TelnetConsole',\n",
      " 'scrapy.extensions.memusage.MemoryUsage',\n",
      " 'scrapy.extensions.logstats.LogStats']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bạn muốn lấy bệnh nhân tới trang thứ mấy (100 bệnh nhân/trang)? 109\n",
      "Hoàn tất. Dữ liệu đã được xuất ra file MOH_PatientTable_15_06.csv\n"
     ]
    }
   ],
   "source": [
    "process = CrawlerProcess(get_project_settings())\n",
    "process.crawl(PatientSpider)\n",
    "process.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "re.match(r'^([\\s\\d]+)$', 'BN1372')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1372']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(\"\\d+\", ' BN1372 ,')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}