step-02-2gis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import random\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('dataset_fixed-3.csv', sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p 2gis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Обкачиваем 2GIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.common.exceptions import ElementNotVisibleException"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Chrome()\n",
    "driver.implicitly_wait(5)\n",
    "driver.get(\"https://2gis.ru/moscow\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "address_prev = None\n",
    "\n",
    "next_id = None\n",
    "\n",
    "for r in df.itertuples():\n",
    "    if next_id is not None and r.id != next_id:\n",
    "        continue\n",
    "        \n",
    "    next_id = None\n",
    "    \n",
    "    if r.address != address_prev:\n",
    "        try:\n",
    "            driver.find_element_by_name(\"search[query]\").click()\n",
    "        except ElementNotVisibleException as e:\n",
    "            driver.get(\"https://2gis.ru/moscow\")\n",
    "            driver.find_element_by_name(\"search[query]\").click()\n",
    "            \n",
    "        driver.find_element_by_name(\"search[query]\").clear()\n",
    "        # driver.find_element_by_name(\"search[query]\").send_keys(r.address)\n",
    "        driver.find_element_by_name(\"search[query]\").send_keys(f'{r.lat} {r.lng}')\n",
    "        driver.find_element_by_name(\"search[query]\").send_keys(Keys.ENTER)\n",
    "    \n",
    "        time.sleep(2 + random.random() * 0.5)\n",
    "    \n",
    "    with open(f'2gis/{r.id}.html', mode='w') as f_name:\n",
    "        f_name.write(driver.page_source)\n",
    "        \n",
    "    address_prev = r.address"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Обрабатываем скачанные странички"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "\n",
    "from operator import attrgetter, itemgetter\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class UknownCaseException(Exception):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_building(soup):\n",
    "    elem = soup.find('div', class_='_purpose_building')\n",
    "    if elem:\n",
    "        elem = elem.text.rsplit(',', 1)\n",
    "        if len(elem) == 2:\n",
    "            type_, floors = elem[0], int(re.search('\\d+', elem[1]).group(0))\n",
    "        else:\n",
    "            type_, floors = '', int(re.search('\\d+', elem[0]).group(0))\n",
    "    else:\n",
    "        raise UknownCaseException('Unknown case')\n",
    "    \n",
    "    elem = soup.find('section', class_='card__firmsInBuilding')\n",
    "    if elem:\n",
    "        elem = elem.find('div', class_='cardInfrastructureItem__counter')\n",
    "        n_orgs = int(elem.text)\n",
    "    else:\n",
    "        n_orgs = 0\n",
    "    \n",
    "    if n_orgs > 0:\n",
    "        elem = soup.find_all('li', class_='cardInfrastructureFirmsList__listItem')\n",
    "        name = '\\n'.join(map(attrgetter('text'), elem))\n",
    "    else:\n",
    "        name = ''\n",
    "    \n",
    "    rating, rating_style = -1, None\n",
    "\n",
    "    elem = soup.find('div', class_='customRating__stars')\n",
    "    if elem:\n",
    "        rating = int(next(filter(lambda s: s.startswith('_value'), elem.attrs['class'])).split('_')[-1])\n",
    "        rating_style = 'customRating'\n",
    "        \n",
    "    elem = soup.find('div', class_='rating')\n",
    "    if elem:\n",
    "        rating = int(next(filter(lambda s: s.startswith('_value'), elem.attrs['class'])).split('_')[-1])\n",
    "        rating_style = next(filter(lambda s: s.startswith('_style'), elem.attrs['class'])).split('_')[-1]\n",
    "    \n",
    "    res = {\n",
    "        'name': name,\n",
    "        'type': type_,\n",
    "        'floors': floors,\n",
    "        'norgs': n_orgs, \n",
    "    }\n",
    "    \n",
    "    if rating > 0 and rating_style:\n",
    "        res.update({'rating': rating, 'rating_style': rating_style})\n",
    "        \n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_organisation(soup):    \n",
    "    name = soup.find('h1', class_='cardHeader__headerNameText').text\n",
    "    \n",
    "    elem = soup.find('div', class_='cardHeader__headerDescriptionText')\n",
    "    type_ = elem.text if elem else ''\n",
    "    \n",
    "    elem = soup.find('div', class_='_purpose_shortbuildinginfo')\n",
    "    if elem:\n",
    "        floors = int(re.search('\\d+', elem.text).group(0))\n",
    "    else:\n",
    "        floors = 0\n",
    "    \n",
    "    elem = soup.find('div', class_='_purpose_building')\n",
    "    if elem:\n",
    "        raise UknownCaseException()\n",
    "        \n",
    "    elem = soup.find('section', class_='card__firmsInBuilding')\n",
    "    if elem:\n",
    "        elem = elem.find('div', class_='cardInfrastructureItem__counter')\n",
    "        n_orgs = int(elem.text)\n",
    "    else:\n",
    "        n_orgs = 0\n",
    "        \n",
    "    rating, rating_style = -1, None\n",
    "    \n",
    "    elem = soup.find('div', class_='customRating__stars')\n",
    "    if elem:\n",
    "        rating = int(next(filter(lambda s: s.startswith('_value'), elem.attrs['class'])).split('_')[-1])\n",
    "        rating_style = 'customRating'\n",
    "        \n",
    "    elem = soup.find('div', class_='rating')\n",
    "    if elem:\n",
    "        rating = int(next(filter(lambda s: s.startswith('_value'), elem.attrs['class'])).split('_')[-1])\n",
    "        rating_style = next(filter(lambda s: s.startswith('_style'), elem.attrs['class'])).split('_')[-1]\n",
    "        \n",
    "    res = {\n",
    "        'isorg': 1,\n",
    "        'name': name,\n",
    "        'type': type_,\n",
    "        'floors': floors,\n",
    "        'norgs': n_orgs, \n",
    "    }\n",
    "    \n",
    "    if rating > 0 and rating_style:\n",
    "        res.update({'rating': rating, 'rating_style': rating_style})\n",
    "        \n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_media_content(soup):\n",
    "    name = soup.find('h1', class_='mediaCardHeader__cardHeaderName').text\n",
    "    \n",
    "    elem = soup.find('span', class_='mediaAddress__linkCount')\n",
    "    n_orgs = int(elem.text) if elem else 0\n",
    "    \n",
    "    rating, rating_style = -1, None\n",
    "    \n",
    "    elem = soup.find('div', class_='customRating__stars')\n",
    "    if elem:\n",
    "        rating = int(next(filter(lambda s: s.startswith('_value'), elem.attrs['class'])).split('_')[-1])\n",
    "        rating_style = 'customRating'\n",
    "        \n",
    "    elem = soup.find('div', class_='rating')\n",
    "    if elem:\n",
    "        rating = int(next(filter(lambda s: s.startswith('_value'), elem.attrs['class'])).split('_')[-1])\n",
    "        rating_style = next(filter(lambda s: s.startswith('_style'), elem.attrs['class'])).split('_')[-1]\n",
    "    \n",
    "    res = {\n",
    "        'isorg': 1,\n",
    "        'name': name,\n",
    "        'norgs': n_orgs,\n",
    "    }\n",
    "    \n",
    "    if rating > 0 and rating_style:\n",
    "        res.update({'rating': rating, 'rating_style': rating_style})\n",
    "        \n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_other_case(soup):\n",
    "    name = soup.find('h1', class_='card__name').text\n",
    "    \n",
    "    elem = soup.find('div', class_='_purpose_building')\n",
    "    if elem:\n",
    "        elem = elem.text.rsplit(',', 1)\n",
    "        if len(elem) == 2:\n",
    "            type_, floors = elem[0], int(re.search('\\d+', elem[1]).group(0))\n",
    "        else:\n",
    "            type_, floors = '', int(re.search('\\d+', elem[0]).group(0))\n",
    "    else:\n",
    "        raise UknownCaseException('Unknown case')\n",
    "        \n",
    "    elem = soup.find('section', class_='card__firmsInBuilding')\n",
    "    if elem:\n",
    "        elem = elem.find('div', class_='cardInfrastructureItem__counter')\n",
    "        n_orgs = int(elem.text)\n",
    "    else:\n",
    "        n_orgs = 0\n",
    "        \n",
    "    rating, rating_style = -1, None\n",
    "\n",
    "    elem = soup.find('div', class_='customRating__stars')\n",
    "    if elem:\n",
    "        rating = int(next(filter(lambda s: s.startswith('_value'), elem.attrs['class'])).split('_')[-1])\n",
    "        rating_style = 'customRating'\n",
    "        \n",
    "    elem = soup.find('div', class_='rating')\n",
    "    if elem:\n",
    "        rating = int(next(filter(lambda s: s.startswith('_value'), elem.attrs['class'])).split('_')[-1])\n",
    "        rating_style = next(filter(lambda s: s.startswith('_style'), elem.attrs['class'])).split('_')[-1]\n",
    "    \n",
    "    res = {\n",
    "        'name': name,\n",
    "        'type': type_,\n",
    "        'floors': floors,\n",
    "        'norgs': n_orgs, \n",
    "    }\n",
    "    \n",
    "    if rating > 0 and rating_style:\n",
    "        res.update({'rating': rating, 'rating_style': rating_style})\n",
    "        \n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from multiprocessing import Pool\n",
    "from contextlib import suppress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_dumpfile(filename, root):\n",
    "    id_, _ = os.path.splitext(filename)\n",
    "        \n",
    "    filename = os.path.join(root, filename)\n",
    "        \n",
    "    with open(filename) as f_name:\n",
    "        soup = BeautifulSoup(f_name, 'html.parser')\n",
    "    \n",
    "    res = {'id': int(id_)}\n",
    "    \n",
    "    org = soup.find('div', class_='cardHeader__headerName')\n",
    "    bld = soup.find('span', class_='card__namePart')\n",
    "    media = soup.find('div', class_='mediaCardHeader__card')\n",
    "    \n",
    "    r = {}\n",
    "    \n",
    "    try:    \n",
    "        if org is not None:\n",
    "            r = process_organisation(soup)\n",
    "        elif bld is not None:\n",
    "            r = process_building(soup)\n",
    "        elif media is not None:\n",
    "            r = process_media_content(soup)\n",
    "        else:\n",
    "            with suppress(UknownCaseException):\n",
    "                r = process_other_case(soup)\n",
    "    except Exception as e:\n",
    "        print(id_)\n",
    "        raise e\n",
    "    \n",
    "    res.update(r)\n",
    "    \n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for root, dirs, filenames in os.walk('2gis'):\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 2481,\n",
       " 'isorg': 1,\n",
       " 'name': 'Измайлово',\n",
       " 'type': 'Гостиничный комплекс',\n",
       " 'floors': 31,\n",
       " 'norgs': 104,\n",
       " 'rating': 90,\n",
       " 'rating_style': 'customRating'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "process_dumpfile(filenames[0], root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 553 ms, sys: 301 ms, total: 855 ms\n",
      "Wall time: 14min 55s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from functools import partial\n",
    "    \n",
    "pool = Pool(10)\n",
    "df_names = pool.map(partial(process_dumpfile, root=root), filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>floors</th>\n",
       "      <th>id</th>\n",
       "      <th>isorg</th>\n",
       "      <th>name</th>\n",
       "      <th>norgs</th>\n",
       "      <th>rating</th>\n",
       "      <th>rating_style</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>31.0</td>\n",
       "      <td>2481</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Измайлово</td>\n",
       "      <td>104.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>customRating</td>\n",
       "      <td>Гостиничный комплекс</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.0</td>\n",
       "      <td>2397</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Малинэль, парикмахерский магазин, ИП Дмитриева...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>smallBlue</td>\n",
       "      <td>Жилой дом с административными помещениями</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9.0</td>\n",
       "      <td>6436</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Банк ВТБ, ПАО\\nБанкомат, Банк ВТБ, ПАО\\nСвязис...</td>\n",
       "      <td>13.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Жилой дом с административными помещениями</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>3314</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>3337</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2.0</td>\n",
       "      <td>4601</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Абсолют</td>\n",
       "      <td>14.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>customRating</td>\n",
       "      <td>Торговая группа</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2.0</td>\n",
       "      <td>1470</td>\n",
       "      <td>1.0</td>\n",
       "      <td>М5 Молл</td>\n",
       "      <td>183.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>customRating</td>\n",
       "      <td>Торгово-развлекательный центр</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>4921</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1454</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>5.0</td>\n",
       "      <td>7825</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Варшавская плаза</td>\n",
       "      <td>51.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>customRating</td>\n",
       "      <td>Бизнес-центр</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   floors    id  isorg                                               name  \\\n",
       "0    31.0  2481    1.0                                          Измайлово   \n",
       "1     9.0  2397    NaN  Малинэль, парикмахерский магазин, ИП Дмитриева...   \n",
       "2     9.0  6436    NaN  Банк ВТБ, ПАО\\nБанкомат, Банк ВТБ, ПАО\\nСвязис...   \n",
       "3     NaN  3314    NaN                                                NaN   \n",
       "4     NaN  3337    NaN                                                NaN   \n",
       "5     2.0  4601    1.0                                            Абсолют   \n",
       "6     2.0  1470    1.0                                            М5 Молл   \n",
       "7     NaN  4921    NaN                                                NaN   \n",
       "8     NaN  1454    NaN                                                NaN   \n",
       "9     5.0  7825    1.0                                   Варшавская плаза   \n",
       "\n",
       "   norgs  rating  rating_style                                       type  \n",
       "0  104.0    90.0  customRating                       Гостиничный комплекс  \n",
       "1    8.0   100.0     smallBlue  Жилой дом с административными помещениями  \n",
       "2   13.0     NaN           NaN  Жилой дом с административными помещениями  \n",
       "3    NaN     NaN           NaN                                        NaN  \n",
       "4    NaN     NaN           NaN                                        NaN  \n",
       "5   14.0    40.0  customRating                            Торговая группа  \n",
       "6  183.0    90.0  customRating              Торгово-развлекательный центр  \n",
       "7    NaN     NaN           NaN                                        NaN  \n",
       "8    NaN     NaN           NaN                                        NaN  \n",
       "9   51.0   100.0  customRating                               Бизнес-центр  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_names = pd.DataFrame(df_names)\n",
    "df_names.iloc[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = df_names['rating_style'] == 'minicard'\n",
    "df_names.loc[mask, ['rating', 'rating_style']] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['customRating', 'smallBlue', nan, 'mediumGray'], dtype=object)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_names['rating_style'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>floors</th>\n",
       "      <th>id</th>\n",
       "      <th>isorg</th>\n",
       "      <th>name</th>\n",
       "      <th>norgs</th>\n",
       "      <th>rating</th>\n",
       "      <th>rating_style</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>31.0</td>\n",
       "      <td>2481</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Измайлово</td>\n",
       "      <td>104.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>customRating</td>\n",
       "      <td>Гостиничный комплекс</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.0</td>\n",
       "      <td>2397</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Малинэль, парикмахерский магазин, ИП Дмитриева...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>smallBlue</td>\n",
       "      <td>Жилой дом с административными помещениями</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9.0</td>\n",
       "      <td>6436</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Банк ВТБ, ПАО\\nБанкомат, Банк ВТБ, ПАО\\nСвязис...</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>norating</td>\n",
       "      <td>Жилой дом с административными помещениями</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>3314</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>norating</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>3337</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>norating</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   floors    id  isorg                                               name  \\\n",
       "0    31.0  2481    1.0                                          Измайлово   \n",
       "1     9.0  2397    NaN  Малинэль, парикмахерский магазин, ИП Дмитриева...   \n",
       "2     9.0  6436    NaN  Банк ВТБ, ПАО\\nБанкомат, Банк ВТБ, ПАО\\nСвязис...   \n",
       "3     4.0  3314    NaN                                                      \n",
       "4     4.0  3337    NaN                                                      \n",
       "\n",
       "   norgs  rating  rating_style                                       type  \n",
       "0  104.0    90.0  customRating                       Гостиничный комплекс  \n",
       "1    8.0   100.0     smallBlue  Жилой дом с административными помещениями  \n",
       "2   13.0     0.0      norating  Жилой дом с административными помещениями  \n",
       "3   14.0     0.0      norating                                             \n",
       "4   14.0     0.0      norating                                             "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_names.fillna({'floors': df_names['floors'].median(),\n",
    "                 'norgs': df_names['norgs'].median(),\n",
    "                 'name': '', 'type': '',\n",
    "                 'rating': 0.0, 'rating_style': 'noRating'}, inplace=True)\n",
    "df_names.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['customRating', 'smallBlue', 'noRating', 'mediumGray'], dtype=object)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_names['rating_style'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names['rating_style'] = pd.Categorical(\n",
    "    df_names['rating_style'],\n",
    "    categories=['noRating', 'smallBlue', 'mediumGray', 'customRating'],\n",
    "    ordered=True).codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_name(name):\n",
    "    name = name.split('\\n')\n",
    "    name = filter(lambda s: 'Банкомат, ' not in s, name)\n",
    "    name = map(lambda s: 'ст. метро' if 'ст. метро' in s else s, name)\n",
    "    return '\\n'.join(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_type(s):\n",
    "    return re.sub('(?:, |\\b)?([гГ]|Рп|[пП]ос)\\. (?:.*?)$', '', s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names['name'] = df_names['name'].map(process_name)\n",
    "df_names['type'] = df_names['type'].map(process_type)\n",
    "df_names['floors'] = df_names['floors'].astype(int)\n",
    "df_names['norgs'] = df_names['norgs'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              2131\n",
      "Жилой дом                                     96\n",
      "Частный дом                                   60\n",
      "Административное здание                       54\n",
      "Жилой дом с административными помещениями     32\n",
      "Малоэтажный жилой дом                         10\n",
      "Хозяйственный корпус                          9\n",
      "Производственный корпус                       8\n",
      "Сооружение                                    4\n",
      "Ремонтируемое здание                          4\n",
      "Проходная, КПП                                3\n",
      "Медицинское учреждение                        3\n",
      "Коттедж                                       2\n",
      "Магазин                                       2\n",
      "Склад                                         2\n",
      "Строящееся административное здание            1\n",
      "Общежитие                                     1\n",
      "Таунхаус                                      1\n",
      "Гараж                                         1\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "for k, f in Counter(df_names['type'][df_names['name'] == '']).most_common():\n",
    "    print(f\"{k:<45s} {f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "typeset = set(df_names['type'][df_names['name'] == ''].unique())\n",
    "typeset.difference_update({'Медицинское учреждение', 'Магазин', 'Проходная, КПП'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>floors</th>\n",
       "      <th>id</th>\n",
       "      <th>isorg</th>\n",
       "      <th>name</th>\n",
       "      <th>norgs</th>\n",
       "      <th>rating</th>\n",
       "      <th>rating_style</th>\n",
       "      <th>type</th>\n",
       "      <th>comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>31</td>\n",
       "      <td>2481</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Измайлово</td>\n",
       "      <td>104</td>\n",
       "      <td>90.0</td>\n",
       "      <td>3</td>\n",
       "      <td>Гостиничный комплекс</td>\n",
       "      <td>Измайлово\\nГостиничный комплекс</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>2397</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Малинэль, парикмахерский магазин, ИП Дмитриева...</td>\n",
       "      <td>8</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Жилой дом с административными помещениями</td>\n",
       "      <td>Малинэль, парикмахерский магазин, ИП Дмитриева...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>6436</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Банк ВТБ, ПАО\\nСвязист, телекоммуникационная к...</td>\n",
       "      <td>13</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>Жилой дом с административными помещениями</td>\n",
       "      <td>Банк ВТБ, ПАО\\nСвязист, телекоммуникационная к...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>3314</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>3337</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   floors    id  isorg                                               name  \\\n",
       "0      31  2481    1.0                                          Измайлово   \n",
       "1       9  2397    NaN  Малинэль, парикмахерский магазин, ИП Дмитриева...   \n",
       "2       9  6436    NaN  Банк ВТБ, ПАО\\nСвязист, телекоммуникационная к...   \n",
       "3       4  3314    NaN                                                      \n",
       "4       4  3337    NaN                                                      \n",
       "\n",
       "   norgs  rating  rating_style                                       type  \\\n",
       "0    104    90.0             3                       Гостиничный комплекс   \n",
       "1      8   100.0             1  Жилой дом с административными помещениями   \n",
       "2     13     0.0             0  Жилой дом с административными помещениями   \n",
       "3     14     0.0             0                                              \n",
       "4     14     0.0             0                                              \n",
       "\n",
       "                                             comment  \n",
       "0                    Измайлово\\nГостиничный комплекс  \n",
       "1  Малинэль, парикмахерский магазин, ИП Дмитриева...  \n",
       "2  Банк ВТБ, ПАО\\nСвязист, телекоммуникационная к...  \n",
       "3                                                     \n",
       "4                                                     "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_names['comment'] = df_names['name']\n",
    "\n",
    "mask = ~df_names['type'].isin(typeset)\n",
    "df_names.loc[mask, 'comment'] += '\\n' + df_names.loc[mask, 'type']\n",
    "\n",
    "df_names.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names.drop(columns=['isorg'], inplace=True)\n",
    "df_names.to_csv('dataset_2gis.csv', sep=',', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}