scripts/west_bengal/parse_wb_bengali.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pdf2image\n",
    "from PIL import Image\n",
    "import pytesseract\n",
    "import difflib\n",
    "import re\n",
    "import pandas as pd\n",
    "import sys\n",
    "sys.path.append('../')\n",
    "from helper import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = '../../data/'\n",
    "STATE = 'wb/'\n",
    "\n",
    "PARSE_DATA_PAGES = \"../../parseData/images/\"+STATE\n",
    "create_path(PARSE_DATA_PAGES)\n",
    "\n",
    "PARSE_DATA_BLOCKS = \"../../parseData/blocks/\"+STATE\n",
    "create_path(PARSE_DATA_BLOCKS)\n",
    "\n",
    "PARSE_DATA_CSVS = \"../../parseData/csvs/\"+STATE\n",
    "create_path(PARSE_DATA_CSVS)\n",
    "\n",
    "COLUMNS = [\"number\",\"id\", \"elector_name\", \"father_or_husband_name\", \"relationship\", \"house_no\", \"age\", \"sex\", \"ac_name\", \"parl_constituency\", \"part_no\", \"year\", \"state\", \"filename\", \"main_town\", \"police_station\", \"mandal\", \"revenue_division\", \"district\", \"pin_code\", \"polling_station_name\", \"polling_station_address\", \"net_electors_male\", \"net_electors_female\", \"net_electors_third_gender\", \"net_electors_total\"]\n",
    "\n",
    "state_pdfs_path = DATA_PATH+STATE\n",
    "state_pdfs_files = os.listdir(state_pdfs_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.DS_Store',\n",
       " 'a0010018.pdf',\n",
       " 'a0010173.pdf',\n",
       " 'a0010179.pdf',\n",
       " 'a0010205.pdf',\n",
       " 'a0020082.pdf',\n",
       " 'a0020211.pdf',\n",
       " 'a0030020.pdf',\n",
       " 'a0030099.pdf',\n",
       " 'a0030155.pdf',\n",
       " 'a0040067.pdf',\n",
       " 'a0040244.pdf',\n",
       " 'a0050003.pdf',\n",
       " 'a0050008.pdf',\n",
       " 'a0050097.pdf',\n",
       " 'a0060042.pdf',\n",
       " 'a0060194.pdf',\n",
       " 'a0070128.pdf',\n",
       " 'a0070191.pdf',\n",
       " 'a0070274.pdf',\n",
       " 'a0080088.pdf',\n",
       " 'a0080118.pdf',\n",
       " 'a0080199.pdf',\n",
       " 'a0090081.pdf',\n",
       " 'a0090133.pdf',\n",
       " 'a0100041.pdf',\n",
       " 'a0100209.pdf',\n",
       " 'a0100217.pdf',\n",
       " 'a0100268.pdf',\n",
       " 'a0110145.pdf',\n",
       " 'a0120064.pdf',\n",
       " 'a0120087.pdf',\n",
       " 'a0130145.pdf',\n",
       " 'a0130240.pdf',\n",
       " 'a0140016.pdf',\n",
       " 'a0150040.pdf',\n",
       " 'a0150156.pdf',\n",
       " 'a0160140.pdf',\n",
       " 'a0160249.pdf',\n",
       " 'a0160256.pdf',\n",
       " 'a0170001.pdf',\n",
       " 'a0170027.pdf',\n",
       " 'a0170052.pdf',\n",
       " 'a0170134.pdf',\n",
       " 'a0180005.pdf',\n",
       " 'a0180024.pdf',\n",
       " 'a0180218.pdf',\n",
       " 'a0190058.pdf',\n",
       " 'a0200045.pdf',\n",
       " 'a0200156.pdf',\n",
       " 'a0210044.pdf',\n",
       " 'a0210179.pdf',\n",
       " 'a0230140.pdf',\n",
       " 'a0230151.pdf',\n",
       " 'a0230229.pdf',\n",
       " 'a0240216.pdf',\n",
       " 'a0250119.pdf',\n",
       " 'a0250275.pdf',\n",
       " 'a0260170.pdf',\n",
       " 'a0280025.pdf',\n",
       " 'a0290050.pdf',\n",
       " 'a0290115.pdf',\n",
       " 'a0290155.pdf',\n",
       " 'a0300044.pdf',\n",
       " 'a0300105.pdf',\n",
       " 'a0300223.pdf',\n",
       " 'a0320011.pdf',\n",
       " 'a0320028.pdf',\n",
       " 'a0320110.pdf',\n",
       " 'a0340153.pdf',\n",
       " 'a0370005.pdf',\n",
       " 'a0370042.pdf',\n",
       " 'a0370084.pdf',\n",
       " 'a0370090.pdf',\n",
       " 'a0370205.pdf',\n",
       " 'a0380107.pdf',\n",
       " 'a0390165.pdf',\n",
       " 'a0400096.pdf',\n",
       " 'a0400213.pdf',\n",
       " 'a0410007.pdf',\n",
       " 'a0410206.pdf',\n",
       " 'a0420082.pdf',\n",
       " 'a0420153.pdf',\n",
       " 'a0420201.pdf',\n",
       " 'a0430043.pdf',\n",
       " 'a0430104.pdf',\n",
       " 'a0440092.pdf',\n",
       " 'a0450134.pdf',\n",
       " 'a0470059.pdf',\n",
       " 'a0470193.pdf',\n",
       " 'a0480007.pdf',\n",
       " 'a0480010.pdf',\n",
       " 'a0480134.pdf',\n",
       " 'a0490004.pdf',\n",
       " 'a0490013.pdf',\n",
       " 'a0490030.pdf',\n",
       " 'a0500181.pdf',\n",
       " 'a0500194.pdf',\n",
       " 'a0520034.pdf',\n",
       " 'a0530099.pdf',\n",
       " 'a0570018.pdf',\n",
       " 'a0570020.pdf',\n",
       " 'a0570096.pdf',\n",
       " 'a0570218.pdf',\n",
       " 'a0580021.pdf',\n",
       " 'a0590002.pdf',\n",
       " 'a0590033.pdf',\n",
       " 'a0590147.pdf',\n",
       " 'a0590169.pdf',\n",
       " 'a0600146.pdf',\n",
       " 'a0610030.pdf',\n",
       " 'a0630186.pdf',\n",
       " 'a0630207.pdf',\n",
       " 'a0640037.pdf',\n",
       " 'a0640059.pdf',\n",
       " 'a0640073.pdf',\n",
       " 'a0650152.pdf',\n",
       " 'a0650230.pdf',\n",
       " 'a0660076.pdf',\n",
       " 'a0660128.pdf',\n",
       " 'a0680021.pdf',\n",
       " 'a0680113.pdf',\n",
       " 'a0690048.pdf',\n",
       " 'a0690097.pdf',\n",
       " 'a0700085.pdf',\n",
       " 'a0700212.pdf',\n",
       " 'a0720027.pdf',\n",
       " 'a0720261.pdf',\n",
       " 'a0740026.pdf',\n",
       " 'a0740086.pdf',\n",
       " 'a0740155.pdf',\n",
       " 'a0750021.pdf',\n",
       " 'a0750268.pdf',\n",
       " 'a0760005.pdf',\n",
       " 'a0760020.pdf',\n",
       " 'a0760034.pdf',\n",
       " 'a0760257.pdf',\n",
       " 'a0770029.pdf',\n",
       " 'a0770049.pdf',\n",
       " 'a0770125.pdf',\n",
       " 'a0770132.pdf',\n",
       " 'a0790155.pdf',\n",
       " 'a0800150.pdf',\n",
       " 'a0810223.pdf',\n",
       " 'a0820112.pdf',\n",
       " 'a0820138.pdf',\n",
       " 'a0830032.pdf',\n",
       " 'a0830223.pdf',\n",
       " 'a0830239.pdf',\n",
       " 'a0840026.pdf',\n",
       " 'a0840189.pdf',\n",
       " 'a0850221.pdf',\n",
       " 'a0860179.pdf',\n",
       " 'a0860216.pdf',\n",
       " 'a0870063.pdf',\n",
       " 'a0870247.pdf',\n",
       " 'a0880118.pdf',\n",
       " 'a0880161.pdf',\n",
       " 'a0880280.pdf',\n",
       " 'a0890015.pdf',\n",
       " 'a0890130.pdf',\n",
       " 'a0900064.pdf',\n",
       " 'a0900112.pdf',\n",
       " 'a0910131.pdf',\n",
       " 'a0910206.pdf',\n",
       " 'a0920073.pdf',\n",
       " 'a0930060.pdf',\n",
       " 'a0930122.pdf',\n",
       " 'a0930139.pdf',\n",
       " 'a0930142.pdf',\n",
       " 'a0940028.pdf',\n",
       " 'a0940264.pdf',\n",
       " 'a0950029.pdf',\n",
       " 'a0950115.pdf',\n",
       " 'a0950134.pdf',\n",
       " 'a0950147.pdf',\n",
       " 'a0960067.pdf',\n",
       " 'a0960126.pdf',\n",
       " 'a0970021.pdf',\n",
       " 'a0980167.pdf',\n",
       " 'a0990051.pdf',\n",
       " 'a0990177.pdf',\n",
       " 'a0990243.pdf',\n",
       " 'a1000230.pdf',\n",
       " 'a1020158.pdf',\n",
       " 'a1030135.pdf',\n",
       " 'a1040095.pdf',\n",
       " 'a1040140.pdf',\n",
       " 'a1040195.pdf',\n",
       " 'a1050043.pdf',\n",
       " 'a1070232.pdf',\n",
       " 'a1090004.pdf',\n",
       " 'a1090151.pdf',\n",
       " 'a1100113.pdf',\n",
       " 'a1100124.pdf',\n",
       " 'a1120056.pdf',\n",
       " 'a1120078.pdf',\n",
       " 'a1120108.pdf',\n",
       " 'a1150017.pdf',\n",
       " 'a1150106.pdf',\n",
       " 'a1160199.pdf',\n",
       " 'a1170109.pdf',\n",
       " 'a1170135.pdf',\n",
       " 'a1180236.pdf',\n",
       " 'a1180238.pdf',\n",
       " 'a1190088.pdf',\n",
       " 'a1190185.pdf',\n",
       " 'a1190223.pdf',\n",
       " 'a1200112.pdf',\n",
       " 'a1230082.pdf',\n",
       " 'a1230215.pdf',\n",
       " 'a1240125.pdf',\n",
       " 'a1240162.pdf',\n",
       " 'a1250145.pdf',\n",
       " 'a1250210.pdf',\n",
       " 'a1250262.pdf',\n",
       " 'a1260087.pdf',\n",
       " 'a1260114.pdf',\n",
       " 'a1260131.pdf',\n",
       " 'a1260170.pdf',\n",
       " 'a1260238.pdf',\n",
       " 'a1270215.pdf',\n",
       " 'a1290112.pdf',\n",
       " 'a1290138.pdf',\n",
       " 'a1300084.pdf',\n",
       " 'a1310076.pdf',\n",
       " 'a1310219.pdf',\n",
       " 'a1320184.pdf',\n",
       " 'a1320258.pdf',\n",
       " 'a1330025.pdf',\n",
       " 'a1330040.pdf',\n",
       " 'a1330064.pdf',\n",
       " 'a1330067.pdf',\n",
       " 'a1360097.pdf',\n",
       " 'a1360177.pdf',\n",
       " 'a1380037.pdf',\n",
       " 'a1380090.pdf',\n",
       " 'a1380182.pdf',\n",
       " 'a1390090.pdf',\n",
       " 'a1390167.pdf',\n",
       " 'a1390185.pdf',\n",
       " 'a1400103.pdf',\n",
       " 'a1430152.pdf',\n",
       " 'a1430222.pdf',\n",
       " 'a1440012.pdf',\n",
       " 'a1440240.pdf',\n",
       " 'a1450048.pdf',\n",
       " 'a1450075.pdf',\n",
       " 'a1450107.pdf',\n",
       " 'a1450253.pdf',\n",
       " 'a1460202.pdf',\n",
       " 'a1460222.pdf',\n",
       " 'a1460230.pdf',\n",
       " 'a1470027.pdf',\n",
       " 'a1470097.pdf',\n",
       " 'a1480006.pdf',\n",
       " 'a1480010.pdf',\n",
       " 'a1490040.pdf',\n",
       " 'a1490061.pdf',\n",
       " 'a1510093.pdf',\n",
       " 'a1510152.pdf',\n",
       " 'a1520051.pdf',\n",
       " 'a1530123.pdf',\n",
       " 'a1530149.pdf',\n",
       " 'a1530237.pdf',\n",
       " 'a1540043.pdf',\n",
       " 'a1540183.pdf',\n",
       " 'a1540211.pdf',\n",
       " 'a1540256.pdf',\n",
       " 'a1580127.pdf',\n",
       " 'a1590257.pdf',\n",
       " 'a1600097.pdf',\n",
       " 'a1690050.pdf',\n",
       " 'a1690156.pdf',\n",
       " 'a1700011.pdf',\n",
       " 'a1700210.pdf',\n",
       " 'a1710060.pdf',\n",
       " 'a1710067.pdf',\n",
       " 'a1710163.pdf',\n",
       " 'a1720095.pdf',\n",
       " 'a1720161.pdf',\n",
       " 'a1730247.pdf',\n",
       " 'a1730270.pdf',\n",
       " 'a1740204.pdf',\n",
       " 'a1740241.pdf',\n",
       " 'a1760011.pdf',\n",
       " 'a1770167.pdf',\n",
       " 'a1780022.pdf',\n",
       " 'a1780118.pdf',\n",
       " 'a1780142.pdf',\n",
       " 'a1790162.pdf',\n",
       " 'a1800021.pdf',\n",
       " 'a1810036.pdf',\n",
       " 'a1810114.pdf',\n",
       " 'a1810206.pdf',\n",
       " 'a1810241.pdf',\n",
       " 'a1820119.pdf',\n",
       " 'a1820186.pdf',\n",
       " 'a1830056.pdf',\n",
       " 'a1830090.pdf',\n",
       " 'a1830281.pdf',\n",
       " 'a1850130.pdf',\n",
       " 'a1860128.pdf',\n",
       " 'a1870031.pdf',\n",
       " 'a1880150.pdf',\n",
       " 'a1880179.pdf',\n",
       " 'a1890142.pdf',\n",
       " 'a1900190.pdf',\n",
       " 'a1900355.pdf',\n",
       " 'a1900360.pdf',\n",
       " 'a1920230.pdf',\n",
       " 'a1920247.pdf',\n",
       " 'a1930102.pdf',\n",
       " 'a1930116.pdf',\n",
       " 'a1930127.pdf',\n",
       " 'a1960063.pdf',\n",
       " 'a1960264.pdf',\n",
       " 'a1960290.pdf',\n",
       " 'a1980036.pdf',\n",
       " 'a2000023.pdf',\n",
       " 'a2000103.pdf',\n",
       " 'a2000134.pdf',\n",
       " 'a2000184.pdf',\n",
       " 'a2030159.pdf',\n",
       " 'a2040028.pdf',\n",
       " 'a2040241.pdf',\n",
       " 'a2050033.pdf',\n",
       " 'a2050161.pdf',\n",
       " 'a2070102.pdf',\n",
       " 'a2070215.pdf',\n",
       " 'a2080050.pdf',\n",
       " 'a2080081.pdf',\n",
       " 'a2080141.pdf',\n",
       " 'a2090099.pdf',\n",
       " 'a2090154.pdf',\n",
       " 'a2090166.pdf',\n",
       " 'a2100083.pdf',\n",
       " 'a2100181.pdf',\n",
       " 'a2100255.pdf',\n",
       " 'a2110055.pdf',\n",
       " 'a2110176.pdf',\n",
       " 'a2120065.pdf',\n",
       " 'a2130100.pdf',\n",
       " 'a2130117.pdf',\n",
       " 'a2140004.pdf',\n",
       " 'a2140279.pdf',\n",
       " 'a2150156.pdf',\n",
       " 'a2150183.pdf',\n",
       " 'a2160001.pdf',\n",
       " 'a2160108.pdf',\n",
       " 'a2160177.pdf',\n",
       " 'a2170140.pdf',\n",
       " 'a2170258.pdf',\n",
       " 'a2170259.pdf',\n",
       " 'a2170271.pdf',\n",
       " 'a2170278.pdf',\n",
       " 'a2180079.pdf',\n",
       " 'a2190246.pdf',\n",
       " 'a2200020.pdf',\n",
       " 'a2200109.pdf',\n",
       " 'a2200144.pdf',\n",
       " 'a2200238.pdf',\n",
       " 'a2200257.pdf',\n",
       " 'a2210013.pdf',\n",
       " 'a2220048.pdf',\n",
       " 'a2230007.pdf',\n",
       " 'a2230221.pdf',\n",
       " 'a2250123.pdf',\n",
       " 'a2260060.pdf',\n",
       " 'a2260082.pdf',\n",
       " 'a2260290.pdf',\n",
       " 'a2270099.pdf',\n",
       " 'a2270125.pdf',\n",
       " 'a2270209.pdf',\n",
       " 'a2290128.pdf',\n",
       " 'a2300106.pdf',\n",
       " 'a2310243.pdf',\n",
       " 'a2310260.pdf',\n",
       " 'a2310311.pdf',\n",
       " 'a2320246.pdf',\n",
       " 'a2330172.pdf',\n",
       " 'a2330255.pdf',\n",
       " 'a2340020.pdf',\n",
       " 'a2340036.pdf',\n",
       " 'a2340115.pdf',\n",
       " 'a2340178.pdf',\n",
       " 'a2360059.pdf',\n",
       " 'a2360112.pdf',\n",
       " 'a2360202.pdf',\n",
       " 'a2370004.pdf',\n",
       " 'a2380055.pdf',\n",
       " 'a2380253.pdf',\n",
       " 'a2380313.pdf',\n",
       " 'a2390103.pdf',\n",
       " 'a2390138.pdf',\n",
       " 'a2390217.pdf',\n",
       " 'a2390229.pdf',\n",
       " 'a2400065.pdf',\n",
       " 'a2400164.pdf',\n",
       " 'a2410005.pdf',\n",
       " 'a2420082.pdf',\n",
       " 'a2420192.pdf',\n",
       " 'a2430005.pdf',\n",
       " 'a2440087.pdf',\n",
       " 'a2440105.pdf',\n",
       " 'a2440193.pdf',\n",
       " 'a2450004.pdf',\n",
       " 'a2450035.pdf',\n",
       " 'a2450059.pdf',\n",
       " 'a2450221.pdf',\n",
       " 'a2460005.pdf',\n",
       " 'a2460253.pdf',\n",
       " 'a2470092.pdf',\n",
       " 'a2470161.pdf',\n",
       " 'a2490011.pdf',\n",
       " 'a2490114.pdf',\n",
       " 'a2490132.pdf',\n",
       " 'a2490172.pdf',\n",
       " 'a2490199.pdf',\n",
       " 'a2520090.pdf',\n",
       " 'a2520146.pdf',\n",
       " 'a2540027.pdf',\n",
       " 'a2540159.pdf',\n",
       " 'a2540226.pdf',\n",
       " 'a2550053.pdf',\n",
       " 'a2550219.pdf',\n",
       " 'a2560052.pdf',\n",
       " 'a2560113.pdf',\n",
       " 'a2560188.pdf',\n",
       " 'a2560220.pdf',\n",
       " 'a2570018.pdf',\n",
       " 'a2570183.pdf',\n",
       " 'a2580242.pdf',\n",
       " 'a2590127.pdf',\n",
       " 'a2590242.pdf',\n",
       " 'a2600081.pdf',\n",
       " 'a2610198.pdf',\n",
       " 'a2610286.pdf',\n",
       " 'a2630091.pdf',\n",
       " 'a2640026.pdf',\n",
       " 'a2640072.pdf',\n",
       " 'a2640089.pdf',\n",
       " 'a2640226.pdf',\n",
       " 'a2660128.pdf',\n",
       " 'a2660237.pdf',\n",
       " 'a2670044.pdf',\n",
       " 'a2670126.pdf',\n",
       " 'a2680002.pdf',\n",
       " 'a2680022.pdf',\n",
       " 'a2680163.pdf',\n",
       " 'a2690139.pdf',\n",
       " 'a2700288.pdf',\n",
       " 'a2730193.pdf',\n",
       " 'a2730213.pdf',\n",
       " 'a2740063.pdf',\n",
       " 'a2740190.pdf',\n",
       " 'a2750107.pdf',\n",
       " 'a2750200.pdf',\n",
       " 'a2760085.pdf',\n",
       " 'a2760114.pdf',\n",
       " 'a2760204.pdf',\n",
       " 'a2760265.pdf',\n",
       " 'a2770231.pdf',\n",
       " 'a2770279.pdf',\n",
       " 'a2770289.pdf',\n",
       " 'a2790074.pdf',\n",
       " 'a2790092.pdf',\n",
       " 'a2790216.pdf',\n",
       " 'a2790243.pdf',\n",
       " 'a2810016.pdf',\n",
       " 'a2820101.pdf',\n",
       " 'a2820214.pdf',\n",
       " 'a2830051.pdf',\n",
       " 'a2830135.pdf',\n",
       " 'a2830199.pdf',\n",
       " 'a2840214.pdf',\n",
       " 'a2850042.pdf',\n",
       " 'a2850068.pdf',\n",
       " 'a2850203.pdf',\n",
       " 'a2860001.pdf',\n",
       " 'a2860131.pdf',\n",
       " 'a2870136.pdf',\n",
       " 'a2870182.pdf',\n",
       " 'a2870236.pdf',\n",
       " 'a2870247.pdf',\n",
       " 'a2870261.pdf',\n",
       " 'a2880074.pdf',\n",
       " 'a2880130.pdf',\n",
       " 'a2890234.pdf',\n",
       " 'a2890251.pdf',\n",
       " 'a2900007.pdf',\n",
       " 'a2920135.pdf',\n",
       " 'a2920142.pdf',\n",
       " 'a2930071.pdf',\n",
       " 'a2930141.pdf',\n",
       " 'a2940114.pdf',\n",
       " 'a2940153.pdf',\n",
       " 'a2940180.pdf',\n",
       " 'a2940215.pdf',\n",
       " 'a2940229.pdf']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sort_nicely(state_pdfs_files)\n",
    "state_pdfs_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_data(data):\n",
    "    seps = [\":\",\"ব্লক\",\"-\"]\n",
    "    \n",
    "    for s in seps:\n",
    "        if s in data:\n",
    "            break\n",
    "\n",
    "    data = data.split(s)\n",
    "    data = [ i for i in data if i.strip()!='']\n",
    "    if len(data)>1:\n",
    "        data = data[1].strip()\n",
    "        return data\n",
    "    else:\n",
    "        data = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def arrange_columns(first_page_list,block_list,filename):\n",
    "    \n",
    "    year = 2018\n",
    "    state = 'wb'\n",
    "    \n",
    "    ac_name,parl_constituency,part_no,main_town,police_station,polling_station_name,polling_station_address,revenue_division,mandal,district,pin_code,net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = first_page_list\n",
    "    name,rel_name,rel_type,house_no,age,gender,voter_id,number = block_list\n",
    "    \n",
    "    final_list = [number,voter_id,name,rel_name,rel_type,house_no,age,gender,ac_name,\n",
    "                 parl_constituency,part_no,year,state,filename,main_town,police_station,mandal,\n",
    "                 revenue_division,district,pin_code,polling_station_name,polling_station_address,\n",
    "                 net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total]\n",
    "    \n",
    "    \n",
    "    return final_list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mandal(a,b,c,d,img,crop_path):\n",
    "            \n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "\n",
    "    crop_det_path = crop_path+\"det.jpg\"\n",
    "    crop_img.save(crop_det_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_det_path, config='--psm 6', lang='ben'))\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "\n",
    "    return text\n",
    "    \n",
    "def get_stats(a,b,c,d,img,crop_path):\n",
    "\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "\n",
    "    crop_path = input_images_blocks_path+\"page/\"\n",
    "    create_path(crop_path)\n",
    "\n",
    "    crop_stat_path = crop_path+\"stat.jpg\"\n",
    "    crop_img.save(crop_stat_path)\n",
    "\n",
    "    a_n,b_n,c_n,d_n = extract_4_numbers(crop_stat_path)\n",
    "\n",
    "    return a_n,b_n,c_n,d_n\n",
    "\n",
    "def get_police_det(a,b,c,d,img,crop_path):\n",
    "\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "\n",
    "    crop_police_path = crop_path+\"police.jpg\"\n",
    "    crop_img.save(crop_police_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_police_path, config='--psm 6', lang='ben')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "\n",
    "    return text\n",
    "\n",
    "def extract_4_numbers(crop_stat_path):\n",
    "    \n",
    "    text = (pytesseract.image_to_string(crop_stat_path, config='--psm 6', lang='eng')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "\n",
    "    text = re.findall(r'\\d+', text)\n",
    "\n",
    "    if len(text)==4:\n",
    "        if int(text[0]) + int(text[1]) == int(text[2]):\n",
    "            net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],\"0\",text[2]\n",
    "        elif int(text[0]) + int(text[1]) == int(text[3]):\n",
    "            net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],\"0\",text[3]\n",
    "        else:\n",
    "            net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],text[2],text[3]\n",
    "    elif len(text) == 3 and int(text[2])>=int(text[1]) and int(text[2])>=int(text[0]):\n",
    "        net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],text[1],\"0\",text[2]\n",
    "    elif len(text) == 2 and int(text[0])*2-100<int(text[1]):\n",
    "        net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = text[0],int(text[1])-int(text[0]),\"0\",text[1]\n",
    "    else:\n",
    "        net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total = \"\",\"\",\"\",\"\"\n",
    "    \n",
    "    return net_electors_male,net_electors_female,net_electors_third_gender,net_electors_total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_first_page_details(path):\n",
    "    \n",
    "    img = Image.open(path)\n",
    "        \n",
    "    a,b,c,d = 2557,5205,1237,105  # stats for male and female\n",
    "    crop_path = input_images_blocks_path+\"page/\"\n",
    "    create_path(crop_path)\n",
    "    \n",
    "    a_n,b_n,c_n,d_n = get_stats(a,b,c,d,img,crop_path)\n",
    "    \n",
    "    if a_n == \"\" or b_n == \"\":\n",
    "        print(a_n,b_n,c_n,d_n)\n",
    "    \n",
    "    a,b,c,d = 2520,3172,1386,1140  # mandal block\n",
    "    main_town,police_station,revenue_division,mandal,district,pin_code = \"\",\"\",\"\",\"\",\"\",\"\"\n",
    "\n",
    "    text = get_mandal(a,b,c,d,img,crop_path)\n",
    "        \n",
    "    \n",
    "    if len(text) == 8:\n",
    "        \n",
    "        main_town = split_data(text[0])\n",
    "        police_station = split_data(text[4])\n",
    "        revenue_division = split_data(text[5])\n",
    "        mandal = split_data(text[2])\n",
    "        district = split_data(text[6])\n",
    "        pin_code = split_data(text[7])\n",
    "        \n",
    " \n",
    "    a,b,c,d = 3008,280,640,200 # part no\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "    \n",
    "    crop_part_path = crop_path+\"part.jpg\"\n",
    "    crop_img.save(crop_part_path)\n",
    "\n",
    "    text = (pytesseract.image_to_string(crop_part_path, config='--psm 6', lang='ben')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = re.findall(r'\\d+', text)\n",
    "    \n",
    "    if len(text)>0:\n",
    "        part_no = text[0]\n",
    "    else:\n",
    "        part_no = \"\"\n",
    "            \n",
    "        \n",
    "    a,b,c,d = 280,3610,1850,714 # police name name and address\n",
    "    \n",
    "    text = get_police_det(a,b,c,d,img,crop_path)\n",
    "    \n",
    "    if len(text) == 6:\n",
    "        polling_station_name = text[2].strip()\n",
    "        polling_station_address = text[5].strip()\n",
    "    elif len(text) >=3:\n",
    "        polling_station_name = text[2].strip()\n",
    "        polling_station_address = ''\n",
    "    else:\n",
    "        polling_station_name, polling_station_address = \"\",\"\"\n",
    "    \n",
    "        \n",
    "    a,b,c,d = 266,207,2654,585 # ac name and parl\n",
    "    crop_img = crop_section(a,b,c,d,img)\n",
    "    \n",
    "    crop_ac_path = crop_path+\"ac.jpg\"\n",
    "    crop_img.save(crop_ac_path)\n",
    "    \n",
    "    text = (pytesseract.image_to_string(crop_ac_path, config='--psm 6', lang='eng+ben')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "    text = text.split('\\n')\n",
    "    text = [ i for i in text if i!='' and i!='\\x0c']\n",
    "    \n",
    "    ac_name,parl_constituency = '',''\n",
    "    \n",
    "        \n",
    "\n",
    "    if len(text) == 4:\n",
    "        ac_name = text[1].split(\",\")\n",
    "        if len(ac_name)>0:\n",
    "            ac_name = ac_name[-1].strip()\n",
    "            \n",
    "        parl_constituency= text[3].split(\",\")\n",
    "        if len(parl_constituency)>0:\n",
    "            parl_constituency = parl_constituency[-1].strip()\n",
    "\n",
    "    \n",
    "    return [ac_name,parl_constituency,part_no,main_town,police_station,polling_station_name,polling_station_address,revenue_division,mandal,district,pin_code,a_n,b_n,c_n,d_n]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_poll_blocks_from_page(page_full_path,page_blocks_path,amend_page):\n",
    "    \n",
    "    img = Image.open(page_full_path)\n",
    "\n",
    "    amend = False\n",
    "    \n",
    "    def generate(intial_width,a,b,gap):\n",
    "        count = 0\n",
    "        crop_width = 1223\n",
    "        crop_height = 460\n",
    "\n",
    "        for col in range(1,11):\n",
    "\n",
    "            for row in range(1,4):\n",
    "                c = a+crop_width\n",
    "                d = b+crop_height\n",
    "                area = (a, b, c, d)\n",
    "                cropped_img = img.crop(area)\n",
    "                count = count+1\n",
    "                cropped_img.save(page_blocks_path+str(count)+\".jpg\")\n",
    "\n",
    "                a = c\n",
    "\n",
    "            a = intial_width\n",
    "            b = b+crop_height+gap\n",
    "    \n",
    "    page_type,intial_height = check_page_type(img,amend_page)\n",
    "    \n",
    "    if page_type == 1:\n",
    "        intial_width = 232\n",
    "        generate(intial_width,intial_width,intial_height,0)\n",
    "\n",
    "        \n",
    "\n",
    "def check_page_type(img,amend_page):\n",
    "    \n",
    "    return 1,500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_name(name):\n",
    "    \n",
    "    row = name.split(\":\")\n",
    "    if len(row)!=2:\n",
    "        return \"\"\n",
    "    else:\n",
    "        return row[1].strip()\n",
    "    \n",
    "def extract_id(v_id):\n",
    "    row = v_id.split(\" \")\n",
    "    if len(row)==2:\n",
    "        return row[0].strip(), row[-1].strip()\n",
    "    elif len(row)>2:\n",
    "        return row[0], row[1]+row[2].strip()\n",
    "    elif len(row)==1:\n",
    "        return \"\",row[-1]\n",
    "    else:\n",
    "        return \"\",\"\"\n",
    "\n",
    "def extract_house_no(house_no):\n",
    "    row = house_no.split(\":\")\n",
    "    if len(row)==2:\n",
    "         return row[1].strip()\n",
    "    else:\n",
    "        return \"\"\n",
    "    \n",
    "def extract_gender(age_gender):\n",
    "    \n",
    "    text = re.findall(r'\\d+', age_gender)\n",
    "    \n",
    "    if len(text)>0:\n",
    "        age = text[0]\n",
    "    else:\n",
    "        age = \"\"\n",
    "        \n",
    "    row = age_gender.split(\"লিঙ্গ\")\n",
    "    \n",
    "    if len(row)>=2:\n",
    "        gender = row[-1].replace(\":\",\"\").strip()\n",
    "        \n",
    "    else:\n",
    "        gender =  \"\"\n",
    "        \n",
    "    return age, gender\n",
    "\n",
    "\n",
    "def extract_rel_name(rel_name):\n",
    "    row = rel_name.split(\":\")\n",
    "    if len(row)!=2:\n",
    "        \n",
    "        row = rel_name.split(\";\")\n",
    "        if len(row)!=2:\n",
    "            return \"\",\"\"\n",
    "        else:\n",
    "            rel_type = extract_rel_type(row[0].strip())\n",
    "            return row[1].strip(),rel_type\n",
    "    else:\n",
    "        rel_type = extract_rel_type(row[0].strip())\n",
    "        \n",
    "        return row[1].strip(),rel_type\n",
    "    \n",
    "def extract_rel_type(rel_type):\n",
    "    line = rel_type\n",
    "    if line.startswith(\"স্বামীর\"):\n",
    "        rel_type = 'husband'\n",
    "    elif line.startswith(\"পিতার\"):\n",
    "        rel_type = 'father'\n",
    "    else:\n",
    "        rel_type = \"father\"\n",
    "    \n",
    "    return rel_type \n",
    "\n",
    "\n",
    "def extract_details_from_block(block):\n",
    "        \n",
    "    v_id = block[0]\n",
    "    name = block[1]\n",
    "    rel_name = block[2]\n",
    "    house_no = block[3]\n",
    "    age_gender = block[4]\n",
    "    \n",
    "    name = extract_name(name)\n",
    "    rel_name,rel_type = extract_rel_name(rel_name)\n",
    "    house_no = extract_house_no(house_no)\n",
    "    age, gender = extract_gender(age_gender)\n",
    "    number, voter_id = extract_id(v_id)\n",
    "    \n",
    "    \n",
    "    return [name,rel_name,rel_type,house_no,age,gender,voter_id,number]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a0010018.pdf\n",
      "page done :  3.jpg\n",
      "page done :  4.jpg\n",
      "page done :  5.jpg\n",
      "page done :  6.jpg\n",
      "page done :  7.jpg\n",
      "page done :  8.jpg\n",
      "page done :  9.jpg\n",
      "page done :  10.jpg\n",
      "page done :  11.jpg\n",
      "page done :  12.jpg\n",
      "page done :  13.jpg\n",
      "page done :  14.jpg\n",
      "page done :  15.jpg\n",
      "page done :  16.jpg\n",
      "page done :  17.jpg\n",
      "page done :  18.jpg\n",
      "page done :  19.jpg\n",
      "page done :  20.jpg\n",
      "page done :  21.jpg\n",
      "page done :  22.jpg\n",
      "page done :  23.jpg\n",
      "page done :  24.jpg\n",
      "page done :  25.jpg\n",
      "page done :  26.jpg\n",
      "page done :  27.jpg\n",
      "page done :  28.jpg\n",
      "page done :  29.jpg\n",
      "page done :  30.jpg\n",
      "page done :  31.jpg\n",
      "page done :  32.jpg\n",
      "page done :  33.jpg\n",
      "page done :  34.jpg\n",
      "CSV saved\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for pdf_file_name in ['a0010018.pdf']:\n",
    "# for pdf_file_name in state_pdfs_files:\n",
    "    \n",
    "    if pdf_file_name == '.DS_Store':\n",
    "        continue\n",
    "    \n",
    "    print(pdf_file_name)\n",
    "    \n",
    "    \n",
    "    #create images,blocks and csvs paths for each file\n",
    "    pdf_file_name_without_ext = pdf_file_name.split('.pdf')[0]\n",
    "    input_pdf_images_path = PARSE_DATA_PAGES+pdf_file_name_without_ext+\"/\"\n",
    "    create_path(input_pdf_images_path)\n",
    "\n",
    "    input_images_blocks_path = PARSE_DATA_BLOCKS+pdf_file_name_without_ext+\"/\"\n",
    "    create_path(input_images_blocks_path)\n",
    "    \n",
    "    if os.path.exists(PARSE_DATA_CSVS+pdf_file_name_without_ext+\".csv\"):\n",
    "        print(pdf_file_name_without_ext+\".csv\", \"already exists\")\n",
    "        continue\n",
    "    \n",
    "    #convert pdf into bunch of images\n",
    "    try:\n",
    "        pdf_2_images_list = pdf_to_img(state_pdfs_path+pdf_file_name, input_pdf_images_path,dpi=500)\n",
    "    except:\n",
    "        continue\n",
    "    \n",
    "    #sort pages for looping\n",
    "    input_images = os.listdir(input_pdf_images_path)\n",
    "    sort_nicely(input_images)\n",
    "    \n",
    "    #empty intial data\n",
    "    df = pd.DataFrame(columns = COLUMNS)\n",
    "    order_problem = []\n",
    "    \n",
    "    amend_page = False\n",
    "    \n",
    "    #for each page, parse the data\n",
    "    for page in input_images:\n",
    "    \n",
    "        page_full_path = input_pdf_images_path+page\n",
    "        \n",
    "        #extract first page content\n",
    "        if page == '1.jpg':\n",
    "            first_page_list = extract_first_page_details(page_full_path)\n",
    "            continue\n",
    "            \n",
    "\n",
    "        #ingnore 2nd page and last page\n",
    "        if page == '2.jpg' or input_images[-1] == page:\n",
    "            continue\n",
    "            \n",
    "\n",
    "        #loop from 3 page onwards\n",
    "        if page.endswith('.jpg'):\n",
    "            \n",
    "            final_invidual_blocks = []\n",
    "            blocks_path = input_images_blocks_path+\"blocks/\"\n",
    "            create_path(blocks_path)\n",
    "\n",
    "            page_idx = page.split(\".jpg\")[0] + \"/\"\n",
    "            page_blocks_path = blocks_path+page_idx\n",
    "            create_path(page_blocks_path)\n",
    "                \n",
    "            generate_poll_blocks_from_page(page_full_path,page_blocks_path,amend_page)\n",
    "            \n",
    "            \n",
    "            sorted_blocks = os.listdir(page_blocks_path)\n",
    "            sort_nicely(sorted_blocks)\n",
    "            \n",
    "            for jpg_file in sorted_blocks:\n",
    "                \n",
    "                if jpg_file.endswith('.jpg') :\n",
    "\n",
    "                    text = (pytesseract.image_to_string(page_blocks_path+jpg_file, config='--psm 6', lang='eng+ben')) #config='--psm 4' config='-c preserve_interword_spaces=1'\n",
    "                    params_list = text.split('\\n')\n",
    "                    params_list = [ i for i in params_list if i!='' and i!='\\x0c']\n",
    "                    \n",
    "                    if len(params_list) == 5:\n",
    "                        final_invidual_blocks.append(params_list)\n",
    "                    else:\n",
    "                        order_problem.append((page, jpg_file,params_list))\n",
    "        \n",
    "        #put the data into dataframe\n",
    "        for block in final_invidual_blocks:\n",
    "            \n",
    "            block_list = extract_details_from_block(block)\n",
    "                        \n",
    "            final_list = arrange_columns(first_page_list,block_list,pdf_file_name_without_ext)            \n",
    "            df_length = len(df)\n",
    "            df.loc[df_length] = final_list\n",
    "        \n",
    "        print(\"page done : \",page)\n",
    "                \n",
    "    #save the dataframe(pdf) data into csv\n",
    "    save_to_csv(df,PARSE_DATA_CSVS+pdf_file_name_without_ext+\".csv\")\n",
    "    print(\"CSV saved\")\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#combine all state files into one csv\n",
    "combine_all_csvs(\"wb_final.csv\",PARSE_DATA_CSVS)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}