web_scrape_arxiv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#with arxiv api\n",
    "# https://stackoverflow.com/questions/11763028/python-3-urllib-http-error-407-proxy-authentication-required\n",
    "# https://arxiv.org/help/api\n",
    "\n",
    "import urllib.request as req\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "\n",
    "proxy = req.ProxyHandler({'http': r'#insert header and IP address'})\n",
    "auth = req.HTTPBasicAuthHandler()\n",
    "opener = req.build_opener(proxy, auth, req.HTTPHandler)\n",
    "req.install_opener(opener)\n",
    "conn = req.urlopen('http://export.arxiv.org/api/query?search_query=all:\"missing+data\"&start=0&max_results=1000')\n",
    "return_str = conn.read()\n",
    "\n",
    "soup = BeautifulSoup(return_str, \"html\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = soup.find_all('title')[1:]\n",
    "links = soup.find_all('id')[1:]\n",
    "dates = soup.find_all('published')[1:]\n",
    "categories = soup.find_all('arxiv:primary_category')[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_list = []\n",
    "link_list = []\n",
    "date_list = []\n",
    "categ_list = []\n",
    "\n",
    "for title,link,date,category in zip(titles,links,dates,categories):\n",
    "    title_list.append(title.getText())\n",
    "    link_list.append(link.getText()+'.pdf')\n",
    "    date_list.append(date.getText()[:4])\n",
    "    categ_list.append(category.get('term').split('.')[0])\n",
    "\n",
    "arxiv_df = pd.DataFrame(list(zip(link_list,title_list,date_list,categ_list)),columns=['id','title','date','category'])\n",
    "\n",
    "path = \"c:/Users/soirk/Krisztian/Egyetem/Survey Statisztika Msc/Szakdolgozat/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "arxiv_df = pd.DataFrame(list(zip(link_list,title_list,date_list,categ_list)),columns=['id','title','date','category'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(arxiv_df['id'])):\n",
    "    a_id=arxiv_df['id'][i]\n",
    "    a_id=a_id[a_id.find('abs/')+4:a_id.find('.pdf')]\n",
    "    if '/' in a_id:\n",
    "        arxiv_df.iloc[i,0]=a_id[a_id.find('/')+1:len(a_id)]\n",
    "    else:\n",
    "        arxiv_df.iloc[i,0]=a_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlretrieve\n",
    "\n",
    "def download_file(download_url,name):\n",
    "    urlretrieve(download_url, 'c:/Users/soirk/Krisztian/Egyetem/Survey Statisztika Msc/Szakdolgozat/arxiv_pdfs/'+name+'.pdf')\n",
    "   \n",
    "for i in link_list:\n",
    "    download_file(i,i[-7:])\n",
    "    print(i[-7:])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}