sars_cov_2_pubmed_data_extracting.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "sars_cov_2_pubmed_data_extracting.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1hVR7JKwH8RmmR5REDZMSySXwPtecLEjT",
      "authorship_tag": "ABX9TyMj8OcLuPJmpwS0lr8VmaYO"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# SARS CoV 2 PubMed Metadata Extraction"
      ],
      "metadata": {
        "id": "COfoIusNOYR5"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "- https://www.december.com/html/spec/httpstat.html\n",
        "- https://erilu.github.io/pubmed-abstract-compiler/\n",
        "- https://www.ncbi.nlm.nih.gov/books/NBK25500/\n",
        "- https://www.ncbi.nlm.nih.gov/books/NBK25499/\n",
        "- https://www.nlm.nih.gov/bsd/mms/medlineelements.html"
      ],
      "metadata": {
        "id": "w3TNx4BxPQTy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Installing Biopython\n",
        "! pip install -q biopython"
      ],
      "metadata": {
        "id": "V_0bIqmSXnpK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "adf72381-596b-41e4-bbab-d222bc774cb4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 2.3 MB 5.9 MB/s \n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "import urllib\n",
        "from http.client import IncompleteRead\n",
        "\n",
        "from Bio import Entrez\n",
        "from Bio import Medline\n",
        "\n",
        "import re\n",
        "import time\n",
        "\n",
        "from pprint import pprint\n",
        "from tqdm import tqdm\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "M6q72vKOOeIv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "search_start = time.time()"
      ],
      "metadata": {
        "id": "ON-c39RTLHHk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "######## ESearch ########\n",
        "query_term = 'covid%19+OR+SARS%Cov%2'\n",
        "\n",
        "base_url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/'\n",
        "\n",
        "esearch_db = 'db=pubmed'\n",
        "\n",
        "esearch_eutil = 'esearch.fcgi?'\n",
        "\n",
        "esearch_query = '&term=' + query_term\n",
        "\n",
        "esearch_history = '&usehistory=y'\n",
        "\n",
        "esearch_rettype = '&rettype=json'\n",
        "\n",
        "esearch_url = base_url + esearch_eutil + esearch_db + esearch_query + esearch_history + esearch_rettype\n",
        "\n",
        "url_result = urllib.request.urlopen(esearch_url)\n",
        "\n",
        "esearch_url_result = url_result.read().decode('utf-8')\n",
        "\n",
        "total_pmids = int(re.findall(\"<Count>(\\d+?)</Count>\", esearch_url_result)[0])"
      ],
      "metadata": {
        "id": "dAYmg_Cc8Lju"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "esearch_url"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "ZbXp9lO7sOQd",
        "outputId": "30b2b5f7-ea8c-40a5-e9ac-d8db5cd85e23"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=covid%19+OR+SARS%Cov%2&usehistory=y&rettype=json'"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "total_pmids"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "How7sXUQ8WsG",
        "outputId": "cd746377-86e4-4d82-bb66-23380302db6c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "214668"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "######## EFetch ########\n",
        "\n",
        "efetch_webenv = \"&WebEnv=\" + re.findall (\"<WebEnv>(\\S+)<\\/WebEnv>\", esearch_url_result)[0]\n",
        "\n",
        "efetch_querykey = \"&query_key=\" + re.findall(\"<QueryKey>(\\d+?)</QueryKey>\", esearch_url_result)[0]"
      ],
      "metadata": {
        "id": "LkHyG_xQ8f5W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "retmax = 10000\n",
        "retstart = 0\n",
        "all_pmids = list()\n",
        "run = True\n",
        "\n",
        "while run:\n",
        "  efetch_eutil = 'efetch.fcgi?'\n",
        "  efetch_retstart = \"&retstart=\" + str(retstart)\n",
        "  efetch_retmax = \"&retmax=\" + str(retmax)\n",
        "  efetch_retmode = \"&retmode=text\"\n",
        "  efetch_rettype = \"&rettype=uilist\"\n",
        "\n",
        "  efetch_url = base_url + efetch_eutil + esearch_db + efetch_querykey + efetch_webenv + efetch_retstart + efetch_retmax + efetch_retmode + efetch_rettype\n",
        "  efetch_url_result = urllib.request.urlopen(efetch_url)\n",
        "  efetch_data = efetch_url_result.read().decode('utf-8')\n",
        "\n",
        "  all_pmids.extend(efetch_data.splitlines())\n",
        "\n",
        "  print(\"a total of \" + str(len(all_pmids)) + \" PMIDs have been downloaded.\\n\")\n",
        "\n",
        "  time.sleep(2)\n",
        "  retstart = retstart + retmax\n",
        "\n",
        "  if retstart >= total_pmids:\n",
        "    run = False"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nnTluOla8YV2",
        "outputId": "70c84d29-0793-4c23-b04d-d0fec06819b1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "a total of 10000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 20000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 30000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 40000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 50000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 60000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 70000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 80000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 90000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 100000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 110000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 120000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 130000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 140000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 150000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 160000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 170000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 180000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 190000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 200000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 210000 PMIDs have been downloaded.\n",
            "\n",
            "a total of 214668 PMIDs have been downloaded.\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "search_end = time.time() \n",
        "print(f\"Esearch Took:\\t{(search_end-search_start)/60}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "43KX96hALU_z",
        "outputId": "00686717-b584-4844-9d7e-3d93b84fad28"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Esearch Took:\t1.2240100423494975\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fetch_start = time.time()"
      ],
      "metadata": {
        "id": "IccWWRc1EknE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_data = list()\n",
        "\n",
        "for i in tqdm(range(0, len(all_pmids), 10000)):\n",
        "\n",
        "  batch_pmids = all_pmids[i:i+10000]\n",
        "\n",
        "  try:\n",
        "    Entrez.email = 'akishirsath@gmail.com'\n",
        "    handle = Entrez.efetch(db=\"pubmed\", \n",
        "                        sort='best', \n",
        "                        id=\",\".join(batch_pmids), \n",
        "                        rettype=\"medline\", \n",
        "                        retmode=\"text\")\n",
        "\n",
        "    records = Medline.parse(handle)\n",
        "    all_data.append(list(records))\n",
        "    handle.close()\n",
        "    time.sleep(2)\n",
        "  except IncompleteRead:\n",
        "    pass"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Mme8eTZCKKq",
        "outputId": "fce805ff-ccc2-4e5f-f16e-a62c6d0ec504"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 22/22 [1:02:04<00:00, 169.30s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fetch_end = time.time()\n",
        "print(f\"Efetch Took:\\t{(fetch_end-fetch_start)/60}\")"
      ],
      "metadata": {
        "id": "yR1ooc38LJbC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "83ac79b3-f87f-4546-93c0-4adad7d111c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Efetch Took:\t62.07600411574046\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "total_pmids - len(all_data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HRU-rya_EC1a",
        "outputId": "ae337ffe-879a-4136-a8dc-38623af3dd00"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "214647"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(round(fetch_end - fetch_start)/60)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6VGAAXTFLTpT",
        "outputId": "0ce07952-2ffb-4af0-92d3-d116d592c9a9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "62.083333333333336\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(all_data[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jyifrFSdhKCV",
        "outputId": "3e1f89cf-89f4-481a-a8fd-8f63ce69cf25"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10000"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "required_keys = ['PMID', 'TI', 'AB', 'AD', 'FAU', 'DP', 'TA', 'JT', 'LA', \n",
        "                 'MH', 'OAB', 'OT', 'PL', 'PST', 'PT', 'RN', 'SI']"
      ],
      "metadata": {
        "id": "Dbn4y9OcjNcI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "main_data = list()\n",
        "\n",
        "for batch_data in tqdm(all_data):\n",
        "  for single_data in batch_data:\n",
        "    temp = list()\n",
        "    for key in required_keys:\n",
        "      value = single_data.get(key, \"NONE\")\n",
        "      if isinstance(value, list):\n",
        "        value = \";\".join(set(value))\n",
        "        temp.append(value)\n",
        "      else:\n",
        "        temp.append(value)\n",
        "\n",
        "    main_data.append(temp)      "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GQuJh1TYhOhU",
        "outputId": "9e011835-b436-4395-c50d-1b2b0b6a37f1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 21/21 [00:04<00:00,  5.12it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "main_dataframe = pd.DataFrame(main_data, columns=required_keys)"
      ],
      "metadata": {
        "id": "ivv7nkBCivxr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "main_dataframe.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jb1dXIPJjpam",
        "outputId": "f4fcaa1b-0350-4e45-e290-1529bf13955b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 204668 entries, 0 to 204667\n",
            "Data columns (total 17 columns):\n",
            " #   Column  Non-Null Count   Dtype \n",
            "---  ------  --------------   ----- \n",
            " 0   PMID    204668 non-null  object\n",
            " 1   TI      204668 non-null  object\n",
            " 2   AB      204668 non-null  object\n",
            " 3   AD      204668 non-null  object\n",
            " 4   FAU     204668 non-null  object\n",
            " 5   DP      204668 non-null  object\n",
            " 6   TA      204668 non-null  object\n",
            " 7   JT      204668 non-null  object\n",
            " 8   LA      204668 non-null  object\n",
            " 9   MH      204668 non-null  object\n",
            " 10  OAB     204668 non-null  object\n",
            " 11  OT      204668 non-null  object\n",
            " 12  PL      204668 non-null  object\n",
            " 13  PST     204668 non-null  object\n",
            " 14  PT      204668 non-null  object\n",
            " 15  RN      204668 non-null  object\n",
            " 16  SI      204668 non-null  object\n",
            "dtypes: object(17)\n",
            "memory usage: 26.5+ MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "main_dataframe.head(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 243
        },
        "id": "uW-bYR9MjqV_",
        "outputId": "470aafb6-e3a2-424d-ddee-34d856ad9db8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-4c376b91-1eeb-4288-b7a5-72320ddb7825\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>PMID</th>\n",
              "      <th>TI</th>\n",
              "      <th>AB</th>\n",
              "      <th>AD</th>\n",
              "      <th>FAU</th>\n",
              "      <th>DP</th>\n",
              "      <th>TA</th>\n",
              "      <th>JT</th>\n",
              "      <th>LA</th>\n",
              "      <th>MH</th>\n",
              "      <th>OAB</th>\n",
              "      <th>OT</th>\n",
              "      <th>PL</th>\n",
              "      <th>PST</th>\n",
              "      <th>PT</th>\n",
              "      <th>RN</th>\n",
              "      <th>SI</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>35052023</td>\n",
              "      <td>Does a Mental Health Diagnosis Worsen Outcomes...</td>\n",
              "      <td>NONE</td>\n",
              "      <td>Department of Psychiatry, Icahn School of Medi...</td>\n",
              "      <td>Vadukapuram, Ramu;Mansuri, Zeeshan;Trivedi, Ch...</td>\n",
              "      <td>2022 Jan 20</td>\n",
              "      <td>Prim Care Companion CNS Disord</td>\n",
              "      <td>The primary care companion for CNS disorders</td>\n",
              "      <td>eng</td>\n",
              "      <td>NONE</td>\n",
              "      <td>NONE</td>\n",
              "      <td>NONE</td>\n",
              "      <td>United States</td>\n",
              "      <td>epublish</td>\n",
              "      <td>Journal Article</td>\n",
              "      <td>NONE</td>\n",
              "      <td>NONE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>35052021</td>\n",
              "      <td>The Opioid Epidemic and the COVID-19 Pandemic:...</td>\n",
              "      <td>NONE</td>\n",
              "      <td>Department of Psychiatry, AdventHealth Orlando...</td>\n",
              "      <td>Das, Aparna;Padala, Akshay P;Wilson, Kerrie B</td>\n",
              "      <td>2022 Jan 20</td>\n",
              "      <td>Prim Care Companion CNS Disord</td>\n",
              "      <td>The primary care companion for CNS disorders</td>\n",
              "      <td>eng</td>\n",
              "      <td>NONE</td>\n",
              "      <td>NONE</td>\n",
              "      <td>NONE</td>\n",
              "      <td>United States</td>\n",
              "      <td>epublish</td>\n",
              "      <td>Journal Article</td>\n",
              "      <td>NONE</td>\n",
              "      <td>NONE</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4c376b91-1eeb-4288-b7a5-72320ddb7825')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4c376b91-1eeb-4288-b7a5-72320ddb7825 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4c376b91-1eeb-4288-b7a5-72320ddb7825');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "       PMID                                                 TI  ...    RN    SI\n",
              "0  35052023  Does a Mental Health Diagnosis Worsen Outcomes...  ...  NONE  NONE\n",
              "1  35052021  The Opioid Epidemic and the COVID-19 Pandemic:...  ...  NONE  NONE\n",
              "\n",
              "[2 rows x 17 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "main_dataframe.to_csv(\"all-covid-19-pubmed-metadata.csv\", index=False)"
      ],
      "metadata": {
        "id": "V1WCFBQbjvhm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "! cp -r /content/all-covid-19-pubmed-metadata.csv /content/drive/MyDrive/04-Work/EDA-Comparison-21st-Epidemics-PubMed"
      ],
      "metadata": {
        "id": "B2HqEn3bj1p-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Citation"
      ],
      "metadata": {
        "id": "KC68T1UXrhjF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_citation_count(pmid):\n",
        "\n",
        "  url = f\"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=pubmed&linkname=pubmed_pubmed_citedin&id={pmid}\"\n",
        "\n",
        "  url_result = urllib.request.urlopen(url)\n",
        "\n",
        "  esearch_url_result = url_result.read().decode('utf-8')\n",
        "\n",
        "  pmids_list = set(re.findall(\"<Id>(\\d+?)</Id>\", esearch_url_result))\n",
        "\n",
        "  pmids_list.remove(pmid)\n",
        "\n",
        "  return len(pmids_list), \";\".join(list(pmids_list))\n"
      ],
      "metadata": {
        "id": "Z2PfI1DNrkQc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pmid = '34986503'\n",
        "\n",
        "url = f\"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=pubmed&linkname=pubmed_pubmed_citedin&id={pmid}\"\n",
        "\n",
        "url_result = urllib.request.urlopen(url)\n",
        "\n",
        "esearch_url_result = url_result.read().decode('utf-8')\n",
        "\n",
        "pmids_list = set(re.findall(\"<Id>(\\d+?)</Id>\", esearch_url_result))\n",
        "\n",
        "pmids_list.remove(pmid)"
      ],
      "metadata": {
        "id": "7KOpyOF_r89-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(pmids_list)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DUGGQ4BBxP29",
        "outputId": "88395144-8e5d-428b-abcf-d418232aefde"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fin_end = time.time()\n",
        "\n"
      ],
      "metadata": {
        "id": "W1ThelClxT37"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(round(fin_end - search_start)/60)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bGxmONmqkNrW",
        "outputId": "d54bb1c7-f9bb-453a-d3c1-2ec39970a06a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "63.71666666666667\n"
          ]
        }
      ]
    }
  ]
}