EP_Career_History_Scraper.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "EP Career History Scraper",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyOs/4tdBpOO234oZLOUhasA",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "44c2d58aea204140900d51feab38298d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c7d3c395a8c74904ae3564ab778837ea",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cf5fa152db714e468d3b34b2f19496a7",
              "IPY_MODEL_b903366768514dd79e2c4611f2f1df2e"
            ]
          }
        },
        "c7d3c395a8c74904ae3564ab778837ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cf5fa152db714e468d3b34b2f19496a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cb22c6e733294b1e95ef8ce8e14d26ab",
            "_dom_classes": [],
            "description": "Leagues:   0%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 0,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1ee0f7dac52d4232a7302f89039f1419"
          }
        },
        "b903366768514dd79e2c4611f2f1df2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4b4364cbea9c4b5e8cdbd003518b45bc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0/1 [00:24&lt;?, ?it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c6a6c047ad334badb6b0b7f6efed3c23"
          }
        },
        "cb22c6e733294b1e95ef8ce8e14d26ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1ee0f7dac52d4232a7302f89039f1419": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4b4364cbea9c4b5e8cdbd003518b45bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c6a6c047ad334badb6b0b7f6efed3c23": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "54bf772a41fb466a985b5ab473e4022f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_709ac977f0604fc384311dcb3eacdd53",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_50c928d6e9a945c2b49ac81e088f2864",
              "IPY_MODEL_cfe6c69d8f1e430fbeed5ce48a98b9a7"
            ]
          }
        },
        "709ac977f0604fc384311dcb3eacdd53": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "50c928d6e9a945c2b49ac81e088f2864": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0515513b91d1424283ec0d1d9bf4937f",
            "_dom_classes": [],
            "description": "Players:   3%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 258,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 8,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6d51d525d5b642aca4d5e2f8a671fbe9"
          }
        },
        "cfe6c69d8f1e430fbeed5ce48a98b9a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_faeed857660747ed8e1c5149a3187f72",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8/258 [00:16&lt;08:22,  2.01s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_18f27bc331714599a6fc221edd2deb86"
          }
        },
        "0515513b91d1424283ec0d1d9bf4937f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6d51d525d5b642aca4d5e2f8a671fbe9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "faeed857660747ed8e1c5149a3187f72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "18f27bc331714599a6fc221edd2deb86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/colinrsmall/AATPt/blob/master/EP_Career_History_Scraper.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S_MhoPkzgQS0",
        "colab_type": "text"
      },
      "source": [
        "# Instructions:\n",
        "\n",
        "To add leagues to the list, copy and paste an entry in the following list and replace the league's name and EliteProspects link with the name and link of the league you want to scrape. Make sure that all entries except the last end with a comma (as you can see with the first entry). The name you choose for the entry only influences the name of the output file. You can get a league's URL by going to the league's homepage on EP and copying the URL for that page from your browser.\n",
        "\n",
        "To change which season you're scraping for, change the season string following the list of leagues. The string should be of the format 'YYYY-YYYY' (such as '2019-2020' or '2017-2018'). This will the season for which the scraper gets players. For example, setting this to 2020-21 will scrape all players that are contracted in the given league for that year.\n",
        "\n",
        "Change latest_season to the season you want stop scraping after. For example, if you want to get a player's history up to and including the 2007-08 season, set this to 2007-08.\n",
        "\n",
        "If you want the scraper to print out links for players who are missing information on their EP page, change show_error_links to True.\n",
        "\n",
        "To run the scraper, click runtime -> run all."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1tzBQoJsgDoG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "leagues = [\n",
        "           ('del2', 'https://www.eliteprospects.com/league/del2')\n",
        "]\n",
        "latest_season = \"2020-21\"\n",
        "season_to_scrape = \"2019-20\"\n",
        "show_error_links = True"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vy7JIACigLs-",
        "colab_type": "text"
      },
      "source": [
        "# Expand this if you want to look at the code (optional)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jiJNPVH7hiXQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir '/content/leagues/'"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x9_CSQLbjXrR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "f0bbf518-94e1-46b2-d67c-b2279e8e5906"
      },
      "source": [
        "!pip install unidecode"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: unidecode in /usr/local/lib/python3.6/dist-packages (1.1.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nA6_8mgBLj2H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import requests, random, csv, traceback, time, urllib.request\n",
        "from bs4 import BeautifulSoup\n",
        "from tqdm.notebook import tqdm\n",
        "from datetime import datetime\n",
        "from google.colab import files\n",
        "import unidecode"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LG9qclV3wWHz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_name(player_page):\n",
        "  name = name = player_page.find('div', class_='ep-entity-header__name').text.strip().split(' ')\n",
        "  first_name = name[0]\n",
        "  last_name = ' '.join(name[1:]).split('\\n')[0].strip()\n",
        "  first_name = unidecode.unidecode(first_name)\n",
        "  last_name = unidecode.unidecode(last_name)\n",
        "  return first_name, last_name\n",
        "\n",
        "\n",
        "def get_dob(player_page):\n",
        "  dob_search_text = \"\"\"\n",
        "                                    Date of Birth\n",
        "                                \"\"\"\n",
        "  try:\n",
        "    dob_text = player_page.find('div', text=dob_search_text).next_element.next_element.next_element.text.strip()\n",
        "    dob = datetime.strptime(dob_text, '%b %d, %Y').strftime('%-d.%-m.%Y')\n",
        "  except Exception as e:\n",
        "    try:\n",
        "      dob = datetime.strptime(dob_text, '%Y').strftime('1.1.%Y')\n",
        "    except Exception as e:\n",
        "      dob = \"\"\n",
        "      print(f'Missing dob information: {get_name(player_page)[0]} {get_name(player_page)[1]}')\n",
        "  return dob\n",
        "\n",
        "def scrape_history(player_page):\n",
        "  season_text = \"\"\n",
        "  seasons = []\n",
        "  for season in player_page.select(\".player-statistics.regular.postseason\")[0].select('tr[class*=\"team-continent-\"]'):\n",
        "      p_gp = \"0\"\n",
        "      p_g = \"0\"\n",
        "      p_a = \"0\"\n",
        "      p_pim = \"0\"\n",
        "      p_pm = \"0\"\n",
        "\n",
        "      if season.select(\".season.sorted\")[0].text.strip() != \"\":\n",
        "        season_text = season.select(\".season.sorted\")[0].text.strip()\n",
        "      \n",
        "      if season_text == latest_season:\n",
        "        break\n",
        "\n",
        "      if len(season.select(\"fa.fa-loan\")) > 0:\n",
        "        loan = \"y\"\n",
        "      else:\n",
        "        loan = \"n\"\n",
        "\n",
        "      club = season.select(\".team\")[0].text.strip()\n",
        "      if '“A”' in club or '“C”' in club:\n",
        "        club = club[:-5]\n",
        "      club = unidecode.unidecode(club)\n",
        "\n",
        "      competition = season.select(\".league\")[0].text.strip()\n",
        "      r_gp = season.select(\".regular.gp\")[0].text.strip().replace(\"-\", \"0\")\n",
        "      r_g = season.select(\".regular.g\")[0].text.strip().replace(\"-\", \"0\")\n",
        "      r_a = season.select(\".regular.a\")[0].text.strip().replace(\"-\", \"0\")\n",
        "      r_pim = season.select(\".regular.pim\")[0].text.strip().replace(\"-\", \"0\")\n",
        "      r_pm = season.select(\".regular.pm\")[0].text.strip().replace(\"-\", \"0\")\n",
        "\n",
        "      playoffs = None\n",
        "      if season.select(\".postseason.gp\")[0].text.strip() != \"-\" and season.select(\".postseason\")[0].text.strip() == \"Playoffs\":\n",
        "        p_gp = season.select(\".postseason.gp\")[0].text.strip().replace(\"-\", \"0\")\n",
        "        p_g = season.select(\".postseason.g\")[0].text.strip().replace(\"-\", \"0\")\n",
        "        p_a = season.select(\".postseason.a\")[0].text.strip().replace(\"-\", \"0\")\n",
        "        p_pim = season.select(\".postseason.pim\")[0].text.strip().replace(\"-\", \"0\")\n",
        "        p_pm = season.select(\".postseason.pm\")[0].text.strip().replace(\"-\", \"0\")\n",
        "        playoffs = [loan, \"y\", season_text, club, competition, p_gp, p_g, p_a, p_pim, p_pm, 0, 0, 0, 0, 0, 0, 0]\n",
        "\n",
        "      regular_season = [loan, \"n\", season_text, club, competition, r_gp, r_g, r_a, r_pim, r_pm, 0, 0, 0, 0, 0, 0, 0]\n",
        "      seasons.append((regular_season, playoffs))\n",
        "\n",
        "  return seasons\n",
        "\n",
        "def scrape_player_page(link, csvwr):\n",
        "  player_page = requests.get(link, headers = {\n",
        "        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',\n",
        "        'referrer': 'https://google.com',\n",
        "        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',\n",
        "        'Accept-Encoding': 'gzip, deflate, br',\n",
        "        'Accept-Language': 'en-US,en;q=0.9',\n",
        "        'Pragma': 'no-cache',\n",
        "    })\n",
        "  \n",
        "  player_page = BeautifulSoup(player_page.content)\n",
        "  first_name, last_name = get_name(player_page)\n",
        "  dob = get_dob(player_page)\n",
        "  position_text = \"\"\"\n",
        "                                    Position\n",
        "                                \"\"\"\n",
        "  position = player_page.find('div', text=position_text).next_element.next_element.next_element.text.strip()\n",
        "\n",
        "  if position != \"G\":\n",
        "    seasons = scrape_history(player_page)\n",
        "    \n",
        "    for (regular_season, playoffs) in seasons:\n",
        "      csvwr.writerow([first_name, last_name, dob] + regular_season)\n",
        "      if playoffs is not None:\n",
        "        csvwr.writerow([first_name, last_name, dob] + playoffs)"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ytrwo4LJP1Rb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def scrape():\n",
        "  season_suffix = season_to_scrape[:5] + '20' + season_to_scrape[5:]\n",
        "  for league in tqdm(leagues, desc='Leagues'):\n",
        "    # Get draft page's HTML and parse with BeautifulSoup\n",
        "    home_page_link = league[1]+'/'+season_suffix\n",
        "    home_page_page = requests.get(home_page_link)\n",
        "    home_page = BeautifulSoup(home_page_page.content)\n",
        "\n",
        "    with open(f'/content/leagues/{league[0]}.csv', 'w+', newline='', encoding='UTF-8') as csvfile:\n",
        "      csvfile.write('\\ufeff')\n",
        "      csvwriter = csv.writer(csvfile, delimiter=',')\n",
        "      top_header = 'staff_history, dmy'\n",
        "      second_header = 'First name, Second name, Date of Birth, On loan, Playoffs, Year, Club, Competition, GP, G, A, PIM, +/-, Mins, GA, SO, W, L, T/OT, Saves, Edit Club, Edit Competition'\n",
        "      csvwriter.writerow(top_header.split(','))\n",
        "      csvwriter.writerow(second_header.split(','))\n",
        "\n",
        "      player_links = []\n",
        "      team_links = set([team['href'] for team in home_page.select('table.standings.table-sortable > tbody > tr > .team > a')])\n",
        "      if not team_links:\n",
        "        team_links = set([(team['href']+'/'+season_suffix) for team in home_page.select('.inner-rtl .leg-home-inner .list-as-columns > .column-4 > li > a')])\n",
        "      for team_link in team_links:\n",
        "        team_page = requests.get(team_link)\n",
        "        team_page = BeautifulSoup(team_page.content)\n",
        "        players = team_page.select('[data-sort-ajax-container=\"#roster\"] > tbody > tr .txt-blue a[href]')\n",
        "        player_links += [player['href'] for player in players]\n",
        "\n",
        "      for link in tqdm(player_links, desc='Players', leave=False):\n",
        "        try:\n",
        "          scrape_player_page(link, csvwriter)  \n",
        "        except Exception as e:\n",
        "          if \"team-captaincy\" not in link and \"nation?total\" not in link and 'apple-touch-icon' not in link and 'player_page.find' not in link and show_error_links:\n",
        "            traceback.print_exc()\n",
        "            print(f'Missing player information for: {link}')\n",
        "\n",
        "        time.sleep(random.random() * 3)"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_H37N4evh1Qc",
        "colab_type": "text"
      },
      "source": [
        "# Output\n",
        "\n",
        "You should see three progress bars: one showing the progress through the leagues you want to scrape, one showing progress through all of the teams for a given league, and one showing progress through all of the players for a given team.\n",
        "\n",
        "Some players (often in low-level or obscure leagues) will be missing information such as shooting hand, height, weight, or full date of birth. If you set show_error_links to be True in the first cell, and if the scraper comes across such a player, it will print out a small error message stating \"Missing player information for: \" followed by a link to the player's EP page. The scraper will still include the player in the output CSV files, although some fields for that player will be empty.\n",
        "\n",
        "To download the .zip, can click the folder icon on the bar to the left of the screen and right-click -> download file the file 'leagues.zip'."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "girLjYxJh2cn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 322,
          "referenced_widgets": [
            "44c2d58aea204140900d51feab38298d",
            "c7d3c395a8c74904ae3564ab778837ea",
            "cf5fa152db714e468d3b34b2f19496a7",
            "b903366768514dd79e2c4611f2f1df2e",
            "cb22c6e733294b1e95ef8ce8e14d26ab",
            "1ee0f7dac52d4232a7302f89039f1419",
            "4b4364cbea9c4b5e8cdbd003518b45bc",
            "c6a6c047ad334badb6b0b7f6efed3c23",
            "54bf772a41fb466a985b5ab473e4022f",
            "709ac977f0604fc384311dcb3eacdd53",
            "50c928d6e9a945c2b49ac81e088f2864",
            "cfe6c69d8f1e430fbeed5ce48a98b9a7",
            "0515513b91d1424283ec0d1d9bf4937f",
            "6d51d525d5b642aca4d5e2f8a671fbe9",
            "faeed857660747ed8e1c5149a3187f72",
            "18f27bc331714599a6fc221edd2deb86"
          ]
        },
        "outputId": "a91ca656-bce8-4a4e-92a3-7e972420a59e"
      },
      "source": [
        "scrape()"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "44c2d58aea204140900d51feab38298d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Leagues', max=1.0, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "54bf772a41fb466a985b5ab473e4022f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Players', max=258.0, style=ProgressStyle(description_widt…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-40-20ef00fd5394>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mscrape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-39-7a381bbb47b4>\u001b[0m in \u001b[0;36mscrape\u001b[0;34m()\u001b[0m\n\u001b[1;32m     33\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Missing player information for: {link}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m         \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dt5CkyAnnAse",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "time.sleep(5)\n",
        "!zip leagues.zip -r '/content/leagues/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "inamXWxnt1CE",
        "colab_type": "text"
      },
      "source": [
        "# Testing code, no need to look here"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Rsi0uIysYPd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "player_page_link = 'https://www.eliteprospects.com/player/42871/carl-hudson'\n",
        "player_page = requests.get(player_page_link, headers = {\n",
        "      'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',\n",
        "      'referrer': 'https://google.com',\n",
        "      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',\n",
        "      'Accept-Encoding': 'gzip, deflate, br',\n",
        "      'Accept-Language': 'en-US,en;q=0.9',\n",
        "      'Pragma': 'no-cache',\n",
        "  })\n",
        "player_page = BeautifulSoup(player_page.content)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dtz2fEf5hPer",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "player_page.select(\".team-continent-NA\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "atzNZoT5kMgN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "season_text = \"\"\n",
        "for season in player_page.select(\".player-statistics.regular.postseason\")[0].select(\".team-continent-EU\"):\n",
        "    p_gp = \"0\"\n",
        "    p_g = \"0\"\n",
        "    p_a = \"0\"\n",
        "    p_pim = \"0\"\n",
        "    p_pm = \"0\"\n",
        "\n",
        "    if season.select(\".season.sorted\")[0].text.strip() != \"\":\n",
        "      season_text = season.select(\".season.sorted\")[0].text.strip()\n",
        "    \n",
        "    if season_text == latest_season:\n",
        "      break\n",
        "\n",
        "    if len(season.select(\"fa.fa-loan\")) > 0:\n",
        "      loan = \"y\"\n",
        "    else:\n",
        "      loan = \"n\"\n",
        "\n",
        "    club = season.select(\".team\")[0].text.strip()\n",
        "    if '“A”' in club or '“C”' in club:\n",
        "      club = club[:-5]\n",
        "\n",
        "    competition = season.select(\".league\")[0].text.strip()\n",
        "    r_gp = season.select(\".regular.gp\")[0].text.strip()\n",
        "    r_g = season.select(\".regular.g\")[0].text.strip()\n",
        "    r_a = season.select(\".regular.a\")[0].text.strip()\n",
        "    r_pim = season.select(\".regular.pim\")[0].text.strip()\n",
        "    r_pm = season.select(\".regular.pm\")[0].text.strip()\n",
        "\n",
        "    playoffs = None\n",
        "    if season.select(\".postseason.gp\")[0].text.strip() != \"\" or season.select(\".postseason.gp\")[0].text.strip() != \"-\":\n",
        "      p_gp = season.select(\".postseason.gp\")[0].text.strip()\n",
        "      p_g = season.select(\".postseason.g\")[0].text.strip()\n",
        "      p_a = season.select(\".postseason.a\")[0].text.strip()\n",
        "      p_pim = season.select(\".postseason.pim\")[0].text.strip()\n",
        "      p_pm = season.select(\".postseason.pm\")[0].text.strip()\n",
        "      playoffs = [loan, \"y\", season_text, club, competition, p_gp, r_g, p_a, p_pim, p_pm, 0, 0, 0, 0, 0, 0, 0]\n",
        "\n",
        "    regular_season = [loan, \"n\", season_text, club, competition, r_gp, r_g, r_a, r_pim, r_pm, 0, 0, 0, 0, 0, 0, 0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "THsBk28ogRtp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "player_page.select(\".player-statistics.regular.postseason\")[0].select('tr[class*=\"team-continent-\"]')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h8BJn9c4UkSq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}