make_sports_news.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 155926,
     "status": "ok",
     "timestamp": 1617533431547,
     "user": {
      "displayName": "하상천",
      "photoUrl": "",
      "userId": "08574945924766762951"
     },
     "user_tz": -540
    },
    "id": "Woy_1lEcExTJ",
    "outputId": "d249dabb-675b-47d4-ceac-d9ef66af9565"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting sports_news_collector\n",
      "  Downloading https://files.pythonhosted.org/packages/ed/d7/aed0a1fda52594eb5f3c9c16312b3a6963140fceb4389e8af7efdbe8775e/sports_news_collector-0.1-py3-none-any.whl\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from sports_news_collector) (2.23.0)\n",
      "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from sports_news_collector) (4.6.3)\n",
      "Collecting pretty-html-table\n",
      "  Downloading https://files.pythonhosted.org/packages/00/64/6c8ebfebfe8c07106faf42ce9b51d3f4f378be10b011a59866df5e11b4d0/pretty_html_table-0.9.dev0.tar.gz\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from sports_news_collector) (1.1.5)\n",
      "Collecting pororo\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/78/ab/f409aab13ba2a4e2576d2ea4b877396029c617d17553edbbb9ba64cf4ee9/pororo-0.4.2-py3-none-any.whl (256kB)\n",
      "\u001b[K     |████████████████████████████████| 266kB 5.3MB/s \n",
      "\u001b[?25hRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->sports_news_collector) (1.24.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->sports_news_collector) (2020.12.5)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->sports_news_collector) (2.10)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->sports_news_collector) (3.0.4)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->sports_news_collector) (2.8.1)\n",
      "Requirement already satisfied: numpy>=1.15.4 in /usr/local/lib/python3.7/dist-packages (from pandas->sports_news_collector) (1.19.5)\n",
      "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->sports_news_collector) (2018.9)\n",
      "Collecting fairseq>=0.10.2\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/15/ab/92c6efb05ffdfe16fbdc9e463229d9af8c3b74dc943ed4b4857a87b223c2/fairseq-0.10.2-cp37-cp37m-manylinux1_x86_64.whl (1.7MB)\n",
      "\u001b[K     |████████████████████████████████| 1.7MB 7.7MB/s \n",
      "\u001b[?25hCollecting nltk>=3.5\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/92/75/ce35194d8e3022203cca0d2f896dbb88689f9b3fce8e9f9cff942913519d/nltk-3.5.zip (1.4MB)\n",
      "\u001b[K     |████████████████████████████████| 1.4MB 23.1MB/s \n",
      "\u001b[?25hCollecting word2word\n",
      "  Downloading https://files.pythonhosted.org/packages/0b/27/f6e5252e44fbf10678189aa7c03dfaec44942d2cd593cb957263862a650a/word2word-1.0.0-py3-none-any.whl\n",
      "Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (from pororo->sports_news_collector) (4.2.6)\n",
      "Collecting transformers>=4.0.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ed/d5/f4157a376b8a79489a76ce6cfe147f4f3be1e029b7144fa7b8432e8acb26/transformers-4.4.2-py3-none-any.whl (2.0MB)\n",
      "\u001b[K     |████████████████████████████████| 2.0MB 21.7MB/s \n",
      "\u001b[?25hCollecting wget\n",
      "  Downloading https://files.pythonhosted.org/packages/47/6a/62e288da7bcda82b935ff0c6cfe542970f04e29c756b0e147251b2fb251f/wget-3.2.zip\n",
      "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.7/dist-packages (from pororo->sports_news_collector) (7.1.2)\n",
      "Collecting marisa-trie\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/20/95/d23071d0992dabcb61c948fb118a90683193befc88c23e745b050a29e7db/marisa-trie-0.7.5.tar.gz (270kB)\n",
      "\u001b[K     |████████████████████████████████| 276kB 36.3MB/s \n",
      "\u001b[?25hCollecting kss\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c9/e2/43ac92280810437a552111db85a0379dfaa5ca8ccd81d27a547e9091e5d5/kss-2.5.0-py3-none-any.whl (68kB)\n",
      "\u001b[K     |████████████████████████████████| 71kB 6.1MB/s \n",
      "\u001b[?25hCollecting g2p-en\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d7/d9/b77dc634a7a0c0c97716ba97dd0a28cbfa6267c96f359c4f27ed71cbd284/g2p_en-2.1.0-py3-none-any.whl (3.1MB)\n",
      "\u001b[K     |████████████████████████████████| 3.1MB 32.8MB/s \n",
      "\u001b[?25hCollecting sentence-transformers>=0.4.1.2\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/35/aa/f672ce489063c4ee7a566ebac1b723c53ac0cea19d9e36599cc241d8ed56/sentence-transformers-1.0.4.tar.gz (74kB)\n",
      "\u001b[K     |████████████████████████████████| 81kB 7.1MB/s \n",
      "\u001b[?25hRequirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from pororo->sports_news_collector) (1.0.1)\n",
      "Collecting torch==1.6.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5d/5e/35140615fc1f925023f489e71086a9ecc188053d263d3594237281284d82/torch-1.6.0-cp37-cp37m-manylinux1_x86_64.whl (748.8MB)\n",
      "\u001b[K     |████████████████████████████████| 748.8MB 22kB/s \n",
      "\u001b[?25hCollecting whoosh\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ba/19/24d0f1f454a2c1eb689ca28d2f178db81e5024f42d82729a4ff6771155cf/Whoosh-2.7.4-py2.py3-none-any.whl (468kB)\n",
      "\u001b[K     |████████████████████████████████| 471kB 48.8MB/s \n",
      "\u001b[?25hCollecting torchvision==0.7.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4d/b5/60d5eb61f1880707a5749fea43e0ec76f27dfe69391cdec953ab5da5e676/torchvision-0.7.0-cp37-cp37m-manylinux1_x86_64.whl (5.9MB)\n",
      "\u001b[K     |████████████████████████████████| 5.9MB 26.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->sports_news_collector) (1.15.0)\n",
      "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from fairseq>=0.10.2->pororo->sports_news_collector) (2019.12.20)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from fairseq>=0.10.2->pororo->sports_news_collector) (4.41.1)\n",
      "Collecting sacrebleu>=1.4.12\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/57/0c7ca4e31a126189dab99c19951910bd081dea5bbd25f24b77107750eae7/sacrebleu-1.5.1-py3-none-any.whl (54kB)\n",
      "\u001b[K     |████████████████████████████████| 61kB 6.7MB/s \n",
      "\u001b[?25hRequirement already satisfied: cffi in /usr/local/lib/python3.7/dist-packages (from fairseq>=0.10.2->pororo->sports_news_collector) (1.14.5)\n",
      "Requirement already satisfied: cython in /usr/local/lib/python3.7/dist-packages (from fairseq>=0.10.2->pororo->sports_news_collector) (0.29.22)\n",
      "Collecting dataclasses\n",
      "  Downloading https://files.pythonhosted.org/packages/26/2f/1095cdc2868052dd1e64520f7c0d5c8c550ad297e944e641dbf1ffbb9a5d/dataclasses-0.6-py3-none-any.whl\n",
      "Collecting hydra-core\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/52/e3/fbd70dd0d3ce4d1d75c22d56c0c9f895cfa7ed6587a9ffb821d6812d6a60/hydra_core-1.0.6-py3-none-any.whl (123kB)\n",
      "\u001b[K     |████████████████████████████████| 133kB 55.3MB/s \n",
      "\u001b[?25hRequirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from nltk>=3.5->pororo->sports_news_collector) (7.1.2)\n",
      "Collecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/08/cd/342e584ee544d044fb573ae697404ce22ede086c9e87ce5960772084cad0/sacremoses-0.0.44.tar.gz (862kB)\n",
      "\u001b[K     |████████████████████████████████| 870kB 34.9MB/s \n",
      "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/23/2ddc317b2121117bf34dd00f5b0de194158f2a44ee2bf5e47c7166878a97/tokenizers-0.10.1-cp37-cp37m-manylinux2010_x86_64.whl (3.2MB)\n",
      "\u001b[K     |████████████████████████████████| 3.2MB 44.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers>=4.0.0->pororo->sports_news_collector) (3.0.12)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers>=4.0.0->pororo->sports_news_collector) (20.9)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers>=4.0.0->pororo->sports_news_collector) (3.8.1)\n",
      "Collecting distance>=0.1.3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5c/1a/883e47df323437aefa0d0a92ccfb38895d9416bd0b56262c2e46a47767b8/Distance-0.1.3.tar.gz (180kB)\n",
      "\u001b[K     |████████████████████████████████| 184kB 44.2MB/s \n",
      "\u001b[?25hRequirement already satisfied: inflect>=0.3.1 in /usr/local/lib/python3.7/dist-packages (from g2p-en->pororo->sports_news_collector) (2.1.0)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from sentence-transformers>=0.4.1.2->pororo->sports_news_collector) (0.22.2.post1)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from sentence-transformers>=0.4.1.2->pororo->sports_news_collector) (1.4.1)\n",
      "Collecting sentencepiece\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
      "\u001b[K     |████████████████████████████████| 1.2MB 34.9MB/s \n",
      "\u001b[?25hRequirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from torch==1.6.0->pororo->sports_news_collector) (0.16.0)\n",
      "Collecting portalocker==2.0.0\n",
      "  Downloading https://files.pythonhosted.org/packages/89/a6/3814b7107e0788040870e8825eebf214d72166adf656ba7d4bf14759a06a/portalocker-2.0.0-py2.py3-none-any.whl\n",
      "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi->fairseq>=0.10.2->pororo->sports_news_collector) (2.20)\n",
      "Requirement already satisfied: importlib-resources; python_version < \"3.9\" in /usr/local/lib/python3.7/dist-packages (from hydra-core->fairseq>=0.10.2->pororo->sports_news_collector) (5.1.2)\n",
      "Collecting antlr4-python3-runtime==4.8\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/56/02/789a0bddf9c9b31b14c3e79ec22b9656185a803dc31c15f006f9855ece0d/antlr4-python3-runtime-4.8.tar.gz (112kB)\n",
      "\u001b[K     |████████████████████████████████| 112kB 49.3MB/s \n",
      "\u001b[?25hCollecting omegaconf<2.1,>=2.0.5\n",
      "  Downloading https://files.pythonhosted.org/packages/d0/eb/9d63ce09dd8aa85767c65668d5414958ea29648a0eec80a4a7d311ec2684/omegaconf-2.0.6-py3-none-any.whl\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers>=4.0.0->pororo->sports_news_collector) (2.4.7)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers>=4.0.0->pororo->sports_news_collector) (3.7.4.3)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers>=4.0.0->pororo->sports_news_collector) (3.4.1)\n",
      "Collecting PyYAML>=5.1.*\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7a/a5/393c087efdc78091afa2af9f1378762f9821c9c1d7a22c5753fb5ac5f97a/PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636kB)\n",
      "\u001b[K     |████████████████████████████████| 645kB 44.2MB/s \n",
      "\u001b[?25hBuilding wheels for collected packages: pretty-html-table, nltk, wget, marisa-trie, sentence-transformers, sacremoses, distance, antlr4-python3-runtime\n",
      "  Building wheel for pretty-html-table (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for pretty-html-table: filename=pretty_html_table-0.9.dev0-cp37-none-any.whl size=3940 sha256=a8476f18dc5bc98511e637bc24f1922671f80b0573a831155026aeaabff63e3c\n",
      "  Stored in directory: /root/.cache/pip/wheels/ca/ba/34/2df2877fbdf547891d44fd85fe8a44d8d3c6a007fb349abd9a\n",
      "  Building wheel for nltk (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for nltk: filename=nltk-3.5-cp37-none-any.whl size=1434675 sha256=23f4949b8926728ea339cd055d041431d0e0180123325a6dadac83c445360ca8\n",
      "  Stored in directory: /root/.cache/pip/wheels/ae/8c/3f/b1fe0ba04555b08b57ab52ab7f86023639a526d8bc8d384306\n",
      "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for wget: filename=wget-3.2-cp37-none-any.whl size=9681 sha256=e5d480a75857ed493c66b07dca2f8a8355a3efc712f43a23c756d7bc90051854\n",
      "  Stored in directory: /root/.cache/pip/wheels/40/15/30/7d8f7cea2902b4db79e3fea550d7d7b85ecb27ef992b618f3f\n",
      "  Building wheel for marisa-trie (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for marisa-trie: filename=marisa_trie-0.7.5-cp37-cp37m-linux_x86_64.whl size=861945 sha256=3fa2a6113f74648f51c994021d4aa497e6da8f507e8e0167eb944e2a7664ff59\n",
      "  Stored in directory: /root/.cache/pip/wheels/45/24/79/022624fc914f0e559fe8a1141aaff1f9df810905a13fc75d57\n",
      "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sentence-transformers: filename=sentence_transformers-1.0.4-cp37-none-any.whl size=114307 sha256=c3b5de13773bae7bb9ecf0709b70f337f3cf759c8385e87c32a9ac4ddc64f8a6\n",
      "  Stored in directory: /root/.cache/pip/wheels/26/ea/89/d0d2e013d951b6d23270aa9ca4018b82632ab7cd933c331316\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.44-cp37-none-any.whl size=886084 sha256=bb57f555661ff669d25d988c8a275e5cc6276baad1bd75a9b7740558ca18ed3f\n",
      "  Stored in directory: /root/.cache/pip/wheels/3e/fb/c0/13ab4d63d537658f448366744654323077c4d90069b6512f3c\n",
      "  Building wheel for distance (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for distance: filename=Distance-0.1.3-cp37-none-any.whl size=16261 sha256=f2f37dffd6a89dd5f3a8531993008043488a8e5d11f2fced98b866573d850544\n",
      "  Stored in directory: /root/.cache/pip/wheels/d5/aa/e1/dbba9e7b6d397d645d0f12db1c66dbae9c5442b39b001db18e\n",
      "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-cp37-none-any.whl size=141231 sha256=a3f51cfd632a4a7eaa7bb583b99b112ed31f3d73fc3ef91722ab07f89142d07c\n",
      "  Stored in directory: /root/.cache/pip/wheels/e3/e2/fa/b78480b448b8579ddf393bebd3f47ee23aa84c89b6a78285c8\n",
      "Successfully built pretty-html-table nltk wget marisa-trie sentence-transformers sacremoses distance antlr4-python3-runtime\n",
      "\u001b[31mERROR: torchtext 0.9.1 has requirement torch==1.8.1, but you'll have torch 1.6.0 which is incompatible.\u001b[0m\n",
      "Installing collected packages: pretty-html-table, portalocker, sacrebleu, dataclasses, torch, antlr4-python3-runtime, PyYAML, omegaconf, hydra-core, fairseq, nltk, wget, word2word, sacremoses, tokenizers, transformers, marisa-trie, kss, distance, g2p-en, sentencepiece, sentence-transformers, whoosh, torchvision, pororo, sports-news-collector\n",
      "  Found existing installation: torch 1.8.1+cu101\n",
      "    Uninstalling torch-1.8.1+cu101:\n",
      "      Successfully uninstalled torch-1.8.1+cu101\n",
      "  Found existing installation: PyYAML 3.13\n",
      "    Uninstalling PyYAML-3.13:\n",
      "      Successfully uninstalled PyYAML-3.13\n",
      "  Found existing installation: nltk 3.2.5\n",
      "    Uninstalling nltk-3.2.5:\n",
      "      Successfully uninstalled nltk-3.2.5\n",
      "  Found existing installation: torchvision 0.9.1+cu101\n",
      "    Uninstalling torchvision-0.9.1+cu101:\n",
      "      Successfully uninstalled torchvision-0.9.1+cu101\n",
      "Successfully installed PyYAML-5.4.1 antlr4-python3-runtime-4.8 dataclasses-0.6 distance-0.1.3 fairseq-0.10.2 g2p-en-2.1.0 hydra-core-1.0.6 kss-2.5.0 marisa-trie-0.7.5 nltk-3.5 omegaconf-2.0.6 pororo-0.4.2 portalocker-2.0.0 pretty-html-table-0.9.dev0 sacrebleu-1.5.1 sacremoses-0.0.44 sentence-transformers-1.0.4 sentencepiece-0.1.95 sports-news-collector-0.1 tokenizers-0.10.1 torch-1.6.0 torchvision-0.7.0 transformers-4.4.2 wget-3.2 whoosh-2.7.4 word2word-1.0.0\n"
     ]
    }
   ],
   "source": [
    "!pip install sports_news_collector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 18846,
     "status": "ok",
     "timestamp": 1617533520433,
     "user": {
      "displayName": "하상천",
      "photoUrl": "",
      "userId": "08574945924766762951"
     },
     "user_tz": -540
    },
    "id": "5mSmm2S0P7-G",
    "outputId": "99e0326c-7cd6-4a31-ac4d-6f896ec7cae7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 1012,
     "status": "ok",
     "timestamp": 1617533523517,
     "user": {
      "displayName": "하상천",
      "photoUrl": "",
      "userId": "08574945924766762951"
     },
     "user_tz": -540
    },
    "id": "gRm1nRDrGr1_"
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "executionInfo": {
     "elapsed": 601,
     "status": "ok",
     "timestamp": 1617533531264,
     "user": {
      "displayName": "하상천",
      "photoUrl": "",
      "userId": "08574945924766762951"
     },
     "user_tz": -540
    },
    "id": "pWySwnAbHdQY"
   },
   "outputs": [],
   "source": [
    "class NewsCrawler(object):\n",
    "    def __init__(\n",
    "            self,\n",
    "            event: str,\n",
    "            default_url: str = 'https://sports.news.naver.com/',\n",
    "    ) -> None:\n",
    "        self.event = event\n",
    "        self.default_url = default_url\n",
    "\n",
    "    def __call__(self) -> list:\n",
    "        news_list = list()\n",
    "\n",
    "        url_list = self._url_crawling(self.default_url, self.event)\n",
    "        for url in url_list:\n",
    "            news = self._news_crawling(url).strip()\n",
    "            if news is not None and news[:2] != 'if' and len(news) > 150:\n",
    "                news_list.append(news)\n",
    "\n",
    "        return news_list\n",
    "\n",
    "    def _url_crawling(self, default_url: str, event: str) -> list:\n",
    "        url_list = list()\n",
    "\n",
    "        html = requests.get(default_url + event + '/index.nhn')\n",
    "        soup = BeautifulSoup(html.text, 'html.parser')\n",
    "\n",
    "        news1 = soup.find('ul', class_='home_news_list division')\n",
    "        news2 = soup.find('ul', class_='home_news_list')\n",
    "\n",
    "        urls = news1.find_all('a')\n",
    "\n",
    "        for url in urls:\n",
    "            href = url.attrs['href']\n",
    "            if href is not None:\n",
    "                url_list.append(default_url + href)\n",
    "\n",
    "        urls = news2.find_all('a')\n",
    "        for url in urls:\n",
    "            href = url.attrs['href']\n",
    "            if href is not None:\n",
    "                url_list.append(default_url + href)\n",
    "\n",
    "        return url_list\n",
    "\n",
    "    def _news_crawling(self, url: str) -> str:\n",
    "        # 스포츠 종목과 언론사마다 틀이 달라서 한번에 예외처리 하기가 힘들다.\n",
    "        START_EXCEPTION = []  # 다른 스포츠와 다르게 국내 야구에서는 기자 이름을 맨 뒤에 해서 [xxx기자]형태로 사용함.\n",
    "        END_EXCEPTION = ['▶', '@', '┌']  # 국내 야구에서는 앞부분 사진 이후에 기자 이름 앞에 ⓒ 표시 있음.\n",
    "\n",
    "        html = requests.get(url)\n",
    "        soup = BeautifulSoup(html.text, 'html.parser')\n",
    "\n",
    "        all_news = soup.find(id='newsEndContents').text\n",
    "        start_idx = 0\n",
    "        end_idx = len(all_news)\n",
    "\n",
    "        for i in range(len(all_news)):\n",
    "            if all_news[i] in START_EXCEPTION:\n",
    "                start_idx = i + 1\n",
    "            if all_news[i] in END_EXCEPTION:\n",
    "                end_idx = i\n",
    "                break\n",
    "\n",
    "        all_news = all_news[start_idx: end_idx]\n",
    "\n",
    "        return all_news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 579,
     "status": "ok",
     "timestamp": 1617533527998,
     "user": {
      "displayName": "하상천",
      "photoUrl": "",
      "userId": "08574945924766762951"
     },
     "user_tz": -540
    },
    "id": "m6se4tsrHiMf"
   },
   "outputs": [],
   "source": [
    "from smtplib import SMTP\n",
    "from email.mime.text import MIMEText\n",
    "from email.mime.multipart import MIMEMultipart\n",
    "import getpass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "executionInfo": {
     "elapsed": 599,
     "status": "ok",
     "timestamp": 1617533535460,
     "user": {
      "displayName": "하상천",
      "photoUrl": "",
      "userId": "08574945924766762951"
     },
     "user_tz": -540
    },
    "id": "mU0IqmcXHn4b"
   },
   "outputs": [],
   "source": [
    "class EmailSender(object):\n",
    "    def __init__(\n",
    "            self,\n",
    "            event: str,\n",
    "    ) -> None:\n",
    "        events = {'kbaseball': '국내야구', 'wbaseball': '해외야구', 'kfootball': '국내축구', 'wfootball': '해외축구',\n",
    "                    'basketball': '농구', 'volleyball': '배구', 'golf': '골프', 'general': '일반', 'esports': 'e스포츠 & 게임'}\n",
    "\n",
    "        self.event = events[event]\n",
    "        self.subject = events[event] + ' 뉴스를 전달해드리겠습니다.'\n",
    "        self.to_email = 'seomk9896@gmail.com'  # 여러 명 일 때는 리스트 형태로 표현\n",
    "        self.basic_text = '오늘 하루도 화이팅 하시고, 많은 피드백 부탁드립니다 !\\n\\n'\n",
    "\n",
    "    def __call__(self, body):\n",
    "        message = MIMEMultipart()\n",
    "\n",
    "        message['Subject'] = self.subject\n",
    "        message['From'] = from_email_id\n",
    "        message['To'] = self.to_email\n",
    "\n",
    "        message.attach(MIMEText(self.basic_text, \"plain\"))\n",
    "        message.attach(MIMEText(body, \"html\"))\n",
    "        msg_body = message.as_string()\n",
    "\n",
    "        server = SMTP('smtp.gmail.com', 587)\n",
    "        server.starttls()\n",
    "        server.login(message['From'], from_email_pw)\n",
    "        server.sendmail(message['From'], message['To'], msg_body)\n",
    "\n",
    "        print(self.event + ' 뉴스를 전달하였습니다.')\n",
    "        server.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 5497,
     "status": "ok",
     "timestamp": 1617533543777,
     "user": {
      "displayName": "하상천",
      "photoUrl": "",
      "userId": "08574945924766762951"
     },
     "user_tz": -540
    },
    "id": "eXBcidoQHwXd"
   },
   "outputs": [],
   "source": [
    "from pororo import Pororo\n",
    "from pandas import DataFrame\n",
    "from pretty_html_table import build_table\n",
    "import datetime\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "executionInfo": {
     "elapsed": 675,
     "status": "ok",
     "timestamp": 1617535804164,
     "user": {
      "displayName": "하상천",
      "photoUrl": "",
      "userId": "08574945924766762951"
     },
     "user_tz": -540
    },
    "id": "kByryOuUINJ3"
   },
   "outputs": [],
   "source": [
    "class SportsNewsCollector(object):\n",
    "    def __init__(\n",
    "            self,\n",
    "            event: str,\n",
    "            lang: str = 'ko',\n",
    "            default_url: str = 'https://sports.news.naver.com/',\n",
    "    ) -> None:\n",
    "        self.lang = lang.lower()\n",
    "        self.default_url = default_url\n",
    "        self.summary = Pororo(task='summarization', model='abstractive', lang=self.lang)\n",
    "        self.sentiment_analysis = Pororo(task=\"sentiment\", model=\"brainbert.base.ko.nsmc\", lang=self.lang)\n",
    "\n",
    "        if event in self.available_events():\n",
    "            self.event = event\n",
    "        else:\n",
    "            raise KeyError('Unknown events : {}, available events are {}'.format(event, self.available_events()))\n",
    "\n",
    "    def collect(self) -> None:\n",
    "        collector = NewsCrawler(self.event, self.default_url)\n",
    "        news_list = collector()\n",
    "\n",
    "        news_info = self._make_news_info(news_list)\n",
    "\n",
    "        sender = EmailSender(self.event)\n",
    "        sender(news_info)\n",
    "\n",
    "    def _make_news_info(self, news_list: list) -> DataFrame:\n",
    "        datetime_list = list()\n",
    "        sentiment_list = list()\n",
    "        summary_list = list()\n",
    "        news_info = {'datetime': datetime_list, 'summary': summary_list, 'sentiment': sentiment_list}  \n",
    "\n",
    "        date = str(datetime.datetime.now())\n",
    "        date = date[:-7]\n",
    "\n",
    "        for idx in range(len(news_list)):\n",
    "            datetime_list.append(date)\n",
    "    \n",
    "            summary = self.summary(news_list[idx])\n",
    "            summary_list.append(summary)\n",
    "            sentiment_list.append(self.sentiment_analysis(summary))\n",
    "\n",
    "        news_info = pd.DataFrame(news_info)\n",
    "        news_info = build_table(news_info, 'orange_light')\n",
    "\n",
    "        return news_info\n",
    "\n",
    "    @staticmethod\n",
    "    def available_events() -> list:\n",
    "        return ['kbaseball', 'wbaseball', 'kfootball', 'wfootball', 'basketball', 'volleyball', 'golf', 'general', 'esports']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6502,
     "status": "ok",
     "timestamp": 1617533560015,
     "user": {
      "displayName": "하상천",
      "photoUrl": "",
      "userId": "08574945924766762951"
     },
     "user_tz": -540
    },
    "id": "veXWGwQdQi5e",
    "outputId": "166af479-87ef-4a37-92f7-c15aed6f7190"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seomk9896@gmail.com's password: ··········\n"
     ]
    }
   ],
   "source": [
    "from_email_id = 'seomk9896@gmail.com'\n",
    "from_email_pw = getpass.getpass(from_email_id + \"'s password: \")  #  비밀번호를 계속 입력해야되는 문제 때문에 class에서 뺌."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 362
    },
    "executionInfo": {
     "elapsed": 967474,
     "status": "error",
     "timestamp": 1617536802036,
     "user": {
      "displayName": "하상천",
      "photoUrl": "",
      "userId": "08574945924766762951"
     },
     "user_tz": -540
    },
    "id": "a7DNW2aAJSjQ",
    "outputId": "39beecf6-a810-465e-9c40-e37a7c7e186c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['kbaseball', 'wbaseball', 'kfootball', 'wfootball', 'basketball', 'volleyball', 'golf', 'general', 'esports']\n",
      "뉴스를 가져옵니다. 현재 시각은  2021-04-04 11:30:47 입니다.\n",
      "해외축구 뉴스를 전달하였습니다.\n",
      "뉴스를 가져옵니다. 현재 시각은  2021-04-04 11:34:47 입니다.\n",
      "해외축구 뉴스를 전달하였습니다.\n",
      "뉴스를 가져옵니다. 현재 시각은  2021-04-04 11:38:50 입니다.\n",
      "해외축구 뉴스를 전달하였습니다.\n",
      "뉴스를 가져옵니다. 현재 시각은  2021-04-04 11:42:51 입니다.\n",
      "해외축구 뉴스를 전달하였습니다.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-5e099afdc66d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m   \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"뉴스를 가져옵니다. 현재 시각은 \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdate\u001b[0m \u001b[0;34m,\u001b[0m\u001b[0;34m\"입니다.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m   \u001b[0mcollector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m   \u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from time import sleep\n",
    "print(SportsNewsCollector.available_events())\n",
    "collector = SportsNewsCollector(event='wfootball')\n",
    "\n",
    "while(True):\n",
    "    date = str(datetime.datetime.now())\n",
    "    date = date[:-7]\n",
    "    print(f\"뉴스를 가져옵니다. 현재 시각은 \", date ,\"입니다.\")\n",
    "    collector.collect()\n",
    "    sleep(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 169
    },
    "executionInfo": {
     "elapsed": 1019,
     "status": "error",
     "timestamp": 1617536807075,
     "user": {
      "displayName": "하상천",
      "photoUrl": "",
      "userId": "08574945924766762951"
     },
     "user_tz": -540
    },
    "id": "t9snnR_sSnf6",
    "outputId": "4d797e66-32a2-4d28-e95b-6e71ce0a100f"
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-534b7a74019f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "while True:pass\n",
    "# https://stackoverflow.com/questions/57113226"
   ]
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "authorship_tag": "ABX9TyP8wUiUZ9CKAaUVc4C0/nzp",
   "collapsed_sections": [],
   "name": "make_sports_news.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}