Crawling Test(Move&Crawl).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "naked-destination",
   "metadata": {},
   "source": [
    "# Thread 사용한 Current Page\n",
    "\n",
    "****!주의해야할점****\n",
    "\n",
    "- Crawling 함수는 어떻게 처리해야 할까?  \n",
    "\n",
    "\n",
    "- 페이지 이동은, 과목별로 해야함을 명심하자!  \n",
    "    - 따라서 클래스 쓰레드를 실행시킬때, 인자로 (id, pw, 과목명 혹은 과목ID)를 받도록 하자  \n",
    "    \n",
    "    - 페이지 이동 순서 : 로그인 -> 과목 목차에서 해당 과목 진입 -> 오른쪽 상단 테이블에서 각 카테코리 별 진입\n",
    "    \n",
    "    \n",
    "- ThreadTimer 를 사용하여 주기적으로 실행 가능하게 되었다.  \n",
    "\n",
    "\n",
    "- Thread의 데이터 동기화를 사용하여 DB에 저장할 수 있다.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "typical-controversy",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import time, threading\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "exposed-alert",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyThreadDriver(threading.Thread):\n",
    "    def __init__(self):\n",
    "        threading.Thread.__init__(self)\n",
    "        options = webdriver.ChromeOptions()\n",
    "        options.add_argument(\"headless\")\n",
    "        #options.add_argument(\"disable-gpu\")\n",
    "        #options.add_argument(\"disable-infobars\")\n",
    "        options.add_argument(\"no-sandbox\")\n",
    "        options.add_argument(\"disable-dev-shm-usage\")\n",
    "        #options.add_argument(\"enable-experimental-web-platform-features\")\n",
    "        #options.add_argument(\"default-background-color FFFFFF00\")\n",
    "        self.id = ''\n",
    "        self.pw = ''\n",
    "        self.subject_name = ''\n",
    "        self.delay = 3\n",
    "        self.__driver = webdriver.Chrome(options=options)\n",
    "    \n",
    "    def setCrawlingInfo(self, _id, _pw, _subject_name):\n",
    "        self.id = _id\n",
    "        self.pw = _pw\n",
    "        self.subject_name = _subject_name\n",
    "        \n",
    "    def printLog(self, string):\n",
    "        print(threading.currentThread().getName() + string)\n",
    "#         pass\n",
    "    \n",
    "    def run(self):\n",
    "        crawling_data = []\n",
    "        try:\n",
    "            self.__driver.get('https://klas.kw.ac.kr/')\n",
    "            self.accessToLogin()\n",
    "            self.getIntoSubject()\n",
    "            # 온라인 강의 크롤링\n",
    "            self.goToAssignmentPage()\n",
    "            # 과제 크롤링\n",
    "            self.goToNoticePage()\n",
    "            # 공지사항 크롤링\n",
    "            self.goToAttachmentPage()\n",
    "            # 첨부자료 크롤링\n",
    "        except Exception as inst:\n",
    "            print(inst)\n",
    "            print(inst.args())\n",
    "        finally:\n",
    "            self.closeDriver()\n",
    "        \n",
    "    def accessToLogin(self):\n",
    "        WebDriverWait(self.__driver, self.delay).until(EC.presence_of_element_located((By.ID, \"loginId\")))\n",
    "        #time.sleep(1)\n",
    "        elemId = self.__driver.find_element_by_id(\"loginId\")\n",
    "        elemId.send_keys(self.id)\n",
    "\n",
    "        elemPW = self.__driver.find_element_by_id(\"loginPwd\")\n",
    "        elemPW.send_keys(self.pw)\n",
    "        elemPW.send_keys(Keys.ENTER)\n",
    "        \n",
    "        self.printLog(\"로그인 완료...\")\n",
    "        \n",
    "    def getIntoSubject(self):\n",
    "        # 해당 과목 페이지로 진입\n",
    "        WebDriverWait(self.__driver, self.delay).until(EC.presence_of_element_located((By.CLASS_NAME, \"subjectlist\")))\n",
    "        subjects = self.__driver.find_elements_by_css_selector(\".subjectlist > li > div.left\")\n",
    "        for subject in subjects:\n",
    "            name = subject.text.split()[0]\n",
    "            if name == self.subject_name:\n",
    "                subject.click()\n",
    "                try:\n",
    "                    WebDriverWait(self.__driver, self.delay).until(EC.presence_of_element_located((By.CLASS_NAME, \"btn2\")))\n",
    "                    self.getDataOnlineLectures()\n",
    "                except:\n",
    "                    pass\n",
    "                finally:\n",
    "                    self.printLog(\"과목 페이지 완료\")\n",
    "                    break\n",
    "    \n",
    "    # ---- 과제, 공지사항, 등등 페이지로 넘어갈 때에는 크롤링 하기전에만 Wait을 해주면 괜찮다. -----#\n",
    "    # -- 뒤로가기 시에는 get 메소드가 아니기 때문에 Explicit Wait 작동 X -- #\n",
    "    # 과제 경로 -> 2, 2\n",
    "    def goToAssignmentPage(self):\n",
    "        ## 과제 제출 페이지까지 가는 경로\n",
    "        url = self.getCssSelector(2, 2)\n",
    "        self.__driver.find_element_by_css_selector(url).click()\n",
    "        \n",
    "        try:\n",
    "            WebDriverWait(self.__driver, self.delay).until(EC.presence_of_element_located((By.CLASS_NAME, \"btn-gray\")))\n",
    "            self.getDataAssignments()\n",
    "        except:\n",
    "            pass\n",
    "        finally:\n",
    "            self.printLog(\"과제 페이지 완료\")\n",
    "            self.goToPrevUrl()\n",
    "        \n",
    "    # 공지사항\n",
    "    def goToNoticePage(self):\n",
    "        # WebDriverWait(self.__driver, self.delay).until(EC.presence_of_element_located((By.CLASS_NAME, \"notice-list\")))\n",
    "        url = \".notice-list > div.bodtitle > a\"\n",
    "        self.__driver.find_element_by_css_selector(url).click()\n",
    "        \n",
    "        try:\n",
    "            # Wait Clause\n",
    "            WebDriverWait(self.__driver, self.delay).until(EC.presence_of_element_located((By.CLASS_NAME, \"lft\")))\n",
    "            self.getDataNotices()\n",
    "        except:\n",
    "            pass\n",
    "        finally:\n",
    "            self.printLog(\"공지사항 페이지 완료\")\n",
    "            self.goToPrevUrl()\n",
    "        #time.sleep(1)\n",
    "        \n",
    "    def goToAttachmentPage(self):\n",
    "        #WebDriverWait(self.__driver, self.delay).until(EC.presence_of_element_located((By.CLASS_NAME, \"subjectlist\")))\n",
    "        url = self.getCssSelector(2, 4)\n",
    "        self.__driver.find_element_by_css_selector(url).click()\n",
    "        self.printLog(\"강의자료 페이지 접근완료\")\n",
    "        time.sleep(1)\n",
    "        self.goToPrevUrl()\n",
    "        #time.sleep(1)\n",
    "    \n",
    "    def getCssSelector(self, div, li):\n",
    "        return \".subjectpresentbox > div.tablelistbox > div > div:nth-child({}) > ul > li:nth-child({}) > a\".format(div, li)\n",
    "    \n",
    "    def goToPrevUrl(self):\n",
    "        self.__driver.execute_script(\"window.history.go(-1)\")\n",
    "        \n",
    "    def getDataOnlineLectures(self):\n",
    "        result = []\n",
    "        soup = BeautifulSoup(self.__driver.page_source, 'lxml')\n",
    "        lectures = soup.select(\"#appModule > div:nth-child(2) > div.mt-4.mb-4 > div.tablelistbox > table > tbody > tr\")[1:]\n",
    "\n",
    "        for lecture in lectures:\n",
    "            lecture_info = []\n",
    "            lists = lecture.select(\"td\")\n",
    "            for _list in lists:\n",
    "                lecture_info.append(_list.text.strip())\n",
    "            result.append(lecture_info)\n",
    "        print(result)\n",
    "#         return result\n",
    "    def getDataAssignments(self):\n",
    "        datas = []\n",
    "        soup = BeautifulSoup(self.__driver.page_source, 'lxml')\n",
    "        assignments = soup.select('div.tablelistbox > table > tbody > tr')\n",
    "        for assignment in assignments:\n",
    "            infos = assignment.select('td')\n",
    "            i = 0\n",
    "            data = []\n",
    "            for info in infos:\n",
    "                if i > 3: break\n",
    "                elif i == 1 or i == 3:\n",
    "                    data.append(info.text)\n",
    "                #print(info.text)\n",
    "                i += 1\n",
    "            datas.append(data)\n",
    "        print(datas)\n",
    "#         return datas\n",
    "    def getDataNotices(self):\n",
    "        result = []\n",
    "        soup = BeautifulSoup(self.__driver.page_source, 'lxml')\n",
    "        notices = soup.select(\"#appModule > table > tbody > tr\")\n",
    "        for notice in notices:\n",
    "            notice_info = []\n",
    "            titles = notice.select(\"td\")\n",
    "            for title in titles:\n",
    "                notice_info.append(title.text)\n",
    "            result.append(notice_info)\n",
    "        print(result)\n",
    "#         return result\n",
    "    def add_Notice(lists):\n",
    "        for data in lists:\n",
    "            title=data[1]\n",
    "            writer=data[3]\n",
    "            date=data[4]\n",
    "            contents=\"\"\n",
    "            serialNum=data[0]\n",
    "            subjectID=\"\"\n",
    "\n",
    "            notice=Notice(title, writer,date, contents, serialNum, subjectID)\n",
    "            db_session.add(notice)\n",
    "            db_session.commit()\n",
    "    \n",
    "    def quitDriver(self):\n",
    "        self.__driver.quit()\n",
    "    \n",
    "    def closeDriver(self):\n",
    "        print(threading.currentThread().getName() + \" 종료\")\n",
    "        self.__driver.close()\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "resistant-kentucky",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thread-14로그인 완료...\n"
     ]
    }
   ],
   "source": [
    "# MyThreadDriver('2018203092', 'tkddlf^^12', \"데이터베이스\").start()\n",
    "# MyThreadDriver('2018203039', 'sdh9606^^', \"소프트웨어공학\").start()\n",
    "# MyThreadDriver('2018203092', 'tkddlf^^12', \"정보시스템응용\").start()\n",
    "\n",
    "infoList = [ \n",
    "#     ['2018203092', 'tkddlf^^12', \"데이터베이스\"], \n",
    "    ['2018203039', 'sdh9606^^', \"소프트웨어공학\"],\n",
    "#     ['2018203092', 'tkddlf^^12', \"정보시스템응용\"],\n",
    "    ['2018203092', 'tkddlf^^12', \"데이터마이닝\"]\n",
    "#     ['2018203092', 'tkddlf^^12', \"데이터베이스\"], \n",
    "#     ['2018203039', 'sdh9606^^', \"소프트웨어공학\"],\n",
    "#     ['2018203092', 'tkddlf^^12', \"정보시스템응용\"],\n",
    "#     ['2018203092', 'tkddlf^^12', \"데이터베이스\"], \n",
    "#     ['2018203039', 'sdh9606^^', \"소프트웨어공학\"],\n",
    "#     ['2018203092', 'tkddlf^^12', \"정보시스템응용\"],\n",
    "#     ['2018203092', 'tkddlf^^12', \"데이터베이스\"], \n",
    "#     ['2018203039', 'sdh9606^^', \"소프트웨어공학\"],\n",
    "#     ['2018203092', 'tkddlf^^12', \"정보시스템응용\"],\n",
    "#     ['2018203092', 'tkddlf^^12', \"데이터베이스\"], \n",
    "#     ['2018203039', 'sdh9606^^', \"소프트웨어공학\"],\n",
    "#     ['2018203092', 'tkddlf^^12', \"정보시스템응용\"],\n",
    "#     ['2018203092', 'tkddlf^^12', \"데이터베이스\"], \n",
    "#     ['2018203039', 'sdh9606^^', \"소프트웨어공학\"],\n",
    "#     ['2018203092', 'tkddlf^^12', \"정보시스템응용\"]\n",
    "]\n",
    "\n",
    "\n",
    "for data in infoList:\n",
    "    myThreadDriver = MyThreadDriver()\n",
    "    myThreadDriver.setCrawlingInfo(data[0], data[1], data[2])\n",
    "    myThreadDriver.start()\n",
    "\n",
    "\n",
    "#MyThreadDriver('2018203092', 'tkddlf^^12', \"\").start()\n",
    "# MyThreadDriver('2018203039', 'sdh9606^^').start()\n",
    "# MyThreadDriver('2018203092', 'tkddlf^^12').start()\n",
    "# MyThreadDriver('2018203092', 'tkddlf^^12').start()\n",
    "# MyThreadDriver('2018203039', 'sdh9606^^').start()\n",
    "# MyThreadDriver('2018203092', 'tkddlf^^12').start()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "expired-mitchell",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Main ThreadSubjthread\n",
      " 499999500000\n",
      "### End ###\n",
      "### End ###\n",
      "Thread-15로그인 완료...\n",
      "http://google.com 14078  chars\n",
      "['http://google.com']\n",
      "http://google.com 14142  chars\n",
      "['http://google.com', 'http://google.com']\n",
      "http://google.com 14138  chars\n",
      "[['1', '3', 'Chapter 2. Software Process', '소프트웨어공학) Chapter 02 - Software Process (1)', '2021-03-02 00:00 ~ 2021-03-16 23:59', '[100%]\\xa05분', '보기'], ['1', '4', 'Chapter 2. Software Process', '소프트웨어공학) Chapter 2 - Software Process (2)', '2021-03-02 00:00 ~ 2021-03-16 23:59', '[100%]\\xa036분', '보기'], ['2', '3', 'Chapter 2. Software Process', '소프트웨어공학) Chapter 2 - Software Process (3)', '2021-03-09 00:00 ~ 2021-03-23 23:59', '[100%]\\xa060분', '보기'], ['3', '3', '애자일 소프트웨어 개발', '소프트웨어공학) 애자일 소프트웨어 개발 Part 1', '2021-03-16 00:00 ~ 2021-03-22 23:59', '[0%]\\xa0분', '보기'], ['3', '4', '애자일 소프트웨어 개발', '소프트웨어공학) 애자일 소프트웨어 개발 Part 2', '2021-03-16 00:00 ~ 2021-03-22 23:59', '[0%]\\xa0분', '보기']]\n",
      "Thread-14과목 페이지 완료\n",
      "Thread-14과제 페이지 완료\n",
      "[[' 4', '오픈채팅 ', '', '이윤구', '2021-03-12', '77'], [' 3', '수업시간에 퀴즈 맞춘 학생 ', '', '이윤구', '2021-03-02', '86'], [' 2', '출석체크 ', '', '이윤구', '2021-03-02', '85'], [' 1', '수업공지 ', '', '이윤구', '2021-03-01', '93']]\n",
      "Thread-14공지사항 페이지 완료\n",
      "Thread-14강의자료 페이지 접근완료\n",
      "Thread-14 종료\n",
      "[['1', '1', '과목 오리엔테이션 및 다변량 데이터 분석 소개', '21y1s_데이터마이닝_1W_소개자', '2021-03-02 00:00 ~ 2021-03-16 23:59', '[100%]\\xa020분', '보기'], ['1', '2', '과목 오리엔테이션 및 다변량 데이터 분석 소개', '21y_DM_intro1', '2021-03-02 00:00 ~ 2021-03-23 23:59', '[100%]\\xa025분', '보기'], ['1', '3', '과목 오리엔테이션 및 다변량 데이터 분석 소개', '21y_DM_intro2', '2021-03-02 00:00 ~ 2021-03-23 23:59', '[100%]\\xa030분', '보기'], ['2', '1', '기초 수학 및 통계 리뷰 I (Statistics review)', '21y_1st_기초선형대수1', '2021-03-08 00:00 ~ 2021-03-29 23:59', '[100%]\\xa025분', '보기'], ['2', '2', '기초 수학 및 통계 리뷰 I (Statistics review)', '21y_1st_기초선형대수2', '2021-03-08 00:00 ~ 2021-03-29 23:59', '[100%]\\xa030분', '보기'], ['2', '3', '기초 수학 및 통계 리뷰 I (Statistics review)', 'Probability and Statistics', '2021-03-08 00:00 ~ 2021-03-29 23:59', '[100%]\\xa060분', '보기'], ['3', '1', 'R 및 Python 기초강의', 'R 프로그래밍 언어 소개', '2021-03-15 00:00 ~ 2021-03-31 23:59', '[0%]\\xa0분', '보기'], ['3', '2', 'R 및 Python 기초강의', '파이썬 언어 소개', '2021-03-15 00:00 ~ 2021-03-31 23:59', '[0%]\\xa0분', '보기'], ['3', '3', 'R 및 Python 기초강의', '21y_1st_DM_3W_R개발', '2021-03-15 00:00 ~ 2021-03-31 23:59', '[0%]\\xa0분', '보기'], ['3', '4', 'R 및 Python 기초강의', '21y_1st_DM_3W_R통계및확', '2021-03-16 00:00 ~ 2021-04-07 23:59', '[0%]\\xa0분', '보기']]\n",
      "Thread-15과목 페이지 완료\n",
      "[['HW1. 자기소개자료 제출', '제출'], []]\n",
      "Thread-15과제 페이지 완료\n",
      "[[' 7', '※ 강의영상 음질 문제 ', '', '이상민', '2021-03-17', '23'], [' 6', '자기소개 자료 체점결과 및 후기 ', '', '이상민', '2021-03-16', '32'], [' 5', '3주차 수업공지 [Python 실습영상은 차주이후 게재] ', '', '이상민', '2021-03-16', '23'], [' 4', '3주차 수업공지 [R, Python programming] ', '', '이상민', '2021-03-15', '31'], [' 3', '자기소개자료 과제제출 기한연장 (일요일까지) ', '', '이상민', '2021-03-11', '28'], [' 2', '3/10 수업 긴급공지 (비대면 전환) ', '', '이상민', '2021-03-10', '31'], [' 1', '수업 공지 ', '', '이상민', '2021-02-28', '37']]\n",
      "Thread-15공지사항 페이지 완료\n",
      "Thread-15강의자료 페이지 접근완료\n",
      "Thread-15 종료\n"
     ]
    }
   ],
   "source": [
    "import threading\n",
    "\n",
    "def sum(low, high):\n",
    "    total=0\n",
    "    for i in range(low, high):\n",
    "        total +=i\n",
    "    print(\"Subjthread\", total)\n",
    "t = threading.Thread(target=sum, args=(1, 1000000))\n",
    "\n",
    "t.start()\n",
    "\n",
    "print(\"Main Thread\")\n",
    "\n",
    "import threading, requests, time\n",
    "\n",
    "def getHtml(url):\n",
    "    resp = requests.get(url)\n",
    "    time.sleep(1)\n",
    "    print(url, len(resp.text), ' chars')\n",
    "\n",
    "t1 = threading.Thread(target=getHtml, args=('http://google.com', ))\n",
    "t1.start()\n",
    "\n",
    "print(\"### End ###\")\n",
    "\n",
    "data = []\n",
    "\n",
    "class HtmlGetter(threading.Thread):\n",
    "    def __init__(self, url):\n",
    "        threading.Thread.__init__(self)\n",
    "        self.url = url\n",
    "        \n",
    "    def run(self):\n",
    "        resp = requests.get(self.url)\n",
    "        time.sleep(1)\n",
    "        print(self.url, len(resp.text), ' chars')\n",
    "        global data\n",
    "        data.append(self.url)\n",
    "        print(data)\n",
    "        \n",
    "HtmlGetter('http://google.com').start()\n",
    "HtmlGetter('http://google.com').start()\n",
    "\n",
    "print(\"### End ###\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "short-party",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from threading import Thread"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "planned-factor",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ThreadWithReturnValue(Thread):\n",
    "    def __init__(self, group=None, target=None, name=None,\n",
    "                 args=(), kwargs={}, Verbose=None):\n",
    "        Thread.__init__(self, group, target, name, args, kwargs)\n",
    "        self._return = None\n",
    "    def run(self):\n",
    "        print(type(self._target))\n",
    "        if self._target is not None:\n",
    "            self._return = self._target(*self._args,\n",
    "                                                **self._kwargs)\n",
    "    def join(self):\n",
    "        Thread.join(self)\n",
    "        return self._return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "excess-crawford",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'function'>\n",
      "Thread-20 1\n",
      "<class 'function'>\n",
      "Thread-21 1\n",
      "Thread-20 2\n",
      "Thread-21 2\n",
      "Thread-20 3\n",
      "Thread-21 3\n",
      "Thread-20 4\n",
      "Thread-21 4\n",
      "Thread-20Thread-21 5\n",
      " 5\n",
      "Thread-20Thread-21  6\n",
      "6\n",
      "Thread-20Thread-21 7\n",
      " 7\n",
      "Thread-21 8\n",
      "Thread-20 8\n",
      "Thread-21 9\n",
      "Thread-20 9\n",
      "Thread-21 10\n",
      "Thread-20 10\n",
      "Thread-21 11\n",
      "Thread-20  I am done\n",
      "Thread-21 12\n",
      "Thread-21 13\n",
      "Thread-21 14\n",
      "Thread-21 15\n",
      "Thread-21  I am done\n"
     ]
    }
   ],
   "source": [
    "def count(n):\n",
    "    for i in range(n):\n",
    "        print(threading.currentThread().getName(), i + 1)\n",
    "        time.sleep(0.5)\n",
    "    return threading.currentThread().getName() + '  I am done'\n",
    "\n",
    "t1 = ThreadWithReturnValue(target=count, args=(10,))\n",
    "t2 = ThreadWithReturnValue(target=count, args=(15,))\n",
    "\n",
    "t1.start()\n",
    "t2.start()\n",
    "\n",
    "print(t1.join())\n",
    "print(t2.join())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "reliable-technology",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}