From 3449bbe989fcaf5266506608878f0ebf519a713e Mon Sep 17 00:00:00 2001 From: Xanonymous Date: Sun, 29 Mar 2020 01:00:02 +0800 Subject: [PATCH 1/7] add board crawler, link to flask, mount to site, add selectBox with animates, fix main-container flex properties. --- app.py | 14 +++++++++++--- crawler.py | 30 ++++++++++++++++++++++++------ website-new/css/main.css | 7 ++++--- website-new/index.html | 6 +++++- website-new/js/index.js | 32 +++++++++++++++++++++++++++----- 5 files changed, 71 insertions(+), 18 deletions(-) diff --git a/app.py b/app.py index aa45d84..ddb879a 100644 --- a/app.py +++ b/app.py @@ -12,18 +12,26 @@ def give_html(): @app.route('/api/articles//info', methods=['GET']) def article_info(article_id): - return cra.get_article(article_id, is_include_content=False) + board = request.args.get('board') + return cra.get_article(article_id, board, is_include_content=False) @app.route('/api/articles/', methods=['GET']) def article_content(article_id): - return cra.get_article(article_id, is_include_content=True) + board = request.args.get('board') + return cra.get_article(article_id, board, is_include_content=True) @app.route('/api/articles', methods=['GET']) def article_ids(): page = request.args.get('page') - return cra.get_article_ids(page) + board = request.args.get('board') + return cra.get_article_ids(board, page) + + +@app.route('/api/boards', methods=['GET']) +def get_boards(): + return cra.get_popular_boards() if __name__ == '__main__': diff --git a/crawler.py b/crawler.py index ae72d0b..6ddadc3 100644 --- a/crawler.py +++ b/crawler.py @@ -1,9 +1,11 @@ import requests from bs4 import BeautifulSoup +base_url = "https://www.ptt.cc/bbs/" + # 取出頁數或是 id -def get_last_session_of_url(url): +def get_last_session_of_url(url) -> str: url_split = url.split('/') page_or_id = url_split[3] page_or_id = page_or_id.replace('index', '') @@ -11,8 +13,11 @@ def get_last_session_of_url(url): return page_or_id -def get_article(id_, is_include_content=False): - url = 'https://www.ptt.cc/bbs/Gossiping/' + id_ + '.html' +def get_article(id_, board="Gossiping", is_include_content=False) -> dict: + global base_url + if board is None: + board = "Gossiping" + url = base_url + board + "/" + id_ + '.html' # 以 GET 傳請求給目標伺服器，伺服器回傳 response 物件 # response 接收回傳值 @@ -51,13 +56,16 @@ def get_article(id_, is_include_content=False): return articles -def get_article_ids(page=""): - +def get_article_ids(board="Gossiping", page="") -> dict: + global base_url if page is None: page = "" + if board is None: + board = "Gossiping" + # format 中的內容會替換 {} 所在的位置 - url = 'https://www.ptt.cc/bbs/Gossiping/index{}.html'.format(page) + url = base_url + board + '/index{}.html'.format(page) cookies = dict(over18="1") response = requests.get(url, cookies=cookies) @@ -96,3 +104,13 @@ def get_article_ids(page=""): error = soup.find('title') error_message = dict(error=error.get_text()) return error_message + + +def get_popular_boards() -> dict: + url = "https://www.ptt.cc/bbs/index.html" + response = requests.get(url) + if not response or response.status_code != 200: + return dict(boards=["Gossiping"]) + soup = BeautifulSoup(response.text, 'html.parser') + board_names = [div.get_text() for div in soup.find_all("div", class_="board-name")] + return dict(boards=board_names) diff --git a/website-new/css/main.css b/website-new/css/main.css index 2b309a6..e2c2502 100755 --- a/website-new/css/main.css +++ b/website-new/css/main.css @@ -26,7 +26,7 @@ html, body { #main-container .main-text { font-size: xx-large; font-weight: bolder; - padding: 20px 0 10px; + padding: 20px 0 0; } .colorful-line { @@ -110,9 +110,10 @@ html, body { #main-title { display: flex; - justify-content: space-between; - flex-wrap: wrap; + justify-content: start; + flex-wrap: nowrap; align-items: end; + white-space: nowrap; } #page-title { diff --git a/website-new/index.html b/website-new/index.html index 4600500..cf5ff36 100644 --- a/website-new/index.html +++ b/website-new/index.html @@ -14,7 +14,11 @@
- ✾ Hey! PTT Gossiping + ✾ Hey! PTT + + + +
diff --git a/website-new/js/index.js b/website-new/js/index.js index dd24e2d..94cd1de 100644 --- a/website-new/js/index.js +++ b/website-new/js/index.js @@ -27,6 +27,7 @@ let loadingIconChildren = loadingIcon.children; colorBars.forEach((coloBar) => { coloBar.style.background = "linear-gradient(to top right," + "#" + randomColor() + "," + "#" + randomColor() + ")"; }); + await showBoardsSelectorOptions(); await showList(); }()); @@ -68,18 +69,37 @@ function leaveError() { async function getArticleWithContent(articleId) { return await request("/api/articles/" + articleId, "GET") - .catch((e) => errorHandler(e)); + .catch((e) => errorHandler(e)) || ""; } async function getArticleTitle(articleId) { return await request("/api/articles/" + articleId + "/info", "GET") - .catch((e) => errorHandler(e)); + .catch((e) => errorHandler(e)) || ""; } async function getList(page) { let list = await request("/api/articles", "GET", `page=${page || ""}`) .catch((e) => errorHandler(e)); - return [list["articles"], list["prev"]]; + return [list["articles"] || [], list["prev"] || []]; +} + +async function getBoards() { + let boards = await request("/api/boards", "GET") + .catch((e) => errorHandler(e)); + return boards["boards"] || {boards: "Gossiping"}; +} + +async function showBoardsSelectorOptions() { + leaveError(); + loadingIcon.style.display = "flex"; + let boards = await getBoards(); + let boardsSelectBox = document.getElementById("boards"); + boards.forEach(board => { + let option = document.createElement("option"); + option.text = board; + boardsSelectBox.add(option); + }); + loadingIcon.style.display = "none"; } async function showList(page) { @@ -93,16 +113,18 @@ async function showList(page) { let listContainer = document.getElementById("list"); let progressBar = document.getElementsByClassName("progress-bar")[0]; let partOfProgress = 100 / list.length; + let progressBarStatus = 0; progressBar.parentNode.style.display = "flex"; progressBar.style.width = "0"; - (await Promise.all(list.map(async (articleId, index) => { + (await Promise.all(list.map(async (articleId) => { let cardData = await getArticleTitle(articleId).catch(e => ({ title: "無法載入文章", time: e, author: "", disabled: true })); - progressBar.style.width = (index + 2) * partOfProgress + "%"; + progressBarStatus += partOfProgress; + progressBar.style.width = progressBarStatus + "%"; await new Promise(resolve => setTimeout(() => resolve(), 800)); return { title: cardData["title"], From 6ffb4934b0a1e4d5eab102ddb089f2eae4e581ed Mon Sep 17 00:00:00 2001 From: Xanonymous Date: Sun, 29 Mar 2020 01:06:59 +0800 Subject: [PATCH 2/7] add -webkit-align-items -> flex-end, fix flex end problem on chrome and safari. --- website-new/css/main.css | 1 + 1 file changed, 1 insertion(+) diff --git a/website-new/css/main.css b/website-new/css/main.css index e2c2502..155c912 100755 --- a/website-new/css/main.css +++ b/website-new/css/main.css @@ -113,6 +113,7 @@ html, body { justify-content: start; flex-wrap: nowrap; align-items: end; + -webkit-align-items: flex-end; white-space: nowrap; } From dcccc4332b600cb8700b4286dd9af308b3778918 Mon Sep 17 00:00:00 2001 From: Xanonymous Date: Sun, 29 Mar 2020 01:30:48 +0800 Subject: [PATCH 3/7] set article title font size to x-large and ellipsis. --- website-new/css/main.css | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/website-new/css/main.css b/website-new/css/main.css index 155c912..8fde7c2 100755 --- a/website-new/css/main.css +++ b/website-new/css/main.css @@ -192,12 +192,13 @@ html, body { } #article-container .title-text { - font-size: xx-large; + font-size: x-large; font-weight: bold; margin: 0 10px 10px; padding: 10px 0 0; white-space: nowrap; - overflow: scroll; + overflow: hidden; + text-overflow: ellipsis; } From 5b6a77bc1a6f817fa1a4130258ffffe462620ea9 Mon Sep 17 00:00:00 2001 From: Xanonymous Date: Sun, 29 Mar 2020 02:02:01 +0800 Subject: [PATCH 4/7] mount change event to selectBox, thrive new variable link to current board, set default and uniform board's parameter to api functions. --- website-new/js/index.js | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/website-new/js/index.js b/website-new/js/index.js index 94cd1de..c0060ad 100644 --- a/website-new/js/index.js +++ b/website-new/js/index.js @@ -1,4 +1,5 @@ -let prevPage; +let currentBoard = "Gossiping"; +let prevPage, boards; let loadMoreButton = document.getElementById("load-more-button"); let loadingIcon = document.getElementById("loading-icon"); let loadingIconChildren = loadingIcon.children; @@ -22,6 +23,9 @@ let loadingIconChildren = loadingIcon.children; } }; + let boardsSelectBox = document.getElementById("boards"); + boardsSelectBox.addEventListener("change", changeBoard); + let colorBars = document.querySelectorAll(".colorful-line"); const randomColor = () => Math.floor(Math.random() * 16777215).toString(16); colorBars.forEach((coloBar) => { @@ -67,18 +71,18 @@ function leaveError() { loadingIconChildren[1].style.display = "inline"; } -async function getArticleWithContent(articleId) { - return await request("/api/articles/" + articleId, "GET") +async function getArticleWithContent(articleId, board) { + return await request("/api/articles/" + articleId, "GET", `board=${board || currentBoard}`) .catch((e) => errorHandler(e)) || ""; } -async function getArticleTitle(articleId) { - return await request("/api/articles/" + articleId + "/info", "GET") +async function getArticleTitle(articleId, board) { + return await request("/api/articles/" + articleId + "/info", "GET", `board=${board || currentBoard}`) .catch((e) => errorHandler(e)) || ""; } -async function getList(page) { - let list = await request("/api/articles", "GET", `page=${page || ""}`) +async function getList(page, board) { + let list = await request("/api/articles", "GET", `page=${page || ""}&board=${board || currentBoard}`) .catch((e) => errorHandler(e)); return [list["articles"] || [], list["prev"] || []]; } @@ -92,7 +96,7 @@ async function getBoards() { async function showBoardsSelectorOptions() { leaveError(); loadingIcon.style.display = "flex"; - let boards = await getBoards(); + boards = await getBoards(); let boardsSelectBox = document.getElementById("boards"); boards.forEach(board => { let option = document.createElement("option"); @@ -102,14 +106,21 @@ async function showBoardsSelectorOptions() { loadingIcon.style.display = "none"; } -async function showList(page) { +async function changeBoard() { + let listContainer = document.getElementById("list"); + listContainer.innerHTML = ""; + currentBoard = boards[this.selectedIndex]; + await showList("", currentBoard); +} + +async function showList(page, board) { leaveError(); loadingIcon.style.display = "flex"; loadMoreButton.classList.add("disabled"); loadMoreButton.disabled = true; loadMoreButton.textContent = "載入中..."; let list; - [list, prevPage] = await getList(page); + [list, prevPage] = await getList(page, board); let listContainer = document.getElementById("list"); let progressBar = document.getElementsByClassName("progress-bar")[0]; let partOfProgress = 100 / list.length; @@ -117,7 +128,7 @@ async function showList(page) { progressBar.parentNode.style.display = "flex"; progressBar.style.width = "0"; (await Promise.all(list.map(async (articleId) => { - let cardData = await getArticleTitle(articleId).catch(e => ({ + let cardData = await getArticleTitle(articleId, board).catch(e => ({ title: "無法載入文章", time: e, author: "", @@ -170,10 +181,10 @@ async function showList(page) { loadingIcon.style.display = "none"; } -async function showArticle(articleId) { +async function showArticle(articleId, board) { leaveError(); loadingIcon.style.display = "flex"; - let article = await getArticleWithContent(articleId); + let article = await getArticleWithContent(articleId, board); if (article === undefined) { errorHandler("404"); return; From 3e337480f1643d9a0ca575bf10a55db16f43a0be Mon Sep 17 00:00:00 2001 From: Xanonymous Date: Sun, 29 Mar 2020 02:26:16 +0800 Subject: [PATCH 5/7] add current board name as a cookie to client when visit them. --- website-new/js/index.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/website-new/js/index.js b/website-new/js/index.js index c0060ad..0fc24c2 100644 --- a/website-new/js/index.js +++ b/website-new/js/index.js @@ -1,4 +1,4 @@ -let currentBoard = "Gossiping"; +let currentBoard = getCookie("board") || "Gossiping"; let prevPage, boards; let loadMoreButton = document.getElementById("load-more-button"); let loadingIcon = document.getElementById("loading-icon"); @@ -59,6 +59,17 @@ function request(url, method, parameters, ...header) { }); } +function getCookie(name) { + let cookieArr = document.cookie.split(";"); + for (let i = 0; i < cookieArr.length; i++) { + let cookiePair = cookieArr[i].split("="); + if (name === cookiePair[0].trim()) { + return decodeURIComponent(cookiePair[1]); + } + } + return null; +} + function errorHandler(e) { loadingIconChildren[0].style.display = "inline"; loadingIconChildren[0].textContent = e.slice(0, 3) + " ERROR"; @@ -103,6 +114,7 @@ async function showBoardsSelectorOptions() { option.text = board; boardsSelectBox.add(option); }); + boardsSelectBox.selectedIndex = Number(getCookie("board-index")) || 0; loadingIcon.style.display = "none"; } @@ -111,6 +123,8 @@ async function changeBoard() { listContainer.innerHTML = ""; currentBoard = boards[this.selectedIndex]; await showList("", currentBoard); + document.cookie = `board=${currentBoard}`; + document.cookie = `board-index=${this.selectedIndex}`; } async function showList(page, board) { From bb5000854ab14f07a52cbb0e748af16b8f7b5132 Mon Sep 17 00:00:00 2001 From: Xanonymous Date: Sun, 29 Mar 2020 02:35:39 +0800 Subject: [PATCH 6/7] Added protection mechanism for boards sequence changes. --- website-new/js/index.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/website-new/js/index.js b/website-new/js/index.js index 0fc24c2..1d5fdb4 100644 --- a/website-new/js/index.js +++ b/website-new/js/index.js @@ -114,7 +114,12 @@ async function showBoardsSelectorOptions() { option.text = board; boardsSelectBox.add(option); }); - boardsSelectBox.selectedIndex = Number(getCookie("board-index")) || 0; + let inferredBoardIndex = getCookie("board-index") || 0; + if (boardsSelectBox[inferredBoardIndex] !== currentBoard) { + boardsSelectBox.selectedIndex = boards.findIndex(board => board === currentBoard); + } else { + boardsSelectBox.selectedIndex = inferredBoardIndex; + } loadingIcon.style.display = "none"; } From 5ba045538a1183367f91b4a717f4678e22102f07 Mon Sep 17 00:00:00 2001 From: Xanonymous Date: Sun, 29 Mar 2020 02:42:16 +0800 Subject: [PATCH 7/7] fix crawler not fit PEP8 global variable declaration problems. --- crawler.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crawler.py b/crawler.py index 6ddadc3..8690d4b 100644 --- a/crawler.py +++ b/crawler.py @@ -1,8 +1,6 @@ import requests from bs4 import BeautifulSoup -base_url = "https://www.ptt.cc/bbs/" - # 取出頁數或是 id def get_last_session_of_url(url) -> str: @@ -14,7 +12,7 @@ def get_last_session_of_url(url) -> str: def get_article(id_, board="Gossiping", is_include_content=False) -> dict: - global base_url + base_url = "https://www.ptt.cc/bbs/" if board is None: board = "Gossiping" url = base_url + board + "/" + id_ + '.html' @@ -57,7 +55,7 @@ def get_article(id_, board="Gossiping", is_include_content=False) -> dict: def get_article_ids(board="Gossiping", page="") -> dict: - global base_url + base_url = "https://www.ptt.cc/bbs/" if page is None: page = ""