-
Notifications
You must be signed in to change notification settings - Fork 0
/
s5_scrape_wikipedia.py
53 lines (37 loc) · 1.36 KB
/
s5_scrape_wikipedia.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
"""Scrape Wikipedia for game genres."""
import os
import pandas as pd
from bs4 import BeautifulSoup
import requests
import time
def get_genre(url):
soup = BeautifulSoup(requests.get(url).content, "lxml")
table = soup.find('table', attrs={'class': 'infobox'})
rows = table.find_all('tr')
att_dict = dict()
for row in rows:
th = row.find('th')
if th and th.text == 'Genre(s)':
att_dict[th.text] = [ele.text.lower() for ele in row.find('td').find_all('a')]
else:
continue
return att_dict
i_filepath = os.path.join('data', 's4_rate_results.csv')
game_df = pd.read_csv(i_filepath)
game_df['has_wikipedia'] = game_df['url'].astype(str).str.startswith('https://en.wikipedia.org/wiki/')
game_df = game_df.loc[game_df['has_wikipedia'], :].reset_index(drop=True)
game_subdf = game_df.drop_duplicates(subset=['game'], keep='first', ignore_index=True)
lookup_lst = []
for _, row in game_subdf.iterrows():
if row['has_wikipedia']:
url = row['url']
time.sleep(1)
att_dict = get_genre(url)
att_dict['game'] = row['game']
lookup_lst.append(att_dict)
print(att_dict)
else:
continue
lookup_df = pd.DataFrame(lookup_lst)
game_df = game_df.merge(lookup_df, how='left', on='game')
game_df.to_csv(os.path.join('data', 's5_scrape_wikipedia.csv'), index=False)