-
Notifications
You must be signed in to change notification settings - Fork 0
/
bilibili_user.py
62 lines (51 loc) · 1.55 KB
/
bilibili_user.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
# _*_coding:utf-8_*_
# user : dudaizhong
# time : 2018/7/19 20:43
# info :
import requests
import json
from sqlconnect import MysqlConnect
import threadpool
from time import ctime, sleep
conn_ = MysqlConnect()
urls = []
base_url = 'https://api.bilibili.com/x/web-interface/card?mid='
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHT'
'ML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'}
for m in range(5216, 5230):
for i in range(m * 100, (m + 10) * 100):
url = base_url + str(i)
urls.append(url)
def getsource(url):
try:
# 网络请求
jscontent = requests.get(url, headers=headers).text
parser(jscontent)
except requests.HTTPError as e:
if hasattr(e, "code"):
print(e.code)
if hasattr(e, "reason"):
print(e.reason)
def parser(jscontent):
global conn_
jsdict = json.loads(jscontent)
jscard = jsdict['data']['card']
mid = jscard['mid']
name = jscard['name']
sex = jscard['sex']
current_level = jscard['level_info']['current_level']
description = jscard['description']
fans = jscard['fans']
friend = jscard['friend']
data = [mid, name, sex, current_level, description, fans, friend]
print(data)
# 插入数据库
conn_.insertdb(data)
if __name__ == "__main__":
pool = threadpool.ThreadPool(1) # 线程池数量
reqs = threadpool.makeRequests(getsource, urls) # 使用线程池
for req in reqs:
pool.putRequest(req)
# sleep(0.1)
pool.wait()