Skip to content

Commit

Permalink
feat: add danjuan api
Browse files Browse the repository at this point in the history
  • Loading branch information
jackluson committed Nov 7, 2022
1 parent 31bafab commit 57de7df
Show file tree
Hide file tree
Showing 5 changed files with 80 additions and 26 deletions.
21 changes: 14 additions & 7 deletions src/acquire_fund_quarter.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

from fund_info.api import FundApier
from fund_info.crawler import FundSpider
from fund_info.csv import FundCSV
from fund_info.fund_csv import FundCSV
from lib.mysnowflake import IdWorker
from models.manager import Manager, ManagerAssoc
from sql_model.fund_insert import FundInsert
Expand All @@ -31,11 +31,14 @@ def get_total_asset(fund_code, platform):
each_fund = FundApier(fund_code, end_date='2021-05-07', platform=platform)
total_asset = each_fund.get_total_asset()
# 如果在爱基金平台找不到,则到展恒基金找
if total_asset == None and platform == 'ai_fund':
print("fund_code", total_asset, fund_code)
if total_asset == None and platform != 'zh_fund':
each_fund = FundApier(
fund_code, end_date='2021-05-10', platform='zh_fund')
total_asset = each_fund.get_total_asset()
if total_asset == None and platform != 'ai_fund':
each_fund = FundApier(
fund_code, end_date='2021-05-10', platform='ai_fund')
total_asset = each_fund.get_total_asset()
return total_asset

def acquire_fund_quarter():
Expand Down Expand Up @@ -182,15 +185,19 @@ def crawlData(start, end):
similar_name = each_fund.fund_name[0:-1]
results = each_fund_query.select_similar_fund(
similar_name) # 获取查询的所有记录
platform = 'zh_fund' if '封闭' in similar_name else 'ai_fund'
# platform = 'zh_fund' if '封闭' in similar_name else 'ai_fund'
platform = 'danjuan'
for i in range(0, len(results)):
item = results[i]
item_code = item[0]
if item_code == each_fund.fund_code:
continue
print("item_code", item_code, platform )
print("item_code", item_code, platform)
total_asset = get_total_asset(item_code, platform)
init_total_asset = init_total_asset - total_asset
if total_asset != None:
init_total_asset = init_total_asset - total_asset
else:
print("total_asset is None", item_code, item[2])
manager_assoc_data = {
'quarter_index': quarter_index,
'manager_start_date': manager_item['manager_start_date'],
Expand Down Expand Up @@ -225,7 +232,7 @@ def crawlData(start, end):
chrome_driver.close()
raise BaseException
chrome_driver.close()
thread_count = 4
thread_count = 6

# for count in range(6):
total_start_time = time()
Expand Down
71 changes: 57 additions & 14 deletions src/fund_info/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,16 @@
sys.path.append('../')
sys.path.append(os.getcwd() + '/src')
from utils.file_op import write_fund_json_data
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry


session = requests.Session()
retry = Retry(connect=3, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)

class FundApier:
def __init__(self, code, *, end_date=None, platform='ai_fund'):
self.fund_code = code
Expand All @@ -35,14 +43,23 @@ def __init__(self, code, *, end_date=None, platform='ai_fund'):
fund_code=code,
end_date=self.end_date
)

def get_client_headers(self, *, referer="https://danjuanfunds.com"):
headers = {
'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36',
'Origin': referer,
'Referer': referer,
}
return headers
def get_total_asset(self):
if self.base_info_is_exist():
return self.get_asset_from_json()
if self.platform == 'ai_fund':
return self.get_base_info_ai()
elif self.platform == 'zh_fund':
return self.get_base_info_zh()
elif self.platform == 'danjuan':
return self.get_base_info_from_danjuan()

def get_asset_from_json(self):
with open(self.file_path) as json_file:
Expand All @@ -59,9 +76,9 @@ def get_asset_from_json(self):
def get_base_info_ai(self):
url = "http://fund.10jqka.com.cn/data/client/myfund/{0}".format(
self.fund_code)

res = requests.get(url) # 自动编码
time.sleep(1)
headers = self.get_client_headers(referer="https://fund.10jqka.com.cn")
res = session.get(url, headers=headers) # 自动编码
time.sleep(2)
try:
if res.status_code == 200:
res_json = res.json()
Expand Down Expand Up @@ -91,20 +108,17 @@ def get_base_info_ai(self):
def get_base_info_zh(self):
url = "https://www.myfund.com/webinterface/Bamboo.ashx?command={0}".format(
'fundInfoHead_NEW')
headers = {
'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8'
}
headers = self.get_client_headers(referer="https://www.myfund.com")
payload = {
'fundcode': self.fund_code,
}
res = requests.post(url, headers=headers, data=payload)
res = session.post(url, headers=headers, data=payload)
res.encoding = "utf-8"
time.sleep(1)
try:
if res.status_code == 200:
res_json = res.json()
fund_scope = res_json.get('FundScope')
pprint(res_json)
if res_json.get('Msg') == 'OK' and fund_scope != None:
end_date = res_json.get('DealDate')
total_asset = fund_scope[0:-1]
Expand All @@ -128,12 +142,11 @@ def get_base_info_zh(self):
def get_analyse_info_zh(self):
url = "https://www.myfund.com/webinterface/Bamboo.ashx?command={0}".format(
'singlefundAnalyse')
headers = {
'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8'
}
headers = self.get_client_headers(referer="https://www.myfund.com")
payload = {
'fundcode': self.fund_code,
}
# res = requests.post(url, headers=headers, data=payload, verify=False)
res = requests.post(url, headers=headers, data=payload)
# print("res", res)
res.encoding = "utf-8"
Expand All @@ -157,6 +170,36 @@ def get_analyse_info_zh(self):
print('code:3', self.fund_code)
raise('中断')

def get_base_info_from_danjuan(self):
url = "https://danjuanfunds.com/djapi/fund/{0}".format(self.fund_code)
headers = self.get_client_headers()
res = session.get(url, headers=headers)
try:
if res.status_code == 200:
res_json = res.json()
if res_json.get('result_code') == 0:
base_info = res.json().get('data')

total_asset = base_info.get('totshare')
if(total_asset.endswith('万')):
total_asset = round(float(total_asset[0:-1]) / 10000, 3)
elif(total_asset.endswith('亿')):
total_asset = float(total_asset[0:-1])
else:
print(total_asset, "not a number")
return
self.total_asset = total_asset
return self.total_asset
else:
pprint(res_json)
print('code:1', self.fund_code)
else:
pprint(res.content)
print('code:2', self.fund_code)
raise('中断')
except:
print('code:3', self.fund_code)
raise('中断')
def write_info_in_json(self, end_date, json_data):
filename = '{fund_code}{end_date}-base.json'.format(
fund_code=self.fund_code,
Expand All @@ -169,6 +212,6 @@ def base_info_is_exist(self):


if __name__ == '__main__':
fund_api = FundApier('000421', end_date='2021-05-31',)
fund_api.get_analyse_info_zh()
fund_api = FundApier('011140', end_date='2021-05-31',)
fund_api.get_base_info_from_danjuan()
# print("fund_api", fund_api)
File renamed without changes.
11 changes: 8 additions & 3 deletions src/sql_model/fund_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,14 @@ def get_fund_from_snapshot_table_no_exist(self, page_start, page_limit):

def get_select_quarter_condition(self):
condition = "WHERE t.fund_cat NOT LIKE '%%货币%%' \
AND t.fund_cat NOT IN ('利率债', '利率债(封闭)', '短债', '短债基金', '短债型', '短债型(封闭)', '短债基金(封闭)',\
'纯债', '纯债基金', '纯债(封闭)', '纯债基金(封闭)',\
'普通债券型', '普通债券型(封闭)', '普通债券', '普通债券(封闭)', '普通债券型基金','普通债券型基金(封闭)', '信用债', '信用债(封闭)','目标日期', '商品 - 贵金属', '商品 - 其它' ) \
AND t.fund_cat NOT LIKE '%%纯债%%' \
AND t.fund_cat NOT LIKE '%%普通债券%%' \
AND t.fund_cat NOT LIKE '%%短债%%' \
AND t.fund_cat NOT LIKE '%%利率债%%' \
AND t.fund_cat NOT LIKE '%%信用债%%' \
AND t.fund_cat NOT LIKE '%%商品%%' \
AND t.fund_cat NOT LIKE '%%环球债券%%' \
AND t.fund_cat NOT IN ('目标日期','亚洲高收益债券') \
AND t.found_date <= %s \
AND t.is_archive = 0 \
AND t.fund_code NOT IN( SELECT fund_code FROM fund_morning_quarter as b \
Expand Down
3 changes: 1 addition & 2 deletions src/sync_fund_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,13 +135,12 @@ def crawlData(start, end):
}
fund_base = FundBase(**base_dict)
fund_base.upsert()
page_start = page_start + page_limit
print('page_start', page_start)
page_start = page_start + page_limit
chrome_driver.close()
bootstrap_thread(crawlData, len(all_funds), 3)
if __name__ == '__main__':
#127, 300, 600-
page_index = 1
# sync_fund_base(page_index)
further_complete_base_info()

0 comments on commit 57de7df

Please sign in to comment.