feat: 2023-Q1 cralwer

Author: jackluson

log/crawler.log

@@ -42,13 +42,15 @@ def main():
     输入：")
     if input_value == '1':
         page_index = 1
-        get_fund_list(page_index)  # 执行申万行业信息入库
+        get_fund_list(page_index)  
     elif input_value == '2':
-        acquire_fund_base()  # 执行行业股票信息入库
+        acquire_fund_base()  
     elif input_value == '3':
-        page_index = 1
+        # 将第1步和第2步合并
+        page_index = 506
         sync_fund_base(page_index)
     elif input_value == '4':
+        # 由于执行3, 快照页基金基础数据不全，需要补充
         further_complete_base_info()
     elif input_value == '5':
         update_fund_archive_status()
@@ -61,6 +63,7 @@ def main():
     elif input_value == '9':
         all_stock_holder_detail()
     elif input_value == '10':
+        #先执行9,再执行10
         calculate_quarter_fund_total()
     elif input_value == '11':
         output_high_score_funds()

main.py

@@ -1 +1,4 @@
 代码,晨星专属号,名称,类型,页码,备注
+519610, 0P00017PZ8, 银河旺利灵活配置混合A, 2016-05-13, 2983, 页面跳转有问题
+519611, 0P00017PZ9, 银河旺利灵活配置混合C, 2016-05-13, 2983, 页面跳转有问题
+519612, 0P00017PZA, 银河旺利灵活配置混合I, 2016-05-13, 2983, 页面跳转有问题

output/fund_morning_quarter_abnormal.csv

@@ -1 +1,3 @@
 代码,晨星专属号,名称,类型,股票总仓位,页码,备注
+010340, 0P0001KZO6, 易方达高质量严选三年持有期混合, 2020-11-12, 91.97, 1999, li[16]
+010314, 0P0001L5IO, 摩根士丹利华鑫内需增长混合A, 2020-12-16, 93.24, 1989, li[16]

output/fund_morning_quarter_catch.csv

@@ -0,0 +1,26 @@
+代码,晨星专属号,名称,类型,三年评级,五年评级,今年回报率
+301153601957396480, 004685, 0P0001BW1T, 金元顺安元启灵活配置混合, 灵活配置, 5, 5, 4.64
+301153619657359360, 001606, 0P00016EBI, 农银汇理工业4.0灵活配置混合, 灵活配置, 5, 5, 10.03
+301153637088886784, 005669, 0P0001CPZD, 前海开源公用事业行业股票, 沪港深股票, 5, 5, -0.51
+301153654499442688, 002190, 0P00017LPU, 农银汇理新能源主题灵活配置混合A, 积极配置 - 大盘成长, 5, 4, -5.32
+301153671842889728, 002258, 0P0001BG97, 大成国企改革灵活配置混合, 积极配置 - 大盘成长, 5, 5, 6.56
+301153689605767168, 003834, 0P0001ACCP, 华夏能源革新股票A, 大盘成长股票, 5, 4, 2.70
+301153707200872448, 002943, 0P0001992T, 广发多因子灵活配置混合, 积极配置 - 中小盘, 5, 5, 7.18
+301153724649177088, 590003, F0000004HJ, 中邮核心优势灵活配置混合, 积极配置 - 大盘成长, 5, 4, 5.48
+301153742114258944, 000729, 0P00013OUD, 建信中小盘先锋股票A, 中盘成长股票, 5, 5, -0.08
+301153759445123072, 001476, 0P0001660F, 中银智能制造股票A, 大盘成长股票, 5, 5, -2.01
+301153776780181504, 165516, 0P0000VO6G, 信诚周期轮动混合（LOF）A, 积极配置 - 大盘成长, 5, 5, 16.76
+301153794090074112, 006234, 0P0001IG9U, 万家汽车新趋势混合C, 灵活配置, 5, 0, 9.83
+301153811383189504, 000991, 0P00015AHP, 工银战略转型股票A, 中盘平衡股票, 5, 5, 2.06
+301153828764385280, 090018, 0P0000VE3A, 大成新锐产业混合, 积极配置 - 中小盘, 5, 5, 1.99
+301153846065889280, 001300, 0P00015YLW, 大成睿景灵活配置混合A, 积极配置 - 中小盘, 5, 5, 1.76
+301153863405142016, 001301, 0P00015YNO, 大成睿景灵活配置混合C, 积极配置 - 中小盘, 5, 5, 1.52
+301153880815697920, 519702, 0P0000RROL, 交银趋势优先混合A, 积极配置 - 大盘成长, 5, 5, 1.85
+301153898192699392, 166301, 0P0001610K, 华商新趋势优选灵活配置混合, 灵活配置, 5, 5, 12.85
+301153915490009088, 000336, 0P0000ZQ30, 农银研究精选混合, 积极配置 - 大盘成长, 5, 5, -2.67
+301153932950896640, 000689, 0P00013NSK, 前海开源新经济灵活配置混合A, 灵活配置, 5, 5, -4.60
+301153950214651904, 540008, 0P0000NX49, 汇丰晋信低碳先锋股票A, 大盘成长股票, 5, 3, -4.69
+301153967482601472, 004475, 0P0001BF42, 华泰柏瑞富利灵活配置混合A, 积极配置 - 大盘平衡, 5, 5, 13.60
+301153984838631424, 006233, 0P0001IG9T, 万家汽车新趋势混合A, 灵活配置, 5, 0, 10.00
+301154002257575936, 001532, 0P0001ABM1, 华安文体健康主题灵活配置混合A, 积极配置 - 中小盘, 5, 5, -3.76
+301154019647160320, 005094, 0P0001C87G, 万家臻选混合, 积极配置 - 大盘成长, 5, 5, 12.52

output/fund_morning_snapshot.csv

@@ -233,8 +233,9 @@ def crawlData(start, end):
                 print(current_thread().getName(), 'page_start', page_start)
                 sleep(3)
         except(BaseException):
+            print('start', start)
             chrome_driver.close()
-            raise BaseException
+            # raise BaseException
         chrome_driver.close()
     thread_count = 6
     total_start_time = time()

output/fund_morning_snapshot_orm.csv

@@ -26,7 +26,8 @@ def update_archive_status(self):
       funds = fund_query.select_quarter_fund(0, 15000)
       print("funds's len", len(funds))
       for index in range(start, len(funds)):
-        # print("index", index)
+        if index % 50 == 0:
+          print("index", index)
         fund_item = funds[index]
         fund_code = fund_item[0]
         fund_api = FundApier(fund_code, platform='zh_fund')

screenshot/top100_compare.jpg

@@ -238,6 +238,7 @@ def all_stocks_rank(each_statistic=None):
     df_other_list.to_excel(writer, sheet_name=other_market)
 
     writer.save()
+    print('输出到:', output_file, "完成\n" )
 
 
 def all_stock_holder_detail(each_statistic=None, *, quarter_index=None, threshold=0):
@@ -273,7 +274,7 @@ def all_stock_holder_detail(each_statistic=None, *, quarter_index=None, threshol
             elif bool(re.search("^(8|4)(3|7)\d{4}$", stock_code)):
                 path = 'A股/北交所'
             else:
-                print('stock_name_code', stock_name_code)
+                print('该%s没有匹配股票市场:', stock_name_code)
         hold_fund_list = sorted(
             stock[1]['fund_list'], key=lambda x: x['持有市值(亿元)'], reverse=True)
         df_list = pd.DataFrame(hold_fund_list)