feat: change get star level methods

main.py

@@ -33,28 +33,28 @@ def main():
         9.“股票持仓基金汇总”\n \
         10.“高分基金”\n \
     输入：")
-    if input_value == '1' or input_value == '快照':
-        page_index = 486
+    if input_value == '1':
+        page_index = 0
         get_fund_list(page_index)  # 执行申万行业信息入库
-    elif input_value == '2' or input_value == '新基入库':
+    elif input_value == '2':
         acquire_fund_base()  # 执行行业股票信息入库
-    elif input_value == '3' or input_value == "季度信息":
+    elif input_value == '3':
         acquire_fund_quarter()
-    elif input_value == '4' or input_value == "基金状态归档":
+    elif input_value == '4':
         fund_supplement = FundSupplement()
         # 补充基金清算维度信息
         fund_supplement.update_archive_status()
-    elif input_value == '5' or input_value == "组合持仓明细":
+    elif input_value == '5':
         get_special_fund_code_holder_stock_detail()
-    elif input_value == '6' or input_value == "基金持仓股排名":
+    elif input_value == '6':
         all_stocks_rank()
-    elif input_value == '7' or input_value == "基金重仓股Top100":
+    elif input_value == '7':
         t100_stocks_rank()
-    elif input_value == '8' or input_value == "股票持仓基金明细":
+    elif input_value == '8':
         all_stock_holder_detail()
-    elif input_value == '9' or input_value == "股票持仓基金汇总":
+    elif input_value == '9':
         calculate_quarter_fund_count()
-    elif input_value == '10' or input_value == "高分基金":
+    elif input_value == '10':
         output_high_score_funds()
     else:
         print('输入有误')

requirements.txt

@@ -10,3 +10,5 @@ Pillow==8.3.1
 python-dotenv==0.19.0
 cryptography==37.0.4
 lxml==4.9.1
+scikit-image==0.19.3
+sewar==0.4.5

src/acquire_fund_snapshot.py

@@ -28,15 +28,12 @@
 connect_instance = connect()
 cursor = connect_instance.cursor()
 
-'''
-判读是否当前页一致，没有的话，切换上一页，下一页操作
-'''
 
 
 def text_to_be_present_in_element(locator, text, next_page_locator):
     """ An expectation for checking if the given text is present in the
     specified element.
-    locator, text
+    locator, text -- 判读是否当前页一致，没有的话，切换上一页，下一页操作
     """
     def _predicate(driver):
         try:
@@ -63,7 +60,6 @@ def get_fund_list(page_index):
     page_count = 25 # 晨星固定分页数
     page_total = math.ceil(int(chrome_driver.find_element_by_xpath(
         '/html/body/form/div[8]/div/div[4]/div[3]/div[2]/span').text) / page_count)
-
     result_dir = './output/'
     output_head = '代码' + ',' + '晨星专属号' + ',' + '名称' + ',' + \
         '类型' + ',' + '三年评级' + ',' + '五年评级' + ',' + '今年回报率' + '\n'
@@ -119,13 +115,19 @@ def get_fund_list(page_index):
                 # 晨星基金专属晨星码
                 morning_star_code_list.append(current_morning_code)
                 name_list.append(tds_text[1].find_all('a')[0].string)
+                # print("name_list", name_list)
                 # 基金分类
                 fund_cat.append(tds_text[2].string)
                 # 三年评级
-                rating = get_star_count(tds_text[3].find_all('img')[0]['src'])
+                # rating = None
+                rating_3_img_ele = tds_text[3].find_all('img')[0]
+                rating_3_src = rating_3_img_ele['src']
+                rating = get_star_count(rating_3_src, current_morning_code, rating_3_img_ele)
                 fund_rating_3.append(rating)
                 # 5年评级
-                rating = get_star_count(tds_text[4].find_all('img')[0]['src'])
+                rating_5_img_ele = tds_text[4].find_all('img')[0]
+                rating_5_src = rating_5_img_ele['src']
+                rating = get_star_count(rating_5_src, current_morning_code, rating_5_img_ele)
                 fund_rating_5.append(rating)
                 # 今年以来回报(%)
                 return_value = tds_nume[3].string if tds_nume[3].string != '-' else None

src/crud/insert.py

@@ -14,7 +14,7 @@
 from sqlalchemy.orm import Session
 from models.manager import Manager, ManagerAssoc
 from models.quarter import Quarter
-from models.var import prefix, ORM_Base, engine
+from models.var import engine
 
 session = Session(engine)
 

src/db/connect.py

@@ -2,7 +2,6 @@
 import pymysql
 from config.env import env_db_host, env_db_name, env_db_user, env_db_password, env_db_stock_name
 
-
 def connect():
     connect = pymysql.connect(
         host=env_db_host, user=env_db_user, password=env_db_password, db=env_db_name, charset='utf8')

src/fund_info/api.py

@@ -79,11 +79,11 @@ def get_base_info_ai(self):
                     pprint(res_json)
                     print('code:1', self.fund_code)
             else:
-                pprint(res.raw)
+                print('url:', url)
                 print('code:2', self.fund_code)
                 raise('中断')
         except:
-            pprint(res.raw)
+            print('url:', url)
             print('code:3', self.fund_code)
             raise('中断')
 
@@ -135,6 +135,7 @@ def get_analyse_info_zh(self):
                 'fundcode': self.fund_code,
             }
             res = requests.post(url, headers=headers, data=payload)
+            # print("res", res)
             res.encoding = "utf-8"
             time.sleep(1)
             try:
@@ -169,5 +170,5 @@ def base_info_is_exist(self):
 
 if __name__ == '__main__':
     fund_api = FundApier('000421', end_date='2021-05-31',)
-    # fund_api.get_analyse_info_zh()
+    fund_api.get_analyse_info_zh()
     # print("fund_api", fund_api)

src/fund_info/crawler.py

@@ -9,8 +9,8 @@
 '''
 
 import re
-from time import sleep
-from bs4 import BeautifulSoup
+from datetime import datetime, timedelta, date
+from time import sleep, time
 from utils.index import get_star_count, get_quarter_index, get_last_quarter_str
 from selenium.common.exceptions import NoSuchElementException
 
@@ -51,6 +51,8 @@ def __init__(self, code, namecode, name, chrome_driver):
         # 十大持仓信息
         self.ten_top_stock_list = []  # 股票十大持仓股信息
 
+    def set_found_data(self, date):
+        self.found_date = date
     # 处理基金详情页跳转
     def go_fund_url(self, cookie_str=None):
         # self.login_morning_star(cookie_str)
@@ -156,10 +158,8 @@ def get_fund_manager_info(self):
                 manager['manager_id'] = manager_id
                 manager['manager_start_date'] = manager_ele.find_element_by_xpath(
                     "li[@class='col1']/i").text[0:10]
-
                 manager['brife'] = manager_ele.find_element_by_xpath(
                     "li[@class='col2']").text
-
                 self.manager_list.append(manager)
 
             except NoSuchElementException:
@@ -173,18 +173,42 @@ def get_fund_manager_info(self):
     def get_fund_morning_rating(self):
         try:
             qt_el = self._chrome_driver.find_element_by_id('qt_star')
-            rating_3_src = qt_el.find_element_by_xpath(
-                "//li[@class='star3']/img").get_attribute('src')
-            rating_5_src = qt_el.find_element_by_xpath(
-                "//li[@class='star5']/img").get_attribute('src')
-            rating_10_src = qt_el.find_element_by_xpath(
-                "//li[@class='star10']/img").get_attribute('src')
-            rating_3 = get_star_count(rating_3_src)
-            rating_5 = get_star_count(rating_5_src)
-            rating_10 = get_star_count(rating_10_src)
-            self.morning_star_rating[3] = rating_3
-            self.morning_star_rating[5] = rating_5
-            self.morning_star_rating[10] = rating_10
+            rating_3_img_ele = qt_el.find_element_by_xpath(
+                "//li[@class='star3']/img")
+            rating_3_src = rating_3_img_ele.get_attribute('src')
+            rating_5_img_ele = qt_el.find_element_by_xpath(
+                "//li[@class='star5']/img")
+            rating_5_src = rating_5_img_ele.get_attribute('src')
+            rating_10_img_ele = qt_el.find_element_by_xpath(
+                "//li[@class='star10']/img")
+            rating_10_src = rating_10_img_ele.get_attribute('src')
+
+            delta = timedelta(days=3 * 365)
+            date_now = date.today()
+            is_more = False
+
+            if date_now - delta > self.found_date:
+                is_more = True
+                rating_3 = get_star_count(rating_3_src,  self.fund_code, rating_3_img_ele)
+                self.morning_star_rating[3] = rating_3
+            if is_more == False:
+                return
+
+            delta = timedelta(days=5 * 365)
+            is_more = False
+            if date_now - delta > self.found_date:
+                is_more = True
+                rating_5 = get_star_count(rating_5_src,  self.fund_code, rating_5_img_ele)
+                self.morning_star_rating[5] = rating_5
+
+            if is_more == False:
+                return
+            delta = timedelta(days=10 * 365)
+            if date_now - delta > self.found_date:
+                rating_10 = get_star_count(rating_10_src,  self.fund_code, rating_10_img_ele)
+                self.morning_star_rating[10] = rating_10
+
+
         except NoSuchElementException:
             self._is_trigger_catch = True
             print('error_fund_info:', self.fund_code,
@@ -225,8 +249,9 @@ def get_fund_qt_rating(self):
 
     def get_fund_season_info(self):
         # 总资产  TODO: 增加一个数据更新时间field
-        self.total_asset = self.get_element_text_by_class_name(
+        total_asset = self.get_element_text_by_class_name(
             "asset", 'qt_base')
+        self.total_asset = float(total_asset) if total_asset else 0
         # 投资风格
         self.investname_style = self.get_element_text_by_class_name(
             'sbdesc', 'qt_base')

src/fund_info/supplement.py

@@ -22,9 +22,12 @@ def __init__(self, code=None):
     def update_archive_status(self):
       fund_query = FundQuery()
       each_fund_update = FundUpdate()
+      start = 0
       funds = fund_query.select_quarter_fund(0, 15000)
       print("funds's len", len(funds))
-      for fund_item in funds:
+      for index in range(start, len(funds)):
+        # print("index", index)
+        fund_item = funds[index]
         fund_code = fund_item[0]
         fund_api = FundApier(fund_code, platform='zh_fund')
         fund_api.get_analyse_info_zh()

src/models/__init__.py

@@ -9,5 +9,6 @@
 '''
 
 print('--models init--');
-
-import fund
+import sys
+sys.path.append('./src')
+import models.fund

src/models/var.py

@@ -19,7 +19,7 @@
 
 prefix = 'fund_morning_'
 
-engine = get_engine(echo=True)
+engine = get_engine(echo=False)
 
 # class ORM_Base(Base):
 #     def __init__(self, **kwargs) -> None:

src/sql_model/fund_query.py

@@ -82,10 +82,8 @@ def get_select_quarter_condition(self):
         '普通债券型', '普通债券型(封闭)', '普通债券', '普通债券(封闭)', '普通债券型基金','普通债券型基金(封闭)', '信用债', '信用债(封闭)','目标日期', '商品 - 贵金属',  '商品 - 其它' ) \
         AND t.found_date <= %s \
         AND t.is_archive = 0 \
-        AND t.fund_name NOT LIKE '%%C' \
-        AND t.fund_name NOT LIKE '%%B' \
         AND t.fund_code	NOT IN( SELECT fund_code FROM fund_morning_quarter as b \
-        WHERE b.quarter_index = %s AND b.stock_position_total != 0)"
+        WHERE b.quarter_index = %s)"
         return condition
 
     # 筛选出要更新的基金季度性信息的基金(B,C类基金除外，因为B、C基金大部分信息与A类一致)的总数
@@ -100,8 +98,9 @@ def select_quarter_fund_total(self):
     @lock_process
     def select_quarter_fund(self, page_start, page_limit):
         sql = "SELECT t.fund_code,\
-            t.morning_star_code, t.fund_name, t.fund_cat \
+            t.morning_star_code, t.fund_name, t.found_date, t.fund_cat \
             FROM fund_morning_base as t " + self.get_select_quarter_condition() + " LIMIT %s, %s;"
+
         self.cursor.execute(
             sql, [self.quarter_date, self.quarter_index, page_start, page_limit])    # 执行sql语句
         return self.cursor.fetchall()    # 获取查询的所有记录
@@ -225,8 +224,9 @@ def select_similar_fund(self, similar_name):
                 t.morning_star_code, t.fund_name \
                 FROM fund_morning_base as t \
                 LEFT JOIN fund_morning_snapshot as f ON f.fund_code = t.fund_code \
-                WHERE t.fund_name LIKE %s \
-                AND t.fund_name NOT LIKE '%%A';"
+                WHERE t.fund_name LIKE %s;"
+
+                # AND t.fund_name NOT LIKE '%%A';"
         self.cursor.execute(sql_similar, [similar_name + '%'])
         results = self.cursor.fetchall()    # 获取查询的所有记录
         return results