Update spider.py

Author: zzxxchen

spider.py

@@ -5,14 +5,16 @@
 from os.path import exists
 
 stock = '博思软件'
-RESULTS_DIR = f'D:\\年报\\{stock}'
+announcement_list = ['分红派息实施公告', '利润分配预案', '年度报告', '半年度|季度', '招股说明书']
+announcement = '季度报告'
+ban = '摘要|已取消|提示性公告'
+RESULTS_DIR = f'D:\\报告\\{stock}\\{announcement}'
 exists(RESULTS_DIR) or makedirs(RESULTS_DIR)
 
 headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0'}
 orgid_url = 'http://www.cninfo.com.cn/new/data/szse_stock.json'
 url = 'http://www.cninfo.com.cn/new/hisAnnouncement/query'
 DETAIL_URL = 'http://static.cninfo.com.cn/'
-
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s: %(message)s')
 
 
@@ -24,25 +26,23 @@ def get_orgid():
         stock_lists = orgids['stockList']
         for stock_list in stock_lists:
             if stock_list['zwjc'] == stock:
-                logging.info(f'获得股票信息: {stock}')
                 return {
                     'code': stock_list['code'],
                     'orgid': stock_list['orgId']
                 }
 
 
 def get_pdf_url(page, data):
-    """获得年报及招股说明书pdf下载信息"""
+    """获得公告的pdf下载信息"""
     code = data.get('code')
     orgid = data.get('orgid')
-
     post_data = {
         'stock': f'{code},{orgid}',
         'tabName': 'fulltext',
         'pageSize': 30,
         'pageNum': page,
         'column': 'szse',
-        'category': 'category_ndbg_szsh;category_sf_szsh;',
+        'category': '',
         'plate': 'sz',
         'seDate': '',
         'searchkey': '',
@@ -58,18 +58,44 @@ def get_pdf_url(page, data):
         dats = an.get('announcements')
         stock_list = []
         for dat in dats:
-            if re.search('摘要|已取消', dat['announcementTitle']):
+            if re.search(ban, dat['announcementTitle']):
                 continue
-            elif re.search('招股说明书|年度报告', dat['announcementTitle']):
+            elif re.search(announcement, dat['announcementTitle']):
                 stock_list.append({
                     'announcementTitle': dat['announcementTitle'],
                     'adjunctUrl': dat['adjunctUrl']
                 })
         return stock_list
 
 
+def get_totalpages(data):
+    """获得公告的总页数"""
+    code = data.get('code')
+    orgid = data.get('orgid')
+    post_data = {
+        'stock': f'{code},{orgid}',
+        'tabName': 'fulltext',
+        'pageSize': 30,
+        'pageNum': 1,
+        'column': 'szse',
+        'category': '',
+        'plate': 'sz',
+        'seDate': '',
+        'searchkey': '',
+        'secid': '',
+        'sortName': '',
+        'sortType': '',
+        'isHLtitle': 'true'
+    }
+    with httpx.Client(headers=headers) as client:
+        res = client.post(url, data=post_data)
+        an = res.json()
+        totalpages = an.get('totalpages')
+        return totalpages
+
+
 def save_pdf(datas):
-    """保存年报pdf"""
+    """保存公告pdf"""
     for data in datas:
         part_url = data.get('adjunctUrl')
         name = data.get('announcementTitle')
@@ -84,9 +110,13 @@ def save_pdf(datas):
 
 
 def main():
-    for page in range(1, 2):
+    pages = get_totalpages(get_orgid())
+    logging.info(f'一共{pages}页公告信息...')
+    for page in range(1, pages + 1):
         pdfdata = get_pdf_url(page, get_orgid())
+        logging.info(f'获得第{page}页股票信息...')
         save_pdf(pdfdata)
+    logging.info('下载完成')
 
 
 if __name__ == '__main__':