@@ -10,6 +10,7 @@ import Vapor
1010import Fluent
1111import FluentPostgreSQL
1212import SwiftSoup
13+ import PerfectICONV
1314
1415private let header : HTTPHeaders = [ " User-Agent " : " Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36 "
1516 , " Cookie " : " yunsuo_session_verify=2a87ab507187674302f32bbc33248656 " ]
@@ -30,6 +31,9 @@ class BookController: RouteCollection {
3031 group. get ( " story " , use: getBookLastChapterContentHandler)
3132 group. get ( " start " , use: crawlerFanRenBookHandler)
3233
34+ //test
35+ group. get ( " html " , use: getHtmlDataHandler)
36+
3337 }
3438}
3539
@@ -38,26 +42,70 @@ extension BookController {
3842 func getBookLastChapterContentHandler( _ req: Request ) throws -> Future < Response > {
3943 let name = req. query [ String . self, at: " name " ] ?? " "
4044
41- return BookChapter . query ( on: req) . filter ( \. bookName, . like, " % \( name) % " ) . all ( ) . flatMap ( { ( books) in
42- if books. count > 0 {
43- return try ResponseJSON < BookChapter > ( data: books. last) . encode ( for: req)
44- } else {
45- return try ResponseJSON < BookChapter > ( status: . error, message: " 没有此书: \( name) " ) . encode ( for: req)
46- }
47- } )
45+ return BookInfo . query ( on: req)
46+ . filter ( \. bookName, . like, " % \( name) % " )
47+ . first ( )
48+ . flatMap ( { ( info) in
49+ guard let info = info else {
50+ return try ResponseJSON < Empty > ( status: . error, message: " 没有此书: \( name) " ) . encode ( for: req)
51+ }
52+ return BookChapter
53+ . query ( on: req)
54+ . filter ( \. bookId == info. bookId)
55+ . sort ( \. chapterId, . descending)
56+ . first ( )
57+ . flatMap ( { ( chapter) in
58+ struct Chapter : Content {
59+ var bookName : String ?
60+ var time : String ?
61+ var chaptName : String ?
62+ var content : String ?
63+ }
64+ let pter = Chapter ( bookName: info. bookName, time: chapter? . updateTime, chaptName: " 最新章节： " + ( chapter? . chapterName ?? " " ) , content: chapter? . content)
65+ return try . view ( ) . render ( " leaf/book " , pter) . encode ( for: req)
66+ } )
67+ } )
68+
4869 }
4970
50- func crawlerFanRenBookHandler( _ req: Request ) throws -> Future < Response > {
71+ func getHtmlDataHandler( _ req: Request ) throws -> Future < String > {
72+
73+ let client = try . make ( Client . self)
74+ let iconv = try Iconv ( from: Iconv . CodePage. GBK, to: Iconv . CodePage. UTF8)
75+ let url = " https://www.piaotian.com/html/9/9102/ "
76+ return client. get ( url)
77+ . flatMap { $0. http. body. consumeData ( on: req) } // 1) read complete body as raw Data
78+ . map { ( data: Data ) -> String in
79+ // 2) convert Data to [UInt8] for Iconv
80+ var bytes = [ UInt8] ( repeating: 0 , count: data. count)
81+ let buffer = UnsafeMutableBufferPointer ( start: & bytes, count: bytes. count)
82+ _ = data. copyBytes ( to: buffer)
83+
84+ // 3) convert GBK -> UTF8
85+ return iconv. utf8 ( buf: bytes) ?? " "
86+ }
87+
88+ }
89+
90+ func crawlerFanRenBookHandler( _ req: Request ) throws -> Future < ResponseJSON < Empty > > {
5191
5292 typeId = 9
5393 bookId = 9102
5494 let url = " https://www.piaotian.com/html/ \( typeId) / \( bookId) / "
5595
56- let client = try . make ( FoundationClient . self)
57- return client. get ( url, headers: header)
58- . flatMap ( to: Response . self, { clientResponse in
96+
97+ return try . client ( ) . get ( url)
98+ . flatMap { $0. http. body. consumeData ( on: req) }
99+ . map { ( data) -> String in
100+ let iconv = try Iconv ( from: Iconv . CodePage. GBK, to: Iconv . CodePage. UTF8)
101+ var bytes = [ UInt8] ( repeating: 0 , count: data. count)
102+ let buffer = UnsafeMutableBufferPointer ( start: & bytes, count: bytes. count)
103+ _ = data. copyBytes ( to: buffer)
104+
105+ return iconv. utf8 ( buf: bytes) ?? " g/u "
106+ } . map ( { html -> ResponseJSON < Empty > in
59107
60- let html = clientResponse . http . body . gbkString
108+ print ( html, " \n \n \n " )
61109 let document = try SwiftSoup . parse ( html)
62110 let mainBody = try . select ( " div[class='mainbody'] " )
63111
@@ -77,21 +125,14 @@ extension BookController {
77125 self . elements = revertLis
78126 self . currentIndex = 0
79127
80- _ = BookInfo . query ( on: req) . filter ( \. bookId == self . bookId) . first ( ) . map ( { ( exist) in
81-
82- if var exist = exist {
83- exist. chapterCount = revertLis. count
84- exist. updateTime = TimeManager . currentTime ( )
85- _ = exist. update ( on: req)
86- debugPrint ( " 本书已存在: \( exist. bookName ?? " " ) \( TimeManager . currentTime ( ) ) " )
87- } else {
88- let bookInfo = BookInfo ( id: nil , typeId: self . typeId, bookId: self . bookId, bookName: bookName, chapterCount: revertLis. count, updateTime: TimeManager . currentTime ( ) , content: nil , auther: auther, bookImg: nil )
89- _ = bookInfo. save ( on: req) . map ( { ( info) in
90- debugPrint ( " 已保存本书: \( info) " )
91- } )
92- }
93-
94- } )
128+ try self . bookExistHandler ( req,
129+ revertLis: revertLis,
130+ bookName: bookName,
131+ auther: auther)
132+
133+ if self . elements == nil || self . elements? . count == 0 {
134+ return ResponseJSON < Empty > ( status: . error, message: " 没有数据 " )
135+ }
95136
96137 func runRepeatTimer( ) throws {
97138
@@ -108,11 +149,30 @@ extension BookController {
108149 }
109150 try runRepeatTimer ( )
110151
111- return try ResponseJSON < Empty > ( status: . ok, message: " 开始爬取 \( self . typeId) / \( self . bookId) " ) . encode ( for : req )
152+ return ResponseJSON < Empty > ( status: . ok, message: " 开始爬取 \( self . typeId) / \( self . bookId) " )
112153 } )
154+
113155 }
114156
115157
158+ func bookExistHandler( _ req: Request , revertLis: [ Element ] , bookName: String , auther: String ) throws {
159+
160+ _ = BookInfo . query ( on: req) . filter ( \. bookId == self . bookId) . first ( ) . map ( { ( exist) in
161+
162+ if var exist = exist {
163+ exist. chapterCount = revertLis. count
164+ exist. updateTime = TimeManager . currentTime ( )
165+ _ = exist. update ( on: req)
166+ debugPrint ( " 本书已存在: \( exist. bookName ?? " " ) \( TimeManager . currentTime ( ) ) " )
167+ } else {
168+ let bookInfo = BookInfo ( id: nil , typeId: self . typeId, bookId: self . bookId, bookName: bookName, chapterCount: revertLis. count, updateTime: TimeManager . currentTime ( ) , content: nil , auther: auther, bookImg: nil )
169+ _ = bookInfo. save ( on: req) . map ( { ( info) in
170+ debugPrint ( " 已保存本书: \( info) " )
171+ } )
172+ }
173+ } )
174+ }
175+
116176 // 保存每一章内容。
117177 func saveBookContentHandler( req: Request , bookName: String , auther: String , bookId: Int , typeId: Int ) throws {
118178
0 commit comments