-
Notifications
You must be signed in to change notification settings - Fork 0
/
Database.rb
49 lines (42 loc) · 944 Bytes
/
Database.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
class Database
@@links = []
@@scrape_link
@@interrupt = false
def initialize(link)
@@scrape_link = link
@@counter = 0
end
def update(thread_num)
counter = 0
threads = []
puts "Updating DB"
while !@@interrupt
thread_num.times do
link = "#{@@scrape_link}#{counter}"
threads << Thread.new{fetch(link)}
counter += 1
#puts "Fetching #{counter} page"
end
threads.each { |thr| thr.join }
end
puts "Updating finished!"
end
def link_by_name(name)
@@links.each do |item|
if item.to_s.include?(name.downcase)
return item
end
end
nil
end
private def fetch(link)
begin
doc = Nokogiri::HTML(open(link.to_s))
ndoc = doc.css('a.layer-link')
found = ndoc.map {|element| element["href"]}.compact
@@links = @@links+found
rescue OpenURI::HTTPError
@@interrupt = true
end
end
end