http://toscrape.com/

• Default http://quotes.toscrape.com/
  • quotes_css.py
  • quotes_xpath.py
• Scroll http://quotes.toscrape.com/scroll
  • quotes_scroll.py
• JavaScript http://quotes.toscrape.com/js
  • quotes_js.py
• Tableful http://quotes.toscrape.com/tableful
  • None
• Login http://quotes.toscrape.com/login
  • login.py
• ViewState http://quotes.toscrape.com/search.aspx
  • None
• Random http://quotes.toscrape.com/random
  • quotes_random
  • yield response.follow(next_page, self.parse, dont_filter=True)

mongodb OK

db.quotestoscrape.aggregate([
        {$group: {_id: {text: '$text', author: '$author'}, count: {$sum: 1}, dups: {$addToSet: '$_id'}}},
        {$match: {count: {$gt: 1}}}
]).forEach(function(doc){
    doc.dups.shift();
    db.quotestoscrape.remove({_id: {$in: doc.dups}});
})

db.quotestoscrape.aggregate([
    {$group: {_id : '$text', count: {$sum : 1}}},
    {$match: {count: {$gt : 1}}}
])