spider.yaml

# coding:utf-8
time_out: 200                  # timeout for crawling and storing user info
min_crawl_interal: 1           # min interal of http request
max_crawl_interal: 5           # max interal of http request
excp_interal: 5*60             # time for sleeping when crawling raises exceptions

# TODO set a default value for max_value of crawling
max_search_page: 30            # max search page for crawling
max_home_page: 50              # max user home page for crawling
max_comment_page: 2000         # max comment page for crawling
max_repost_page: 2000          # max repost page for crawling
max_retries: 5                 # retry times for crawling

# you should set the args below if you login from uncommon place
# It's for verification code indentified
yundama_username: yourname           # account for yundama
yundama_passwd: yourpasswd             # password for yundama

# the value of mode can be normal or quick.
# In noraml mode, it will be more stable, while in quick mode, the crawling speed will
# be much faster, and the weibo account almostly will be banned
mode: quick
# the max number of each cookie can be shared
# if you choose quick mode, your cookie will be used util it's banned
share_host_count: 5
# the expire time(hours) of each weibo cookies
cookie_expire_time: 23



db:
    host: xxx
    port: 3306
    user: weibospider
    password: xxx
    db_name: weibo
    db_type: mysql

redis:
    host: xxx
    port: 6379
    password: weibospider
    cookies: 1                   # store and fetch cookies
    urls: 2                      # store fetched urls and results
    broker: 5                    # broker for celery
    backend: 6                   # backed for celery
    id_name: 8                   # user id and names，for repost info analysis

# warning by email
email:
    # your email must open smtp & pop3 service
    server: smtp.sina.com
    port: 587
    from: xxx@sina.com           #sendingemailaccount
    password: xxx                #youremailpasswd
    to: xxxx@139.com             #bind 139 email,so your phone will receive the warning message
    subject: Warning Of Weibo Spider
    warning_info: Please find out the reason why the spider stops working