Merge branch 'master' of github.com:stewartmckee/cobweb

Author: stewartmckee

README.textile

@@ -95,7 +95,8 @@ Creates a new crawler object based on a base_url
 
     ** :follow_redirects              - transparently follows redirects and populates the :redirect_through key in the content hash (Default: true)
     ** :redirect_limit                - sets the limit to be used for concurrent redirects (Default: 10) 
-    ** :processing_queue              - specifies the processing queue for content to be sent to (Default: ContentProcessJob) 
+    ** :processing_queue              - specifies the processing queue for content to be sent to (Default: 'CobwebProcessJob' when using resque, 'CrawlProcessWorker' when using sidekiq) 
+    ** :crawl_finished_queue          - specifies the processing queue for statistics to be sent to after finishing crawling (Default: 'CobwebFinishedJob' when using resque, 'CrawlFinishedWorker' when using sidekiq)
     ** :debug                         - enables debug output (Default: false)
     ** :quiet                         - hides default output (Default: false)
     ** :cache                         - sets the ttl for caching pages, set to nil to disable caching (Default: 300)
@@ -110,9 +111,12 @@ Creates a new crawler object based on a base_url
     ** :obey_robots                   - boolean determining if robots.txt should be honoured. (default: false)
     ** :user_agent                    - user agent string to match in robots.txt (not sent as user_agent of requests yet) (default: cobweb)
     ** :crawl_limit_by_page           - sets the crawl counter to only use html page types when counting objects crawled
-    ** :valid_mime_types              - an array of mime types that takes wildcards (eg 'text/*') defaults to ['*/*']
+    ** :valid_mime_types              - an array of mime types that takes wildcards (eg 'text/*') defaults to @['*/*']@
     ** :direct_call_process_job       - boolean that specifies whether objects should be passed directly to a processing method or should be put onto a queue
     ** :raise_exceptions              - defaults to handling exceptions with debug output, setting this to true will raise exceptions in your app
+    ** :use_encoding_safe_process_job - Base64-encode the body when storing job in queue; set to true when you are expecting non-ASCII content (Default: false)
+    ** :proxy_addr                    - hostname of a proxy to use for crawling (e. g., 'myproxy.example.net', default: nil)
+    ** :proxy_port                    - port number of the proxy (default: nil)
 
 
 bc. crawler = Cobweb.new(:follow_redirects => false)

lib/cobweb.rb

@@ -60,6 +60,8 @@ def initialize(options = {})
     default_valid_mime_types_to                ["*/*"]
     default_raise_exceptions_to               false
     default_store_inbound_links_to            false
+    default_proxy_addr_to                     nil
+    default_proxy_port_to                     nil
 
   end
 
@@ -154,7 +156,7 @@ def get(url, options = @options)
       # retrieve data
       #unless @http && @http.address == uri.host && @http.port == uri.inferred_port
         puts "Creating connection to #{uri.host}..." if @options[:debug]
-        @http = Net::HTTP.new(uri.host, uri.inferred_port)
+        @http = Net::HTTP.new(uri.host, uri.inferred_port, @options[:proxy_addr], @options[:proxy_port])
       #end
       if uri.scheme == "https"
         @http.use_ssl = true
@@ -327,7 +329,7 @@ def head(url, options = @options)
       # retrieve data
       unless @http && @http.address == uri.host && @http.port == uri.inferred_port
         puts "Creating connection to #{uri.host}..." unless @options[:quiet]
-        @http = Net::HTTP.new(uri.host, uri.inferred_port)
+        @http = Net::HTTP.new(uri.host, uri.inferred_port, @options[:proxy_addr], @options[:proxy_port])
       end
       if uri.scheme == "https"
         @http.use_ssl = true

spec/cobweb/cobweb_spec.rb

@@ -31,6 +31,8 @@
     options[:timeout].should == 10
     options[:redis_options].should == {}
     options[:internal_urls].should == []
+    options[:proxy_addr].should be_nil
+    options[:proxy_port].should be_nil
 
   end
 
@@ -177,26 +179,34 @@
     end
     describe "location setting" do
       it "Get should strip fragments" do
-        Net::HTTP.should_receive(:new).with("www.google.com", 80)
+        Net::HTTP.should_receive(:new).with("www.google.com", 80, nil, nil)
         Net::HTTP::Get.should_receive(:new).with("/", @default_options)
         @cobweb.get("http://www.google.com/#ignore")
       end
       it "head should strip fragments" do
-        Net::HTTP.should_receive(:new).with("www.google.com", 80)
+        Net::HTTP.should_receive(:new).with("www.google.com", 80, nil, nil)
         Net::HTTP::Head.should_receive(:new).with("/", {}).and_return(@mock_http_request)
         @cobweb.head("http://www.google.com/#ignore")
       end
       it "get should not strip path" do
-        Net::HTTP.should_receive(:new).with("www.google.com", 80)
+        Net::HTTP.should_receive(:new).with("www.google.com", 80, nil, nil)
         Net::HTTP::Get.should_receive(:new).with("/path/to/stuff", @default_options)
         @cobweb.get("http://www.google.com/path/to/stuff#ignore")
       end
       it "get should not strip query string" do
-        Net::HTTP.should_receive(:new).with("www.google.com", 80)
+        Net::HTTP.should_receive(:new).with("www.google.com", 80, nil, nil)
         Net::HTTP::Get.should_receive(:new).with("/path/to/stuff?query_string", @default_options)
         @cobweb.get("http://www.google.com/path/to/stuff?query_string#ignore")
       end
     end
+    describe "with proxy" do
+      it "provides proxy parameters to Net::HTTP" do
+        cobweb = Cobweb.new proxy_addr: 'proxy.example.com', proxy_port: 1234
+        Net::HTTP.should_receive(:new).with("www.google.com", 80, "proxy.example.com", 1234)
+
+        cobweb.get("http://www.google.com/")
+      end
+    end
 
   end  
 end