Merge pull request #38 from hahwul/dev

Release 0.15.0

README.md

@@ -34,7 +34,7 @@ deadfinder sitemap https://www.hahwul.com/sitemap.xml
 ```yml
 steps:
 - name: Run DeadFinder
-  uses: hahwul/deadfinder@1.4.4
+  uses: hahwul/deadfinder@1.5.0
   # or uses: hahwul/deadfinder@latest
   id: broken-link
   with:
@@ -44,7 +44,10 @@ steps:
     # concurrency: 50
     # silent: false
     # headers: "X-API-Key: 123444"
+    # worker_headers: "User-Agent: Deadfinder Bot"
     # include30x: false
+    # user_agent: "Apple"
+    # proxy: "http://localhost:8070"
 
 - name: Output Handling
   run: echo '${{ steps.broken-link.outputs.output }}'
@@ -79,7 +82,11 @@ Options:
   t, [--timeout=N]                      # Timeout in seconds
                                         # Default: 10
   o, [--output=OUTPUT]                  # File to write JSON result
-  H, [--headers=one two three]          # Custom HTTP headers to send with request
+  H, [--headers=one two three]          # Custom HTTP headers to send with initial request
+      [--worker-headers=one two three]  # Custom HTTP headers to send with worker requests
+      [--user-agent=USER_AGENT]         # User-Agent string to use for requests
+                                        # Default: Mozilla/5.0 (compatible; DeadFinder/1.5.0;)
+  p, [--proxy=PROXY]                    # Proxy server to use for requests
   s, [--silent], [--no-silent]          # Silent mode
   v, [--verbose], [--no-verbose]        # Verbose mode
 ```

action.yml

@@ -48,5 +48,8 @@ runs:
     - ${{ inputs.concurrency }}
     - ${{ inputs.silent }}
     - ${{ inputs.headers }}
+    - ${{ inputs.worker_headers}}
     - ${{ inputs.verbose }}
     - ${{ inputs.include30x }}
+    - ${{ inputs.user_agent }}
+    - ${{ inputs.proxy }}

github-action/Dockerfile

@@ -1,5 +1,5 @@
 # Use the deadfinder base image from GitHub Container Registry
-FROM ghcr.io/hahwul/deadfinder:1.4.4
+FROM ghcr.io/hahwul/deadfinder:1.5.0
 
 # Install jq for JSON processing
 RUN apt-get update && apt-get install -y jq

github-action/entrypoint.sh

@@ -6,8 +6,11 @@
 # $4 : concurrency
 # $5 : silent
 # $6 : headers
-# $7 : verbose
-# $8 : include30x
+# %7 : worker headers
+# $8 : verbose
+# $9 : include30x
+# $10 : user-agent
+# $11 : proxy
 # -------------
 
 export df=/usr/local/bundle/gems/deadfinder-*/bin/deadfinder
@@ -17,8 +20,10 @@ cmd="$df $1 $2 -o /output.json"
 [ -n "$3" ] && cmd="$cmd --timeout=$3"
 [ -n "$4" ] && cmd="$cmd --concurrency=$4"
 [ "$5" = "true" ] && cmd="$cmd --silent"
-[ "$7" = "true" ] && cmd="$cmd --verbose"
-[ "$8" = "true" ] && cmd="$cmd --include30x"
+[ "$8" = "true" ] && cmd="$cmd --verbose"
+[ "$9" = "true" ] && cmd="$cmd --include30x"
+[ -n "$10" ] && cmd="$cmd --user-agent=$10"
+[ -n "$11" ] && cmd="$cmd --proxy=$11"
 
 # Add headers if provided
 if [ -n "$6" ]; then
@@ -30,6 +35,16 @@ if [ -n "$6" ]; then
   done
 fi
 
+# Add worker headers if provided
+if [ -n "$7" ]; then
+  IFS=',' headers="$7"
+  for header in $headers; do
+    if [ -n "$header" ]; then
+      cmd="$cmd --worker-headers \"$header\""
+    fi
+  done
+fi
+
 # Execute the command
 eval "$cmd"
 echo "Command executed: $cmd"

lib/deadfinder.rb

@@ -23,6 +23,7 @@ def default_options
       'timeout' => 10,
       'output' => '',
       'headers' => [],
+      'worker_headers' => [],
       'silent' => true,
       'verbose' => false,
       'include30x' => false
@@ -81,11 +82,29 @@ def worker(_id, jobs, results, target, options)
           uri = URI.parse(j)
 
           # Create HTTP request with timeout and headers
-          http = Net::HTTP.new(uri.host, uri.port)
+          proxy_uri = URI.parse(options['proxy']) if options['proxy'] && !options['proxy'].empty?
+          http = if proxy_uri
+                   Net::HTTP.new(uri.host, uri.port, proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password)
+                 else
+                   Net::HTTP.new(uri.host, uri.port)
+                 end
           http.use_ssl = (uri.scheme == 'https')
           http.read_timeout = options['timeout'].to_i if options['timeout']
 
+          # Set SSL verification mode
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE if http.use_ssl?
+
           request = Net::HTTP::Get.new(uri.request_uri)
+
+          # Add User-Agent header
+          request['User-Agent'] = options['user_agent']
+
+          # Add worker headers if provided
+          options['worker_headers']&.each do |header|
+            key, value = header.split(':', 2)
+            request[key.strip] = value.strip
+          end
+
           response = http.request(request)
           status_code = response.code.to_i
           Logger.verbose "Status Code: #{status_code} for #{j}" if options['verbose']
@@ -178,7 +197,12 @@ class DeadFinder < Thor
   class_option :concurrency, aliases: :c, default: 50, type: :numeric, desc: 'Number of concurrency'
   class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
   class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write JSON result'
-  class_option :headers, aliases: :H, default: [], type: :array, desc: 'Custom HTTP headers to send with request'
+  class_option :headers, aliases: :H, default: [], type: :array,
+                         desc: 'Custom HTTP headers to send with initial request'
+  class_option :worker_headers, default: [], type: :array, desc: 'Custom HTTP headers to send with worker requests'
+  class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.5.0;)', type: :string,
+                            desc: 'User-Agent string to use for requests'
+  class_option :proxy, aliases: :p, default: '', type: :string, desc: 'Proxy server to use for requests'
   class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
   class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
 

lib/deadfinder/version.rb

@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-VERSION = '1.4.4'
+VERSION = '1.5.0'