diff --git a/README.md b/README.md index 6f5c827..4823660 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ deadfinder sitemap https://www.hahwul.com/sitemap.xml ```yml steps: - name: Run DeadFinder - uses: hahwul/deadfinder@1.4.4 + uses: hahwul/deadfinder@1.5.0 # or uses: hahwul/deadfinder@latest id: broken-link with: @@ -44,7 +44,10 @@ steps: # concurrency: 50 # silent: false # headers: "X-API-Key: 123444" + # worker_headers: "User-Agent: Deadfinder Bot" # include30x: false + # user_agent: "Apple" + # proxy: "http://localhost:8070" - name: Output Handling run: echo '${{ steps.broken-link.outputs.output }}' @@ -79,7 +82,11 @@ Options: t, [--timeout=N] # Timeout in seconds # Default: 10 o, [--output=OUTPUT] # File to write JSON result - H, [--headers=one two three] # Custom HTTP headers to send with request + H, [--headers=one two three] # Custom HTTP headers to send with initial request + [--worker-headers=one two three] # Custom HTTP headers to send with worker requests + [--user-agent=USER_AGENT] # User-Agent string to use for requests + # Default: Mozilla/5.0 (compatible; DeadFinder/1.5.0;) + p, [--proxy=PROXY] # Proxy server to use for requests s, [--silent], [--no-silent] # Silent mode v, [--verbose], [--no-verbose] # Verbose mode ``` diff --git a/action.yml b/action.yml index 32f0d3b..0818cdd 100644 --- a/action.yml +++ b/action.yml @@ -48,5 +48,8 @@ runs: - ${{ inputs.concurrency }} - ${{ inputs.silent }} - ${{ inputs.headers }} + - ${{ inputs.worker_headers}} - ${{ inputs.verbose }} - ${{ inputs.include30x }} + - ${{ inputs.user_agent }} + - ${{ inputs.proxy }} diff --git a/github-action/Dockerfile b/github-action/Dockerfile index bca2708..0e6cc2f 100644 --- a/github-action/Dockerfile +++ b/github-action/Dockerfile @@ -1,5 +1,5 @@ # Use the deadfinder base image from GitHub Container Registry -FROM ghcr.io/hahwul/deadfinder:1.4.4 +FROM ghcr.io/hahwul/deadfinder:1.5.0 # Install jq for JSON processing RUN apt-get update && apt-get install -y jq diff --git a/github-action/entrypoint.sh b/github-action/entrypoint.sh index 0fee927..6e20be1 100755 --- a/github-action/entrypoint.sh +++ b/github-action/entrypoint.sh @@ -6,8 +6,11 @@ # $4 : concurrency # $5 : silent # $6 : headers -# $7 : verbose -# $8 : include30x +# %7 : worker headers +# $8 : verbose +# $9 : include30x +# $10 : user-agent +# $11 : proxy # ------------- export df=/usr/local/bundle/gems/deadfinder-*/bin/deadfinder @@ -17,8 +20,10 @@ cmd="$df $1 $2 -o /output.json" [ -n "$3" ] && cmd="$cmd --timeout=$3" [ -n "$4" ] && cmd="$cmd --concurrency=$4" [ "$5" = "true" ] && cmd="$cmd --silent" -[ "$7" = "true" ] && cmd="$cmd --verbose" -[ "$8" = "true" ] && cmd="$cmd --include30x" +[ "$8" = "true" ] && cmd="$cmd --verbose" +[ "$9" = "true" ] && cmd="$cmd --include30x" +[ -n "$10" ] && cmd="$cmd --user-agent=$10" +[ -n "$11" ] && cmd="$cmd --proxy=$11" # Add headers if provided if [ -n "$6" ]; then @@ -30,6 +35,16 @@ if [ -n "$6" ]; then done fi +# Add worker headers if provided +if [ -n "$7" ]; then + IFS=',' headers="$7" + for header in $headers; do + if [ -n "$header" ]; then + cmd="$cmd --worker-headers \"$header\"" + fi + done +fi + # Execute the command eval "$cmd" echo "Command executed: $cmd" diff --git a/lib/deadfinder.rb b/lib/deadfinder.rb index fd8b2e8..c55f04f 100644 --- a/lib/deadfinder.rb +++ b/lib/deadfinder.rb @@ -23,6 +23,7 @@ def default_options 'timeout' => 10, 'output' => '', 'headers' => [], + 'worker_headers' => [], 'silent' => true, 'verbose' => false, 'include30x' => false @@ -81,11 +82,29 @@ def worker(_id, jobs, results, target, options) uri = URI.parse(j) # Create HTTP request with timeout and headers - http = Net::HTTP.new(uri.host, uri.port) + proxy_uri = URI.parse(options['proxy']) if options['proxy'] && !options['proxy'].empty? + http = if proxy_uri + Net::HTTP.new(uri.host, uri.port, proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password) + else + Net::HTTP.new(uri.host, uri.port) + end http.use_ssl = (uri.scheme == 'https') http.read_timeout = options['timeout'].to_i if options['timeout'] + # Set SSL verification mode + http.verify_mode = OpenSSL::SSL::VERIFY_NONE if http.use_ssl? + request = Net::HTTP::Get.new(uri.request_uri) + + # Add User-Agent header + request['User-Agent'] = options['user_agent'] + + # Add worker headers if provided + options['worker_headers']&.each do |header| + key, value = header.split(':', 2) + request[key.strip] = value.strip + end + response = http.request(request) status_code = response.code.to_i Logger.verbose "Status Code: #{status_code} for #{j}" if options['verbose'] @@ -178,7 +197,12 @@ class DeadFinder < Thor class_option :concurrency, aliases: :c, default: 50, type: :numeric, desc: 'Number of concurrency' class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds' class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write JSON result' - class_option :headers, aliases: :H, default: [], type: :array, desc: 'Custom HTTP headers to send with request' + class_option :headers, aliases: :H, default: [], type: :array, + desc: 'Custom HTTP headers to send with initial request' + class_option :worker_headers, default: [], type: :array, desc: 'Custom HTTP headers to send with worker requests' + class_option :user_agent, default: 'Mozilla/5.0 (compatible; DeadFinder/1.5.0;)', type: :string, + desc: 'User-Agent string to use for requests' + class_option :proxy, aliases: :p, default: '', type: :string, desc: 'Proxy server to use for requests' class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode' class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode' diff --git a/lib/deadfinder/version.rb b/lib/deadfinder/version.rb index 28ac00f..5167387 100644 --- a/lib/deadfinder/version.rb +++ b/lib/deadfinder/version.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -VERSION = '1.4.4' +VERSION = '1.5.0'