Merge pull request #27 from hahwul/dev

Release v1.4.0

.rubocop.yml

@@ -43,5 +43,6 @@ Security/Open:
     - 'lib/deadfinder.rb'
 
 Metrics/BlockLength:
+  Max: 50
   Exclude:
     - 'spec/utils_spec.rb'

Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:3.3.0
+FROM ruby:3.3.5
 
 RUN mkdir /app
 WORKDIR /app

Gemfile.lock

@@ -33,7 +33,7 @@ GEM
     rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.1)
+    rspec-mocks (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
@@ -52,6 +52,7 @@ GEM
 PLATFORMS
   arm64-darwin-22
   arm64-darwin-23
+  arm64-darwin-24
   x86_64-darwin-21
   x86_64-linux
 
@@ -67,4 +68,4 @@ DEPENDENCIES
   thor
 
 BUNDLED WITH
-   2.3.22
+   2.5.16

README.md

@@ -34,11 +34,16 @@ deadfinder sitemap https://www.hahwul.com/sitemap.xml
 ```yml
 steps:
 - name: Run DeadFinder
-  uses: hahwul/deadfinder@1.3.6
+  uses: hahwul/deadfinder@1.4.0
   id: broken-link
   with:
-    command: sitemap
+    command: sitemap # url / file / sitemap
     target: https://www.hahwul.com/sitemap.xml
+    # timeout: 10
+    # concurrency: 50
+    # silent: false
+    # headers: "X-API-Key: 123444"
+    # include30x: false
 
 - name: Output Handling
   run: echo '${{ steps.broken-link.outputs.output }}'
@@ -67,14 +72,15 @@ Commands:
   deadfinder version                # Show version.
 
 Options:
-  -c, [--concurrency=N]                           # Number of concurrency
-                                                  # Default: 50
-  -t, [--timeout=N]                               # Timeout in seconds
-                                                  # Default: 10
-  -o, [--output=OUTPUT]                           # File to write JSON result
-  -H, [--headers=one two three]                   # Custom HTTP headers to send with request
-  -s, [--silent], [--no-silent], [--skip-silent]  # Silent mode
-                                                  # Default: false
+  r, [--include30x], [--no-include30x]  # Include 30x redirections
+  c, [--concurrency=N]                  # Number of concurrency
+                                        # Default: 50
+  t, [--timeout=N]                      # Timeout in seconds
+                                        # Default: 10
+  o, [--output=OUTPUT]                  # File to write JSON result
+  H, [--headers=one two three]          # Custom HTTP headers to send with request
+  s, [--silent], [--no-silent]          # Silent mode
+  v, [--verbose], [--no-verbose]        # Verbose mode
 ```
 
 ## Modes

action.yml

@@ -4,12 +4,74 @@ branding:
   icon: 'link'
   color: 'purple'
 inputs:
-  command: 
-    description: 'command for scan (url/sitemap/file)'
+  command:
+    description: 'The command to run (file, url, sitemap)'
     required: true
   target:
-    description: 'target for command'
+    description: 'The target for the command'
     required: true
+  timeout:
+    description: 'Timeout in seconds'
+    required: false
+    default: ''
+  concurrency:
+    description: 'Number of concurrency'
+    required: false
+    default: ''
+  silent:
+    description: 'Silent mode'
+    required: false
+    default: 'false'
+inputs:
+  command:
+    description: 'The command to run (file, url, sitemap)'
+    required: true
+  target:
+    description: 'The target for the command'
+    required: true
+  timeout:
+    description: 'Timeout in seconds'
+    required: false
+    default: ''
+  concurrency:
+    description: 'Number of concurrency'
+    required: false
+    default: ''
+  silent:
+    description: 'Silent mode'
+    required: false
+    default: 'false'
+inputs:
+  command:
+    description: 'The command to run (file, url, sitemap)'
+    required: true
+  target:
+    description: 'The target for the command'
+    required: true
+  timeout:
+    description: 'Timeout in seconds'
+    required: false
+    default: ''
+  concurrency:
+    description: 'Number of concurrency'
+    required: false
+    default: ''
+  silent:
+    description: 'Silent mode'
+    required: false
+    default: 'false'
+  headers:
+    description: 'Custom HTTP headers to send with request, separated by commas'
+    required: false
+    default: ''
+  verbose:
+    description: 'Verbose mode'
+    required: false
+    default: 'false'
+  include30x:
+    description: 'Include 30x status code'
+    required: false
+    default: 'false'
 outputs:
   output: 
     description: 'JSON Result'
@@ -19,3 +81,9 @@ runs:
   args:
     - ${{ inputs.command }}
     - ${{ inputs.target }}
+    - ${{ inputs.timeout }}
+    - ${{ inputs.concurrency }}
+    - ${{ inputs.silent }}
+    - ${{ inputs.headers }}
+    - ${{ inputs.verbose }}
+    - ${{ inputs.include30x }}

deadfinder.gemspec

@@ -13,6 +13,7 @@ Gem::Specification.new do |s|
   s.files = ['lib/deadfinder.rb', 'lib/deadfinder/utils.rb', 'lib/deadfinder/logger.rb', 'lib/deadfinder/version.rb']
   s.metadata['rubygems_mfa_required'] = 'true'
   s.metadata['source_code_uri'] = 'https://github.com/hahwul/deadfinder'
+  s.required_ruby_version = '>= 3.3.0'
   s.add_runtime_dependency 'colorize', '~> 0.8.0', '>= 0.8.0'
   s.add_runtime_dependency 'concurrent-ruby-edge', '~> 0.6.0', '>= 0.6.0'
   s.add_runtime_dependency 'json', '~> 2.6.0', '>= 2.6.0'

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/hahwul/deadfinder:1.3.6
+FROM ghcr.io/hahwul/deadfinder:1.4.0
 COPY entrypoint.sh /entrypoint.sh
 RUN chmod 755 /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

github-action/entrypoint.sh

@@ -2,21 +2,35 @@
 
 # $1 : commands
 # $2 : target
+# $3 : timeout
+# $4 : concurrency
+# $5 : silent
+# $6 : headers
+# $7 : verbose
+# $8 : include30x
 # -------------
 
 export df=/usr/local/bundle/gems/deadfinder-*/bin/deadfinder
 
-case $1 in
-	'file')
-		$df file $2 -o /output.json
-		;;
-	'url')
-		$df url $2 -o /output.json
-		;;
-	'sitemap')
-		$df sitemap $2 -o /output.json
-		;;
-esac
+# Construct the command with additional options
+cmd="$df $1 $2 -o /output.json"
+[ -n "$3" ] && cmd="$cmd --timeout=$3"
+[ -n "$4" ] && cmd="$cmd --concurrency=$4"
+[ "$5" = "true" ] && cmd="$cmd --silent"
+[ "$7" = "true" ] && cmd="$cmd --verbose"
+[ "$8" = "true" ] && cmd="$cmd --include30x"
 
+# Add headers if provided
+if [ -n "$6" ]; then
+  IFS=',' read -r -a headers_array <<< "$6"
+  for header in "${headers_array[@]}"; do
+    cmd="$cmd -H \"$header\""
+  done
+fi
+
+# Execute the command
+eval $cmd
+
+# Read the output and set it as a GitHub Action output
 out=$(cat /output.json)
-echo "output=$out" >> $GITHUB_OUTPUT
+echo "output=$out" >> $GITHUB_OUTPUT

lib/deadfinder.rb

@@ -9,7 +9,6 @@
 require 'deadfinder/version'
 require 'concurrent-edge'
 require 'sitemap-parser'
-require 'set'
 require 'json'
 
 Channel = Concurrent::Channel
@@ -24,7 +23,9 @@ def default_options
       'timeout' => 10,
       'output' => '',
       'headers' => [],
-      'silent' => true
+      'silent' => true,
+      'verbose' => false,
+      'include30x' => false
     }
   end
 
@@ -77,14 +78,26 @@ def worker(_id, jobs, results, target, options)
         CacheSet[j] = true
         begin
           CacheQue[j] = true
-          URI.open(j, read_timeout: options['timeout'])
-        rescue StandardError => e
-          if e.to_s.include? '404 Not Found'
-            Logger.found "[#{e}] #{j}"
+          uri = URI.parse(j)
+
+          # Create HTTP request with timeout and headers
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = (uri.scheme == 'https')
+          http.read_timeout = options['timeout'].to_i if options['timeout']
+
+          request = Net::HTTP::Get.new(uri.request_uri)
+          response = http.request(request)
+          status_code = response.code.to_i
+          Logger.verbose "Status Code: #{status_code} for #{j}" if options['verbose']
+
+          if status_code >= 400 || (status_code >= 300 && options['include30x'])
+            Logger.found "[#{status_code} #{response.message}] #{j}"
             CacheQue[j] = false
             Output[target] ||= []
             Output[target] << j
           end
+        rescue StandardError => e
+          Logger.verbose "[#{e}] #{j}" if options['verbose']
         end
       end
       results << j
@@ -160,11 +173,13 @@ def gen_output(options)
 end
 
 class DeadFinder < Thor
+  class_option :include30x, aliases: :r, default: false, type: :boolean, desc: 'Include 30x redirections'
   class_option :concurrency, aliases: :c, default: 50, type: :numeric, desc: 'Number of concurrency'
   class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
   class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write JSON result'
   class_option :headers, aliases: :H, default: [], type: :array, desc: 'Custom HTTP headers to send with request'
   class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
+  class_option :verbose, aliases: :v, default: false, type: :boolean, desc: 'Verbose mode'
 
   desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
   def pipe

lib/deadfinder/logger.rb

@@ -36,4 +36,8 @@ def self.sub_done(text)
   def self.found(text)
     puts "  ✘ #{text}".colorize(:red) unless silent?
   end
+
+  def self.verbose(text)
+    puts '  ➜ '.colorize(:yellow) + text.to_s.colorize(:light_yellow) unless silent?
+  end
 end

lib/deadfinder/version.rb

@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-VERSION = '1.3.6'
+VERSION = '1.4.0'