Merge pull request #21 from hahwul/dev

Release  1.3.6

Author: hahwul

.rubocop.yml

@@ -5,5 +5,43 @@ AllCops:
     - 'bin/*'
     - '*.gemspec'
     - 'Gemfile*'
+  SuggestExtensions: false
+
 Style/MutableConstant:
-  Enabled: false
+  Enabled: false
+
+Style/Documentation:
+  Exclude:
+    - 'lib/deadfinder.rb'
+    - 'lib/deadfinder/logger.rb'
+    - 'lib/deadfinder/utils.rb'
+
+Metrics/AbcSize:
+  Exclude:
+    - 'lib/deadfinder.rb'
+    - 'lib/deadfinder/utils.rb'
+
+Metrics/CyclomaticComplexity:
+  Exclude:
+    - 'lib/deadfinder.rb'
+
+Metrics/MethodLength:
+  Exclude:
+    - 'lib/deadfinder.rb'
+    - 'lib/deadfinder/utils.rb'
+
+Metrics/PerceivedComplexity:
+  Exclude:
+    - 'lib/deadfinder.rb'
+
+Lint/SuppressedException:
+  Exclude:
+    - 'lib/deadfinder.rb'
+
+Security/Open:
+  Exclude:
+    - 'lib/deadfinder.rb'
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/utils_spec.rb'

README.md

@@ -2,7 +2,7 @@
 
 Dead link (broken link) means a link within a web page that cannot be connected. These links can have a negative impact to SEO and Security. This tool makes it easy to identify and modify.
 
-![](https://user-images.githubusercontent.com/13212227/192243070-0c960680-ae08-4f30-8cf9-0844eca7c5ea.png)
+![](https://github.com/user-attachments/assets/92129de9-90c6-41e0-a424-883fe30858f6)
 
 ## Installation
 ### Install with Gem
@@ -34,7 +34,7 @@ deadfinder sitemap https://www.hahwul.com/sitemap.xml
 ```yml
 steps:
 - name: Run DeadFinder
-  uses: hahwul/[email protected].5
+  uses: hahwul/[email protected].6
   id: broken-link
   with:
     command: sitemap
@@ -49,7 +49,7 @@ steps:
 require 'deadfinder'
 
 app = DeadFinderRunner.new
-options = {}
+options = app.default_options
 options['concurrency'] = 30
 
 app.run('https://www.hahwul.com/2022/09/30/deadfinder/', options)
@@ -67,12 +67,14 @@ Commands:
   deadfinder version                # Show version.
 
 Options:
-  c, [--concurrency=N]          # Number of concurrncy
-                                # Default: 20
-  t, [--timeout=N]              # Timeout in seconds
-                                # Default: 10
-  o, [--output=OUTPUT]          # File to write JSON result
-  H, [--headers=one two three]  # Custom HTTP headers to send with request
+  -c, [--concurrency=N]                           # Number of concurrency
+                                                  # Default: 50
+  -t, [--timeout=N]                               # Timeout in seconds
+                                                  # Default: 10
+  -o, [--output=OUTPUT]                           # File to write JSON result
+  -H, [--headers=one two three]                   # Custom HTTP headers to send with request
+  -s, [--silent], [--no-silent], [--skip-silent]  # Silent mode
+                                                  # Default: false
 ```
 
 ## Modes

deadfinder.gemspec

@@ -13,12 +13,12 @@ Gem::Specification.new do |s|
   s.files = ['lib/deadfinder.rb', 'lib/deadfinder/utils.rb', 'lib/deadfinder/logger.rb', 'lib/deadfinder/version.rb']
   s.metadata['rubygems_mfa_required'] = 'true'
   s.metadata['source_code_uri'] = 'https://github.com/hahwul/deadfinder'
-  s.add_runtime_dependency 'colorize', '~> 0.8.0'
-  s.add_runtime_dependency 'concurrent-ruby-edge', '~> 0.6.0'
-  s.add_runtime_dependency 'json', '~> 2.6.0'
-  s.add_runtime_dependency 'nokogiri', '~> 1.13.0'
-  s.add_runtime_dependency 'open-uri', '~> 0.2.0'
-  s.add_runtime_dependency 'set', '~> 1.0.0'
-  s.add_runtime_dependency 'sitemap-parser', '~> 0.5.0'
-  s.add_runtime_dependency 'thor', '~> 1.2.0'
+  s.add_runtime_dependency 'colorize', '>= 0.8.0'
+  s.add_runtime_dependency 'concurrent-ruby-edge', '>= 0.6.0'
+  s.add_runtime_dependency 'json', '>= 2.6.0'
+  s.add_runtime_dependency 'nokogiri', '>= 1.13.0'
+  s.add_runtime_dependency 'open-uri', '>= 0.2.0'
+  s.add_runtime_dependency 'set', '>= 1.0.0'
+  s.add_runtime_dependency 'sitemap-parser', '>= 0.5.0'
+  s.add_runtime_dependency 'thor', '>= 1.2.0'
 end

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/hahwul/deadfinder:1.3.5
+FROM ghcr.io/hahwul/deadfinder:1.3.6
 COPY entrypoint.sh /entrypoint.sh
 RUN chmod 755 /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

lib/deadfinder.rb

@@ -18,7 +18,18 @@
 Output = Concurrent::Map.new
 
 class DeadFinderRunner
+  def default_options
+    {
+      'concurrency' => 50,
+      'timeout' => 10,
+      'output' => '',
+      'headers' => [],
+      'silent' => true
+    }
+  end
+
   def run(target, options)
+    Logger.set_silent if options['silent']
     headers = options['headers'].each_with_object({}) do |header, hash|
       kv = header.split(': ')
       hash[kv[0]] = kv[1]
@@ -96,35 +107,49 @@ def extract_links(page)
 end
 
 def run_pipe(options)
+  Logger.set_silent if options['silent']
+
+  Logger.info 'Reading from STDIN'
   app = DeadFinderRunner.new
   while $stdin.gets
     target = $LAST_READ_LINE.chomp
+    Logger.target "Checking: #{target}"
     app.run target, options
   end
   gen_output(options)
 end
 
 def run_file(filename, options)
+  Logger.set_silent if options['silent']
+
+  Logger.info "Reading: #{filename}"
   app = DeadFinderRunner.new
   File.foreach(filename) do |line|
     target = line.chomp
+    Logger.target "Checking: #{target}"
     app.run target, options
   end
   gen_output(options)
 end
 
 def run_url(url, options)
+  Logger.set_silent if options['silent']
+
+  Logger.target "Checking: #{url}"
   app = DeadFinderRunner.new
   app.run url, options
   gen_output(options)
 end
 
 def run_sitemap(sitemap_url, options)
+  Logger.set_silent if options['silent']
+  Logger.info "Parsing sitemap: #{sitemap_url}"
   app = DeadFinderRunner.new
   base_uri = URI(sitemap_url)
   sitemap = SitemapParser.new sitemap_url, { recurse: true }
   sitemap.to_a.each do |url|
     turl = generate_url url, base_uri
+    Logger.target "Checking: #{turl}"
     app.run turl, options
   end
   gen_output(options)
@@ -139,28 +164,25 @@ class DeadFinder < Thor
   class_option :timeout, aliases: :t, default: 10, type: :numeric, desc: 'Timeout in seconds'
   class_option :output, aliases: :o, default: '', type: :string, desc: 'File to write JSON result'
   class_option :headers, aliases: :H, default: [], type: :array, desc: 'Custom HTTP headers to send with request'
+  class_option :silent, aliases: :s, default: false, type: :boolean, desc: 'Silent mode'
 
   desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
   def pipe
-    Logger.info 'Pipe mode'
     run_pipe options
   end
 
   desc 'file <FILE>', 'Scan the URLs from File. (e.g deadfinder file urls.txt)'
   def file(filename)
-    Logger.info 'File mode'
     run_file filename, options
   end
 
   desc 'url <URL>', 'Scan the Single URL.'
   def url(url)
-    Logger.info 'Single URL mode'
     run_url url, options
   end
 
   desc 'sitemap <SITEMAP-URL>', 'Scan the URLs from sitemap.'
   def sitemap(sitemap)
-    Logger.info 'Sitemap mode'
     run_sitemap sitemap, options
   end
 

lib/deadfinder/logger.rb

@@ -3,27 +3,37 @@
 require 'colorize'
 
 class Logger
+  @silent = false
+
+  def self.set_silent
+    @silent = true
+  end
+
+  def self.silent?
+    @silent
+  end
+
   def self.info(text)
-    puts 'ℹ '.colorize(:blue) + text.to_s.colorize(:light_blue)
+    puts 'ℹ '.colorize(:blue) + text.to_s.colorize(:light_blue) unless silent?
   end
 
   def self.error(text)
-    puts '⚠︎ '.colorize(:red) + text.to_s
+    puts '⚠︎ '.colorize(:red) + text.to_s unless silent?
   end
 
   def self.target(text)
-    puts '► '.colorize(:green) + text.to_s.colorize(:light_green)
+    puts '► '.colorize(:green) + text.to_s.colorize(:light_green) unless silent?
   end
 
   def self.sub_info(text)
-    puts '  ● '.colorize(:blue) + text.to_s.colorize(:light_blue)
+    puts '  ● '.colorize(:blue) + text.to_s.colorize(:light_blue) unless silent?
   end
 
   def self.sub_done(text)
-    puts '  ✓ '.colorize(:blue) + text.to_s.colorize(:light_blue)
+    puts '  ✓ '.colorize(:blue) + text.to_s.colorize(:light_blue) unless silent?
   end
 
   def self.found(text)
-    puts "  ✘ #{text}".colorize(:red)
+    puts "  ✘ #{text}".colorize(:red) unless silent?
   end
 end

lib/deadfinder/version.rb

@@ -1,3 +1,3 @@
 # frozen_string_literal: true
 
-VERSION = '1.3.5'
+VERSION = '1.3.6'