implement uploads

Author: elg0nz

.gitignore

@@ -12,4 +12,4 @@ __pycache__
 *.rej
 Thumbs.db
 pkg/
-.gem
+*.gem

Gemfile

@@ -6,6 +6,7 @@ gem 'rack'
 gem 'puma'
 gem 'thor'
 gem 'json' 
+gem 'httpx', '~> 1.3'
 
 group :development, :test do
   gem 'rspec', '~> 3.13'

Gemfile.lock

@@ -2,6 +2,9 @@ GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.5.1)
+    http-2 (1.0.1)
+    httpx (1.3.0)
+      http-2 (>= 1.0.0)
     json (2.7.1)
     nio4r (2.7.1)
     puma (6.4.2)
@@ -29,6 +32,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  httpx (~> 1.3)
   json
   puma
   rack

README.md

@@ -67,6 +67,6 @@ Hawksi is released under the MIT license. See the [LICENSE](LICENSE) file for de
 
 ## TODO
 
-- [ ] Add upload command
+- [x] Add upload command
 - [ ] Add standard for code formatting
 - [ ] Store logs in log/hawksi.log

hawksi.gemspec

@@ -32,6 +32,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'puma', '~> 5.0'
   spec.add_dependency 'thor', '~> 1.1'
   spec.add_dependency 'json', '~> 2.5'
+  spec.add_dependency 'httpx', '~> 1.3'
+
 
   spec.add_development_dependency 'bundler', '~> 2.2'
   spec.add_development_dependency 'rake', '~> 13.0'

lib/cli.rb

@@ -4,6 +4,7 @@
 require_relative 'request_interceptor'
 require_relative 'file_storage'
 require_relative 'captures_cli'
+require_relative 'uploads_cli'
 
 class CLI < Thor
   desc "start", "Starts the Hawksi Interceptor server"
@@ -23,6 +24,9 @@ def stop
   desc "captures", "Manage captures"
   subcommand "captures", CapturesCLI
 
+  desc "uploads", "Uploads captured requests and responses"
+  subcommand "uploads", UploadsCLI
+
   desc "clear", "Clears stored request/response data"
   def clear
     FileUtils.rm_rf('./intercepted_data/requests')

lib/file_handler.rb

@@ -0,0 +1,47 @@
+require 'fileutils'
+require 'securerandom'
+
+class FileHandler
+  def initialize(base_dir, logger)
+    @base_dir = base_dir
+    @logger = logger
+  end
+
+  def find_files
+    glob_pattern = File.join(@base_dir, '{requests,responses}', '*.json')
+    Dir.glob(glob_pattern)
+  end
+
+  def generate_client_uuid
+    uuid_file = File.join(@base_dir, 'client_uuid.txt')
+    if File.exist?(uuid_file)
+      client_uuid = File.read(uuid_file).strip
+      @logger.info "Using existing client UUID: #{client_uuid}"
+    else
+      client_uuid = SecureRandom.uuid
+      File.write(uuid_file, client_uuid)
+      @logger.info "Generated and stored new client UUID: #{client_uuid}"
+    end
+    client_uuid
+  end
+
+  def create_tar_gz_files(files)
+    tar_gz_files = []
+    files.each do |file|
+      tar_file = "#{file}.tar"
+      tar_gz_file = "#{tar_file}.gz"
+
+      unless File.exist?(tar_gz_file)
+        # Create tarball
+        system("tar -cf #{tar_file} #{file}")
+
+        # Compress tarball
+        system("gzip #{tar_file}")
+      end
+
+
+      tar_gz_files << tar_gz_file
+    end
+    tar_gz_files
+  end
+end

lib/file_uploader.rb

@@ -0,0 +1,62 @@
+require 'httpx'
+require 'logger'
+
+class FileUploader
+  UPLOAD_URL = "https://run.mocky.io/v3/86b7c197-1ac7-4d7b-a985-24255ae89722"
+
+  def initialize(logger, client_uuid)
+    @logger = logger
+    @client_uuid = client_uuid
+  end
+
+  def upload_files(tar_gz_files)
+    log_upload_start(tar_gz_files)
+    threads = create_upload_threads(tar_gz_files)
+    wait_for_threads(threads)
+  end
+
+  private
+
+  def log_upload_start(tar_gz_files)
+    @logger.info "Starting upload of #{tar_gz_files.size} files."
+  end
+
+  def create_upload_threads(tar_gz_files)
+    tar_gz_files.each_slice(10).map do |batch|
+      Thread.start { upload_batch(batch) }
+    end
+  end
+
+  def wait_for_threads(threads)
+    threads.each(&:join)
+  end
+
+  def upload_batch(batch)
+    batch.each { |tar_gz_file| upload_file(tar_gz_file) }
+  end
+
+  def upload_file(tar_gz_file)
+    HTTPX.wrap do |client|
+      response = post_file(client, tar_gz_file)
+      log_upload_result(tar_gz_file, response)
+      response
+    end
+  end
+
+  def post_file(client, tar_gz_file)
+    client.post(UPLOAD_URL,
+                headers: { "x-client-id" => @client_uuid },
+                body: File.read(tar_gz_file))
+  rescue => e
+    @logger.error "Failed to upload #{tar_gz_file}: #{e.message}"
+    nil
+  end
+
+  def log_upload_result(tar_gz_file, response)
+    if response
+      @logger.info "Uploaded #{tar_gz_file}: #{response.status}"
+    else
+      @logger.error "Failed to upload #{tar_gz_file}"
+    end
+  end
+end

lib/uploads_cli.rb

@@ -0,0 +1,68 @@
+require 'thor'
+require 'securerandom'
+require 'httpx'
+require 'fileutils'
+require 'open3'
+require_relative 'file_storage'
+require_relative 'file_uploader'
+require_relative 'file_handler'
+require 'logger'
+
+class UploadsCLI < Thor
+  desc "update", "Update uploaded requests and responses"
+  option :base_dir, type: :string, desc: 'Base directory for storing intercepted data. Defaults to ./tmp/intercepted_data'
+  
+  def initialize(*args)
+    super
+    @logger = Logger.new(STDOUT)
+    @logger.level = Logger::INFO
+    @base_dir = options[:base_dir] || FileStorage.base_dir
+    @file_handler = FileHandler.new(@base_dir, @logger)
+    @client_uuid = get_client_uuid
+    @file_uploader = FileUploader.new(@logger, @client_uuid)
+  end
+
+  def update(*args)
+    set_base_dir
+    files = find_files
+
+    if files.empty?
+      @logger.info "No captured requests or responses found."
+      return
+    end
+
+    tar_gz_files = create_tar_gz_files(files)
+    return unless valid_tar_gz_files?(tar_gz_files)
+
+    upload_files(tar_gz_files)
+  end
+
+  private
+
+  def set_base_dir
+    FileStorage.base_dir = @base_dir
+  end
+
+  def get_client_uuid
+    @file_handler.generate_client_uuid
+  end
+
+  def find_files
+    @file_handler.find_files
+  end
+
+  def create_tar_gz_files(files)
+    @file_handler.create_tar_gz_files(files)
+  end
+
+  def valid_tar_gz_files?(tar_gz_files)
+    return true if tar_gz_files.is_a?(Array)
+
+    @logger.error "Expected tar_gz_files to be an array, but got #{tar_gz_files.class}"
+    false
+  end
+
+  def upload_files(tar_gz_files)
+    @file_uploader.upload_files(tar_gz_files)
+  end
+end