Active Scanning and Replacing (AWS) Tokens (#1265)

Gemfile.lock

@@ -325,7 +325,7 @@ GEM
     rb-fsevent (0.9.8)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    rbtrace (0.4.12)
+    rbtrace (0.4.14)
       ffi (>= 1.0.6)
       msgpack (>= 0.4.3)
       optimist (>= 3.0.0)

lib/travis/api/v3/models/scan_result.rb

@@ -0,0 +1,30 @@
+
+module Travis::API::V3
+  class Models::ScanResult
+    attr_reader :id, :log_id, :job_id, :owner_id, :owner_type, :created_at, :formatted_content, :issues_found, :archived, :purged_at, :token, :token_created_at,
+      :job_number, :build_id, :build_number, :job_finished_at, :commit_sha, :commit_compare_url, :commit_branch, :repository_id
+
+    def initialize(attributes = {})
+      @id = attributes.fetch('id')
+      @log_id = attributes.fetch('log_id')
+      @job_id = attributes.fetch('job_id')
+      @owner_id = attributes.fetch('owner_id')
+      @owner_type = attributes.fetch('owner_type')
+      @created_at = attributes.fetch('created_at')
+      @formatted_content = attributes.fetch('formatted_content')
+      @issues_found = attributes.fetch('issues_found')
+      @archived = attributes.fetch('archived')
+      @purged_at = attributes.fetch('purged_at')
+      @token = attributes.fetch('token')
+      @token_created_at = attributes.fetch('token_created_at')
+      @job_number = attributes.fetch('job_number')
+      @build_id = attributes.fetch('build_id')
+      @build_number = attributes.fetch('build_number')
+      @job_finished_at = attributes.fetch('job_finished_at')
+      @commit_sha = attributes.fetch('commit_sha')
+      @commit_compare_url = attributes.fetch('commit_compare_url')
+      @commit_branch = attributes.fetch('commit_branch')
+      @repository_id = attributes.fetch('repository_id')
+    end
+  end
+end

lib/travis/api/v3/models/scanner_collection.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Travis::API::V3
+  class Models::ScannerCollection
+    def initialize(collection, total_count)
+      @collection = collection
+      @total_count = total_count
+    end
+
+    def count(*)
+      @total_count
+    end
+
+    def limit(*)
+      self
+    end
+
+    def offset(*)
+      self
+    end
+
+    def map
+      return @collection.map unless block_given?
+
+      @collection.map { |x| yield x }
+    end
+
+    def to_sql
+      "scanner_query:#{Time.now.to_i}"
+    end
+  end
+end

lib/travis/api/v3/permissions/repository.rb

@@ -42,6 +42,10 @@ def create_request?
       write?
     end
 
+    def check_scan_results?
+      write?
+    end
+
     def admin?
       access_control.adminable? object
     end

lib/travis/api/v3/queries/scan_result.rb

@@ -0,0 +1,15 @@
+module Travis::API::V3
+  class Queries::ScanResult < RemoteQuery
+    params :id
+
+    def find
+      scanner_client.get_scan_result(id)
+    end
+
+    private
+
+    def scanner_client
+      @_scanner_client ||= ScannerClient.new(nil)
+    end
+  end
+end

lib/travis/api/v3/queries/scan_results.rb

@@ -0,0 +1,22 @@
+module Travis::API::V3
+  class Queries::ScanResults < Query
+    params :repository_id, :offset, :limit
+
+    def all
+      # Reset the scan status on viewing the reports
+      Repository.find(repository_id).update!(scan_failed_at: nil)
+
+      page = (offset.to_i / limit.to_i) + 1
+      scanner_client(repository_id).scan_results(
+        page.to_s,
+        limit
+      )
+    end
+
+    private
+
+    def scanner_client(repository_id)
+      @_scanner_client ||= ScannerClient.new(repository_id)
+    end
+  end
+end

lib/travis/api/v3/renderer/repository.rb

@@ -1,9 +1,9 @@
 module Travis::API::V3
   class Renderer::Repository < ModelRenderer
     representation(:minimal,  :id, :name, :slug)
-    representation(:standard, :id, :name, :slug, :description, :github_id, :vcs_id, :vcs_type, :github_language, :active, :private, :owner, :owner_name, :vcs_name, :default_branch, :starred, :managed_by_installation, :active_on_org, :migration_status, :history_migration_status, :shared, :config_validation, :server_type)
-    representation(:experimental, :id, :name, :slug, :description, :vcs_id, :vcs_type, :github_id, :github_language, :active, :private, :owner, :default_branch, :starred, :current_build, :last_started_build, :next_build_number, :server_type)
-    representation(:internal, :id, :name, :slug, :github_id, :vcs_id, :vcs_type, :active, :private, :owner, :default_branch, :private_key, :token, :user_settings, :server_type)
+    representation(:standard, :id, :name, :slug, :description, :github_id, :vcs_id, :vcs_type, :github_language, :active, :private, :owner, :owner_name, :vcs_name, :default_branch, :starred, :managed_by_installation, :active_on_org, :migration_status, :history_migration_status, :shared, :config_validation, :server_type, :scan_failed_at)
+    representation(:experimental, :id, :name, :slug, :description, :vcs_id, :vcs_type, :github_id, :github_language, :active, :private, :owner, :default_branch, :starred, :current_build, :last_started_build, :next_build_number, :server_type, :scan_failed_at)
+    representation(:internal, :id, :name, :slug, :github_id, :vcs_id, :vcs_type, :active, :private, :owner, :default_branch, :private_key, :token, :user_settings, :server_type, :scan_failed_at)
     representation(:additional, :allow_migration)
 
     hidden_representations(:experimental, :internal)

lib/travis/api/v3/renderer/scan_result.rb

@@ -0,0 +1,27 @@
+module Travis::API::V3
+  class Renderer::ScanResult < ModelRenderer
+    representation(:minimal, :id, :created_at, :formatted_content, :issues_found, :job_id, :build_id, :job_number, :build_number, :job_finished_at,
+                             :commit_sha, :commit_compare_url, :commit_branch, :build_created_by)
+    representation(:standard, *representations[:minimal])
+
+    def build_created_by
+      job = Travis::API::V3::Models::Job.find(model.job_id)
+      build = Travis::API::V3::Models::Build.find(job.source_id)
+      return nil unless creator = build.sender
+      {
+        '@type' => build.sender_type.downcase,
+        '@href' => created_by_href(creator),
+        '@representation' => 'minimal'.freeze,
+        'id' => creator.id,
+        'login' => creator.login
+      }
+    end
+
+    private def created_by_href(creator)
+      case creator
+      when V3::Models::Organization then Renderer.href(:organization, script_name: script_name, id: creator.id)
+      when V3::Models::User         then Renderer.href(:user, script_name: script_name, id: creator.id)
+      end
+    end
+  end
+end

lib/travis/api/v3/renderer/scan_results.rb

@@ -0,0 +1,6 @@
+module Travis::API::V3
+  class Renderer::ScanResults < CollectionRenderer
+    type           :scan_results
+    collection_key :scan_results
+  end
+end

lib/travis/api/v3/routes.rb

@@ -283,6 +283,16 @@ module Routes
       end
     end
 
+    resource :scan_results do
+      route '/scan_results'
+      get :all
+    end
+
+    resource :scan_result do
+      route '/scan_result/{scan_result.id}'
+      get :find
+    end
+
     resource :user do
       capture id: :digit
       route '/user/{user.id}'

lib/travis/api/v3/scanner_client.rb

@@ -0,0 +1,84 @@
+
+# frozen_string_literal: true
+
+module Travis::API::V3
+  class ScannerClient
+    class ConfigurationError < StandardError; end
+
+    def initialize(repository_id)
+      @repository_id = repository_id
+    end
+
+    def scan_results(page, limit)
+      query_string = query_string_from_params(
+        repository_id: @repository_id,
+        limit: limit,
+        page: page || '1',
+      )
+      response = connection.get("/scan_results?#{query_string}")
+
+      handle_errors_and_respond(response) do |body|
+        scan_results = body['scan_results'].map do |scan_result|
+          Travis::API::V3::Models::ScanResult.new(scan_result)
+        end
+
+        Travis::API::V3::Models::ScannerCollection.new(scan_results, body.fetch('total_count', 0))
+      end
+    end
+
+    def get_scan_result(id)
+      response = connection.get("/scan_results/#{id}")
+      handle_errors_and_respond(response) do |body|
+        Travis::API::V3::Models::ScanResult.new(body.fetch('scan_result'))
+      end
+    end
+
+    private
+
+    def handle_errors_and_respond(response)
+      case response.status
+      when 200, 201
+        yield(response.body) if block_given?
+      when 202
+        true
+      when 204
+        true
+      when 400
+        raise Travis::API::V3::ClientError, response.body&.fetch('error', '')
+      when 403
+        raise Travis::API::V3::InsufficientAccess, response.body&.fetch('rejection_code', '')
+      when 404
+        raise Travis::API::V3::NotFound, response.body&.fetch('error', '')
+      when 422
+        raise Travis::API::V3::UnprocessableEntity, response.body&.fetch('error', '')
+      else
+        raise Travis::API::V3::ServerError, 'Scanner API failed'
+      end
+    end
+
+    def connection(timeout: 20)
+      @connection ||= Faraday.new(url: scanner_url, ssl: { ca_path: '/usr/lib/ssl/certs' }) do |conn|
+        conn.headers[:Authorization] = "Token token=\"#{scanner_token}\""
+        conn.headers['Content-Type'] = 'application/json'
+        conn.request :json
+        conn.response :json
+        conn.options[:open_timeout] = timeout
+        conn.options[:timeout] = timeout
+        conn.use OpenCensus::Trace::Integrations::FaradayMiddleware if Travis::Api::App::Middleware::OpenCensus.enabled?
+        conn.adapter :net_http
+      end
+    end
+
+    def scanner_url
+      Travis.config.scanner.url || raise(ConfigurationError, 'No Scanner API URL configured!')
+    end
+
+    def scanner_token
+      Travis.config.scanner.token || raise(ConfigurationError, 'No Scanner Auth Token configured!')
+    end
+
+    def query_string_from_params(params)
+      params.delete_if { |_, v| v.nil? || v.empty? }.to_query
+    end
+  end
+end

lib/travis/api/v3/services.rb

@@ -39,6 +39,8 @@ module Services
     Leads                 = Module.new { extend Services }
     Lint                  = Module.new { extend Services }
     Log                   = Module.new { extend Services }
+    ScanResult            = Module.new { extend Services }
+    ScanResults           = Module.new { extend Services }
     Messages              = Module.new { extend Services }
     Organization          = Module.new { extend Services }
     Organizations         = Module.new { extend Services }

lib/travis/api/v3/services/scan_result/find.rb

@@ -0,0 +1,19 @@
+module Travis::API::V3
+  class Services::ScanResult::Find < Service
+    params :id
+
+    def run!
+      raise LoginRequired unless access_control.full_access_or_logged_in?
+      scan_result = query(:scan_result).find
+
+      repository = Travis::API::V3::Models::Repository.find(scan_result.repository_id)
+      check_access(repository)
+
+      result scan_result
+    end
+
+    def check_access(repository)
+      access_control.permissions(repository).check_scan_results!
+    end
+  end
+end

lib/travis/api/v3/services/scan_results/all.rb

@@ -0,0 +1,19 @@
+module Travis::API::V3
+  class Services::ScanResults::All < Service
+    params :repository_id
+    paginate
+
+    def run!
+      raise LoginRequired unless access_control.full_access_or_logged_in?
+
+      repository = Travis::API::V3::Models::Repository.find(params['repository_id'])
+      check_access(repository)
+
+      result query(:scan_results).all
+    end
+
+    def check_access(repository)
+      access_control.permissions(repository).check_scan_results!
+    end
+  end
+end

lib/travis/config/defaults.rb

@@ -41,13 +41,9 @@ def fallback_logs_api_auth_token
             auth:                 { target_origin: nil },
             assets:               { host: HOSTS[Travis.env.to_sym] },
             amqp:                 { username: 'guest', password: 'guest', host: 'localhost', prefetch: 1 },
-            billing:              {},
             closeio:              { key: 'key' },
             gdpr:                 {},
-            insights:             { endpoint: 'https://insights.travis-ci.dev/', auth_token: 'secret' },
             database:             { adapter: 'postgresql', database: "travis_#{Travis.env}", encoding: 'unicode', min_messages: 'warning', variables: { statement_timeout: 10_000 } },
-            fallback_logs_api:    { url: fallback_logs_api_auth_url, token: fallback_logs_api_auth_token },
-            logs_api:             { url: logs_api_url, token: logs_api_auth_token },
             db:                   { max_statement_timeout_in_seconds: 15, slow_host_max_statement_timeout_in_seconds: 60},
             log_options:          { s3: { access_key_id: '', secret_access_key: ''}},
             s3:                   { access_key_id: '', secret_access_key: ''},
@@ -86,10 +82,15 @@ def fallback_logs_api_auth_token
             build_backup_options: ENV['GCE_BUILD_BACKUP_OPTIONS'] ? JSON.parse(ENV['GCE_BUILD_BACKUP_OPTIONS']) : { gcs: { bucket_name: 'fillme', json_key: JSON.parse("{\n  \"type\": \"service_account\",\n  \"project_id\": \"fillme\",\n  \"private_key_id\": \"b1c57117b4a0b8ae2af2f45b19a1cf9727bc6caf\",\n  \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDRaHA5z0vXNVSr\\nlLVd/smJpNkpzk4BoHq+zcuuzvKTf1ZY1LnrAhldUbDTKY67c06eOYwVrQc3tEIp\\nJCNhIDNY1lRfGJag6t2v5C710WY+X7qVnRhSpgthvWFX/Rm9KIv3be8AJvUTMUDQ\\nAL10eYrIWJOI/J59khuKvr7khIlysASwGoZc8UgcufxGuwCziNyEIfH1nxiBKILR\\nM1+LdYi/Avyb4bQth5x03THVEmdRDjV7Yoo2c17XElIXtkl03nUce4w8BCu+T1G2\\nKRkApHcQ8R0BDTjjBzaQuXTtTpLvkmZzJ1i0/kPNdnT70lv6N2nI1AN2DVkXDCkG\\nNlZ283W7AgMBAAECggEAC/W6CyMywqzSFCafISovGoRmvsOAowkmWYVpb6d0JUZt\\niQ9FOw3YowLKZZUHCN+yCslgndBPDDhoWu8schyjshwzn2bJG5GubaBLqlB2VXOk\\nNW1OeVHwbnmheKQE90+8hropn0maT6lNeVPBfkh+y6h7bKR47NUOa6MvRd/n9bvL\\nT7pP5ZAHoPoTcbUftOX0gDq0u+uRULe/rduxB0S2EHDEtZEH+ioUOP9AomnaRDSy\\n0spH1s2FUZxIbKBQzsrqMCai4MSjeUrJMTR3ZlpfXePirettvilSWqEXDLvvwaak\\ngehELuM5lH4T49wf4PmEYZ8Jqkh9ku+oNYJdJvGR2QKBgQD7KjhJx9usl0afrIH3\\nw7saHELluWGqHNa6j+TJDpY7N5lLLIym/br9d+cuLTF5CBEHJ502coDR9cyrLVZX\\na05CGmEfSVrSrLUyAU+mHHdsn8n6CCATmlgtPyzzt2c29J7dHUZL94zW/yG6Btg6\\nm+nY4eBKreLpj0+3KbhI/q0q1wKBgQDVcG8Ek3Kt2buOrpDBqxcwB31QljntT+7+\\nYcTZctYL/y7Lm2VcTjserNa3AjG59Z5iaQjKFPhbAvMHfppklyiVSVBRfn4bLTcx\\nSM9I+lntODtGI/BiHVE7hfoYKzwz/3Aj3npiOO9xnOfAgEubGn9DrOzLXPsvWN7E\\nz+/iSr4zvQKBgHFVB7kjGZizWgbKzIqEI3UQs479K3ibMrlUHKQslNV7rQwiugTQ\\nEQQ2inZnph866JQV5/adjEsxYn0LJB6mKNXjGVgIvZa6n7hEpzAJQEoff//2kqLF\\nzmv8SchfRY+iqdyUTRgSR9broMhUNlWb7NUUdyS7edxx8kJv7NvjLzhZAoGAMre9\\n2bOD26XSeKwof6y9HM+ayox4BVkqLE5lLVqpXD5uCznI0y9PwxFFEEW4NT0VPsNA\\nsGxdO5suzsgZve9hWGAMcuEA7EpJRC/N+cRrm//xrdAabeYTiHZkoFudualoJ03V\\nfQOUekXTmB2kWZ3pQdaUihp1IaIXhWL32Kj0G20CgYAfGInHUZf5mqxRK/id8B5Q\\nYmpnLDXsNu1I4qFKeaBo4dF2SGByNW7fVbK/BdCSg5Ov3Ui6m3QWBbJXh1a8QVYA\\nICvwJWqm53bNpocrFPAeXLy9xL5/5CEeVGQNcxFvUF3QgaPVmjbTsZk8vjbidUZk\\nOU8bArrUjGTxNJOe7GebhA==\\n-----END PRIVATE KEY-----\\n\",\n  \"client_email\": \"[email protected]\",\n  \"client_id\": \"100937792194965642651\",\n  \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n  \"token_uri\": \"https://oauth2.googleapis.com/token\",\n  \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n  \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/service%40fillme.iam.gserviceaccount.com\"\n}\n") } },
             merge:                { auth_token: 'merge-auth-token', api_url: 'https://merge.localhost' },
             force_authentication: false,
-            yml:                  { url: 'https://yml.travis-ci.org', token: 'secret', auth_key: 'abc123' },
             read_only:            ENV['READ_ONLY'] || false,
-            vcs: {},
-            job_log_access_permissions: { time_based_limit: false, access_based_limit: false, older_than_days: 365, max_days_value: 730, min_days_value: 30 }
+            job_log_access_permissions: { time_based_limit: false, access_based_limit: false, older_than_days: 365, max_days_value: 730, min_days_value: 30 },
+            billing:              {},
+            vcs:                  {},
+            yml:                  { url: 'https://yml.travis-ci.org', token: 'secret', auth_key: 'abc123' },
+            logs_api:             { url: logs_api_url, token: logs_api_auth_token },
+            fallback_logs_api:    { url: fallback_logs_api_auth_url, token: fallback_logs_api_auth_token },
+            scanner:              {},
+            insights:             { endpoint: 'https://insights.travis-ci.dev/', auth_token: 'secret' }
 
     default :_access => [:key]
 

spec/spec_helper.rb

@@ -31,6 +31,7 @@
 require 'auth/helpers'
 require 'support/active_record'
 require 'support/billing_spec_helper'
+require 'support/scanner_spec_helper'
 require 'support/env'
 require 'support/formats'
 require 'support/gcs'
@@ -79,6 +80,7 @@ def parsed_body
   c.include Support::Env
   c.include Support::AuthHelpers, auth_helpers: true
   c.include Support::BillingSpecHelper, billing_spec_helper: true
+  c.include Support::ScannerSpecHelper, scanner_spec_helper: true
   c.include Support::GdprSpecHelper, gdpr_spec_helper: true
 
   # for auth tests against staging, how the hell does this work, if at all