From 087996161e0b64e8f766368b550360847483ea28 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Sun, 17 Sep 2023 23:46:38 +0300 Subject: [PATCH] Import updater files from dependabot-core (#799) --- copy-updater-files.ps1 | 127 ++++- updater/Gemfile | 5 + updater/Gemfile.lock | 20 + updater/bin/fetch_files.rb | 26 + updater/bin/update_files.rb | 26 + updater/config/.npmrc | 20 + updater/config/.yarnrc | 6 + updater/lib/dependabot/api_client.rb | 200 +++++++ updater/lib/dependabot/base_command.rb | 87 +++ updater/lib/dependabot/dependency_change.rb | 113 ++++ .../dependabot/dependency_change_builder.rb | 99 ++++ .../lib/dependabot/dependency_group_engine.rb | 84 +++ updater/lib/dependabot/dependency_snapshot.rb | 113 ++++ .../lib/dependabot/file_fetcher_command.rb | 222 ++++++++ updater/lib/dependabot/job.rb | 341 ++++++++++++ updater/lib/dependabot/logger/formats.rb | 48 ++ updater/lib/dependabot/sentry.rb | 29 + updater/lib/dependabot/service.rb | 176 ++++++ updater/lib/dependabot/setup.rb | 63 +++ .../lib/dependabot/update_files_command.rb | 148 +++++ updater/lib/dependabot/updater.rb | 86 +++ .../updater/dependency_group_change_batch.rb | 109 ++++ .../lib/dependabot/updater/error_handler.rb | 210 +++++++ updater/lib/dependabot/updater/errors.rb | 15 + .../updater/group_update_creation.rb | 305 +++++++++++ updater/lib/dependabot/updater/operations.rb | 58 ++ .../create_group_update_pull_request.rb | 69 +++ .../create_security_update_pull_request.rb | 270 +++++++++ .../operations/group_update_all_versions.rb | 121 ++++ .../refresh_group_update_pull_request.rb | 136 +++++ .../refresh_security_update_pull_request.rb | 233 ++++++++ .../refresh_version_update_pull_request.rb | 213 +++++++ .../updater/operations/update_all_versions.rb | 257 +++++++++ .../updater/security_update_helpers.rb | 158 ++++++ updater/spec/dependabot/api_client_spec.rb | 436 +++++++++++++++ .../spec/dependabot/dependency_change_spec.rb | 159 ++++++ .../dependency_group_engine_spec.rb | 246 +++++++++ updater/spec/dependabot/job_spec.rb | 501 +++++++++++++++++ updater/spec/dependabot/sentry_spec.rb | 102 ++++ updater/spec/dependabot/service_spec.rb | 518 ++++++++++++++++++ .../dependabot/updater/error_handler_spec.rb | 223 ++++++++ .../dependabot/updater/operations_spec.rb | 114 ++++ .../spec/fixtures/bundler/original/Gemfile | 6 + .../fixtures/bundler/original/Gemfile.lock | 16 + updater/spec/fixtures/bundler/updated/Gemfile | 6 + .../fixtures/bundler/updated/Gemfile.lock | 16 + .../fixtures/bundler_gemspec/original/Gemfile | 3 + .../bundler_gemspec/original/Gemfile.lock | 39 ++ .../bundler_gemspec/original/library.gemspec | 12 + .../fixtures/bundler_git/original/Gemfile | 5 + .../bundler_git/original/Gemfile.lock | 21 + .../bundler_grouped_by_types/original/Gemfile | 10 + .../original/Gemfile.lock | 35 ++ .../bundler_vendored/original/Gemfile | 7 + .../bundler_vendored/original/Gemfile.lock | 24 + .../docker/original/Dockerfile.bundler | 1 + .../fixtures/docker/original/Dockerfile.cargo | 1 + .../spec/fixtures/job_definitions/README.md | 26 + .../version_updates/group_update_all.yaml | 38 ++ .../group_update_all_by_dependency_type.yaml | 43 ++ .../group_update_all_empty_group.yaml | 38 ++ .../group_update_all_overlapping_groups.yaml | 42 ++ .../group_update_all_semver_grouping.yaml | 40 ++ ...l_semver_grouping_with_global_ignores.yaml | 41 ++ .../group_update_all_with_existing_pr.yaml | 43 ++ .../group_update_all_with_ungrouped.yaml | 38 ++ .../group_update_all_with_vendoring.yaml | 38 ++ .../group_update_peer_manifests.yaml | 1 + .../version_updates/group_update_refresh.yaml | 45 ++ ...p_update_refresh_dependencies_changed.yaml | 48 ++ .../group_update_refresh_empty_group.yaml | 45 ++ .../group_update_refresh_missing_group.yaml | 41 ++ .../group_update_refresh_similar_pr.yaml | 53 ++ ...group_update_refresh_versions_changed.yaml | 45 ++ .../version_updates/update_all_simple.yaml | 33 ++ .../group_update_peer_manifests.yaml | 39 ++ .../fixtures/jobs/job_with_credentials.json | 58 ++ updater/spec/fixtures/rubygems-index | 10 + updater/spec/fixtures/rubygems-info-a | 6 + updater/spec/fixtures/rubygems-info-b | 4 + .../spec/fixtures/rubygems-versions-a.json | 59 ++ .../spec/fixtures/rubygems-versions-b.json | 59 ++ updater/spec/spec_helper.rb | 6 + .../spec/support/dependency_file_helpers.rb | 11 + updater/spec/support/dummy_pkg_helpers.rb | 61 +++ 85 files changed, 7691 insertions(+), 14 deletions(-) create mode 100644 updater/bin/fetch_files.rb create mode 100644 updater/bin/update_files.rb create mode 100644 updater/config/.npmrc create mode 100644 updater/config/.yarnrc create mode 100644 updater/lib/dependabot/api_client.rb create mode 100644 updater/lib/dependabot/base_command.rb create mode 100644 updater/lib/dependabot/dependency_change.rb create mode 100644 updater/lib/dependabot/dependency_change_builder.rb create mode 100644 updater/lib/dependabot/dependency_group_engine.rb create mode 100644 updater/lib/dependabot/dependency_snapshot.rb create mode 100644 updater/lib/dependabot/file_fetcher_command.rb create mode 100644 updater/lib/dependabot/job.rb create mode 100644 updater/lib/dependabot/logger/formats.rb create mode 100644 updater/lib/dependabot/sentry.rb create mode 100644 updater/lib/dependabot/service.rb create mode 100644 updater/lib/dependabot/setup.rb create mode 100644 updater/lib/dependabot/update_files_command.rb create mode 100644 updater/lib/dependabot/updater.rb create mode 100644 updater/lib/dependabot/updater/dependency_group_change_batch.rb create mode 100644 updater/lib/dependabot/updater/error_handler.rb create mode 100644 updater/lib/dependabot/updater/errors.rb create mode 100644 updater/lib/dependabot/updater/group_update_creation.rb create mode 100644 updater/lib/dependabot/updater/operations.rb create mode 100644 updater/lib/dependabot/updater/operations/create_group_update_pull_request.rb create mode 100644 updater/lib/dependabot/updater/operations/create_security_update_pull_request.rb create mode 100644 updater/lib/dependabot/updater/operations/group_update_all_versions.rb create mode 100644 updater/lib/dependabot/updater/operations/refresh_group_update_pull_request.rb create mode 100644 updater/lib/dependabot/updater/operations/refresh_security_update_pull_request.rb create mode 100644 updater/lib/dependabot/updater/operations/refresh_version_update_pull_request.rb create mode 100644 updater/lib/dependabot/updater/operations/update_all_versions.rb create mode 100644 updater/lib/dependabot/updater/security_update_helpers.rb create mode 100644 updater/spec/dependabot/api_client_spec.rb create mode 100644 updater/spec/dependabot/dependency_change_spec.rb create mode 100644 updater/spec/dependabot/dependency_group_engine_spec.rb create mode 100644 updater/spec/dependabot/job_spec.rb create mode 100644 updater/spec/dependabot/sentry_spec.rb create mode 100644 updater/spec/dependabot/service_spec.rb create mode 100644 updater/spec/dependabot/updater/error_handler_spec.rb create mode 100644 updater/spec/dependabot/updater/operations_spec.rb create mode 100644 updater/spec/fixtures/bundler/original/Gemfile create mode 100644 updater/spec/fixtures/bundler/original/Gemfile.lock create mode 100644 updater/spec/fixtures/bundler/updated/Gemfile create mode 100644 updater/spec/fixtures/bundler/updated/Gemfile.lock create mode 100644 updater/spec/fixtures/bundler_gemspec/original/Gemfile create mode 100644 updater/spec/fixtures/bundler_gemspec/original/Gemfile.lock create mode 100644 updater/spec/fixtures/bundler_gemspec/original/library.gemspec create mode 100644 updater/spec/fixtures/bundler_git/original/Gemfile create mode 100644 updater/spec/fixtures/bundler_git/original/Gemfile.lock create mode 100644 updater/spec/fixtures/bundler_grouped_by_types/original/Gemfile create mode 100644 updater/spec/fixtures/bundler_grouped_by_types/original/Gemfile.lock create mode 100644 updater/spec/fixtures/bundler_vendored/original/Gemfile create mode 100644 updater/spec/fixtures/bundler_vendored/original/Gemfile.lock create mode 100644 updater/spec/fixtures/docker/original/Dockerfile.bundler create mode 100644 updater/spec/fixtures/docker/original/Dockerfile.cargo create mode 100644 updater/spec/fixtures/job_definitions/README.md create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_by_dependency_type.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_empty_group.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_overlapping_groups.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_semver_grouping.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_semver_grouping_with_global_ignores.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_existing_pr.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_ungrouped.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_vendoring.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_peer_manifests.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_dependencies_changed.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_empty_group.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_missing_group.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_similar_pr.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_versions_changed.yaml create mode 100644 updater/spec/fixtures/job_definitions/bundler/version_updates/update_all_simple.yaml create mode 100644 updater/spec/fixtures/job_definitions/docker/version_updates/group_update_peer_manifests.yaml create mode 100644 updater/spec/fixtures/jobs/job_with_credentials.json create mode 100644 updater/spec/fixtures/rubygems-index create mode 100644 updater/spec/fixtures/rubygems-info-a create mode 100644 updater/spec/fixtures/rubygems-info-b create mode 100644 updater/spec/fixtures/rubygems-versions-a.json create mode 100644 updater/spec/fixtures/rubygems-versions-b.json create mode 100644 updater/spec/support/dependency_file_helpers.rb create mode 100644 updater/spec/support/dummy_pkg_helpers.rb diff --git a/copy-updater-files.ps1 b/copy-updater-files.ps1 index 3079158ec..e6e053c8c 100644 --- a/copy-updater-files.ps1 +++ b/copy-updater-files.ps1 @@ -2,21 +2,120 @@ Param( [string] $tag = "v0.230.0" ) -$hash = [ordered]@{ - ".ruby-version" = "../.ruby-version" +$files = @( + ".ruby-version" - "updater/lib/dependabot/environment.rb" = "lib/dependabot/environment.rb" - "updater/spec/dependabot/environment_spec.rb" = "spec/dependabot/environment_spec.rb" - # "updater/spec/spec_helper.rb" = "spec/spec_helper.rb" -} + "updater/bin/fetch_files.rb" + "updater/bin/update_files.rb" + + "updater/config/.npmrc" + "updater/config/.yarnrc" + + "updater/lib/dependabot/logger/formats.rb" + "updater/lib/dependabot/updater/operations/create_group_update_pull_request.rb" + "updater/lib/dependabot/updater/operations/create_security_update_pull_request.rb" + "updater/lib/dependabot/updater/operations/group_update_all_versions.rb" + "updater/lib/dependabot/updater/operations/refresh_group_update_pull_request.rb" + "updater/lib/dependabot/updater/operations/refresh_security_update_pull_request.rb" + "updater/lib/dependabot/updater/operations/refresh_version_update_pull_request.rb" + "updater/lib/dependabot/updater/operations/update_all_versions.rb" + "updater/lib/dependabot/updater/dependency_group_change_batch.rb" + "updater/lib/dependabot/updater/error_handler.rb" + "updater/lib/dependabot/updater/errors.rb" + "updater/lib/dependabot/updater/group_update_creation.rb" + "updater/lib/dependabot/updater/operations.rb" + "updater/lib/dependabot/updater/security_update_helpers.rb" + "updater/lib/dependabot/api_client.rb" + "updater/lib/dependabot/base_command.rb" + "updater/lib/dependabot/dependency_change.rb" + "updater/lib/dependabot/dependency_change_builder.rb" + "updater/lib/dependabot/dependency_group_engine.rb" + "updater/lib/dependabot/dependency_snapshot.rb" + "updater/lib/dependabot/environment.rb" + "updater/lib/dependabot/file_fetcher_command.rb" + "updater/lib/dependabot/job.rb" + "updater/lib/dependabot/sentry.rb" + "updater/lib/dependabot/service.rb" + "updater/lib/dependabot/setup.rb" + "updater/lib/dependabot/update_files_command.rb" + "updater/lib/dependabot/updater.rb" + + # "updater/spec/dependabot/updater/operations/group_update_all_versions_spec.rb" + # "updater/spec/dependabot/updater/operations/refresh_group_update_pull_request_spec.rb" + "updater/spec/dependabot/updater/error_handler_spec.rb" + "updater/spec/dependabot/updater/operations_spec.rb" + "updater/spec/dependabot/api_client_spec.rb" + # "updater/spec/dependabot/dependency_change_builder_spec.rb" + "updater/spec/dependabot/dependency_change_spec.rb" + "updater/spec/dependabot/dependency_group_engine_spec.rb" + # "updater/spec/dependabot/dependency_snapshot_spec.rb" + "updater/spec/dependabot/environment_spec.rb" + # "updater/spec/dependabot/file_fetcher_command_spec.rb" + # "updater/spec/dependabot/integration_spec.rb" + "updater/spec/dependabot/job_spec.rb" + "updater/spec/dependabot/sentry_spec.rb" + "updater/spec/dependabot/service_spec.rb" + # "updater/spec/dependabot/update_files_command_spec.rb" + # "updater/spec/dependabot/updater_spec.rb" + + "updater/spec/fixtures/rubygems-index" + "updater/spec/fixtures/rubygems-info-a" + "updater/spec/fixtures/rubygems-versions-a.json" + "updater/spec/fixtures/rubygems-info-b" + "updater/spec/fixtures/rubygems-versions-b.json" + "updater/spec/fixtures/bundler/original/Gemfile" + "updater/spec/fixtures/bundler/original/Gemfile.lock" + "updater/spec/fixtures/bundler/updated/Gemfile" + "updater/spec/fixtures/bundler/updated/Gemfile.lock" + "updater/spec/fixtures/bundler_gemspec/original/Gemfile" + "updater/spec/fixtures/bundler_gemspec/original/Gemfile.lock" + "updater/spec/fixtures/bundler_gemspec/original/library.gemspec" + "updater/spec/fixtures/bundler_git/original/Gemfile" + "updater/spec/fixtures/bundler_git/original/Gemfile.lock" + "updater/spec/fixtures/bundler_grouped_by_types/original/Gemfile" + "updater/spec/fixtures/bundler_grouped_by_types/original/Gemfile.lock" + "updater/spec/fixtures/bundler_vendored/original/Gemfile" + "updater/spec/fixtures/bundler_vendored/original/Gemfile.lock" + "updater/spec/fixtures/docker/original/Dockerfile.bundler" + "updater/spec/fixtures/docker/original/Dockerfile.cargo" + "updater/spec/fixtures/jobs/job_with_credentials.json" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_by_dependency_type.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_empty_group.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_overlapping_groups.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_existing_pr.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_ungrouped.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_vendoring.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_semver_grouping_with_global_ignores.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_semver_grouping.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_peer_manifests.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_dependencies_changed.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_empty_group.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_missing_group.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_versions_changed.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_similar_pr.yaml" + "updater/spec/fixtures/job_definitions/bundler/version_updates/update_all_simple.yaml" + "updater/spec/fixtures/job_definitions/docker/version_updates/group_update_peer_manifests.yaml" + "updater/spec/fixtures/job_definitions/README.md" + + "updater/spec/support/dependency_file_helpers.rb" + "updater/spec/support/dummy_pkg_helpers.rb" + + # "updater/spec/spec_helper.rb" +) $baseUrl = "https://raw.githubusercontent.com/dependabot/dependabot-core" -$destinationFolder = Join-Path -Path '.' -ChildPath 'updater' - -foreach ($h in $hash.GetEnumerator()) { - $sourceUrl = "$baseUrl/$tag/$($h.Name)" - $destinationPath = Join-Path -Path "$destinationFolder" -ChildPath "$($h.Value)" - Write-Host "`Downloading $($h.Name) ..." - [System.IO.Directory]::CreateDirectory("$(Split-Path -Path "$destinationPath")") | Out-Null - Invoke-WebRequest -Uri $sourceUrl -OutFile $destinationPath + +foreach ($name in $files) { + $sourceUrl = "$baseUrl/$tag/$($name)" + $destinationPath = Join-Path -Path '.' -ChildPath "$name" + + # Write-Host "`Downloading $name ..." + # [System.IO.Directory]::CreateDirectory("$(Split-Path -Path "$destinationPath")") | Out-Null + # Invoke-WebRequest -Uri $sourceUrl -OutFile $destinationPath + + echo "Downloading $($name) ..." + mkdir -p "$(dirname "$destinationPath")" + curl -sL "$sourceUrl" -o "$destinationPath" } diff --git a/updater/Gemfile b/updater/Gemfile index 817eac2de..3e1d16149 100644 --- a/updater/Gemfile +++ b/updater/Gemfile @@ -11,6 +11,11 @@ source "https://rubygems.org" # gem "dependabot-omnibus", github: "dependabot/dependabot-core", branch: "main" gem "dependabot-omnibus", "~>0.232.0" +gem "http", "~> 5.1" +gem "octokit", "6.1.1" +gem "sentry-raven", "~> 3.1" +gem "terminal-table", "~> 3.0.2" + group :test do gem "rspec" gem "rubocop" diff --git a/updater/Gemfile.lock b/updater/Gemfile.lock index b38c71c89..38faaf246 100644 --- a/updater/Gemfile.lock +++ b/updater/Gemfile.lock @@ -107,19 +107,32 @@ GEM faraday-net_http (3.0.2) faraday-retry (2.2.0) faraday (~> 2.0) + ffi (1.15.5) + ffi-compiler (1.0.1) + ffi (>= 1.0.0) + rake gitlab (4.19.0) httparty (~> 0.20) terminal-table (>= 1.5.1) hashdiff (1.0.1) + http (5.1.1) + addressable (~> 2.8) + http-cookie (~> 1.0) + http-form_data (~> 2.2) + llhttp-ffi (~> 0.4.0) http-accept (1.7.0) http-cookie (1.0.5) domain_name (~> 0.5) + http-form_data (2.3.0) httparty (0.21.0) mini_mime (>= 1.0.0) multi_xml (>= 0.5.2) jmespath (1.6.2) json (2.6.3) language_server-protocol (3.17.0.3) + llhttp-ffi (0.4.0) + ffi-compiler (~> 1.0) + rake (~> 13.0) mime-types (3.5.1) mime-types-data (~> 3.2015) mime-types-data (3.2023.0808) @@ -143,6 +156,7 @@ GEM public_suffix (5.0.3) racc (1.7.1) rainbow (3.1.1) + rake (13.0.6) regexp_parser (2.8.1) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) @@ -185,6 +199,8 @@ GEM sawyer (0.9.2) addressable (>= 2.3.5) faraday (>= 0.17.3, < 3) + sentry-raven (3.1.2) + faraday (>= 1.0) sorbet-runtime (0.5.11014) stringio (3.0.8) terminal-table (3.0.2) @@ -207,9 +223,13 @@ PLATFORMS DEPENDENCIES dependabot-omnibus (~> 0.232.0) + http (~> 5.1) + octokit (= 6.1.1) rspec rubocop rubocop-performance + sentry-raven (~> 3.1) + terminal-table (~> 3.0.2) vcr webmock diff --git a/updater/bin/fetch_files.rb b/updater/bin/fetch_files.rb new file mode 100644 index 000000000..89d7f1c8a --- /dev/null +++ b/updater/bin/fetch_files.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift(__dir__ + "/../lib") + +$stdout.sync = true + +require "raven" +require "dependabot/setup" +require "dependabot/file_fetcher_command" +require "debug" if ENV["DEBUG"] + +class UpdaterKilledError < StandardError; end + +trap("TERM") do + puts "Received SIGTERM" + error = UpdaterKilledError.new("Updater process killed with SIGTERM") + tags = { update_job_id: ENV.fetch("DEPENDABOT_JOB_ID", nil) } + Raven.capture_exception(error, tags: tags) + exit +end + +begin + Dependabot::FileFetcherCommand.new.run +rescue Dependabot::RunFailure + exit 1 +end diff --git a/updater/bin/update_files.rb b/updater/bin/update_files.rb new file mode 100644 index 000000000..fd9c899e1 --- /dev/null +++ b/updater/bin/update_files.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift(__dir__ + "/../lib") + +$stdout.sync = true + +require "raven" +require "dependabot/setup" +require "dependabot/update_files_command" +require "debug" if ENV["DEBUG"] + +class UpdaterKilledError < StandardError; end + +trap("TERM") do + puts "Received SIGTERM" + error = UpdaterKilledError.new("Updater process killed with SIGTERM") + tags = { update_job_id: ENV.fetch("DEPENDABOT_JOB_ID", nil) } + Raven.capture_exception(error, tags: tags) + exit +end + +begin + Dependabot::UpdateFilesCommand.new.run +rescue Dependabot::RunFailure + exit 1 +end diff --git a/updater/config/.npmrc b/updater/config/.npmrc new file mode 100644 index 000000000..601d250c1 --- /dev/null +++ b/updater/config/.npmrc @@ -0,0 +1,20 @@ +# TODO: Remove these hacks once we've deprecated npm 6 support as it no longer +# spwans a child process to npm install git dependencies. + +# Only set our custom CA cert for npm because the system ca's + our custom ca +# causes npm to blow up when installing git dependencies (E2BIG exception). This +# happens because the ca-file contents are passed as a cli argument to npm +# install from npm/cli/lib/pack.js as --ca="contents of ca file" - "ca" is +# populated automatically by npm when setting "--cafile" and passed through in +# when spawning the cli to install git dependencies. +cafile=/usr/local/share/ca-certificates/dbot-ca.crt +# Because npm doesn't pass through all npm config when doing git installs in +# npm/cli/lib/pack.js we also need to disable audit here to prevent npm from +# auditing git dependencies, we do this to sped up installs +audit=false +# Similarly, dry-run and ignore-scripts are also not passed through when doing +# git installs in npm/cli/lib/pack.js so we set dry-run and ignore-scripts to +# prevent any lifecycle hooks for git installs. dry-run disables "prepare" and +# "prepack" scripts, ignore-scripts disables all other scripts +dry-run=true +ignore-scripts=true diff --git a/updater/config/.yarnrc b/updater/config/.yarnrc new file mode 100644 index 000000000..65545853f --- /dev/null +++ b/updater/config/.yarnrc @@ -0,0 +1,6 @@ +# TODO: Remove these hacks once we've deprecated npm 6 support as it no longer +# spwans a child process to npm install git dependencies. +# yarn lockfile v1 + +# Tell yarn to use the system-wide CA bundle overriding the .npmrc cafile +cafile "/etc/ssl/certs/ca-certificates.crt" diff --git a/updater/lib/dependabot/api_client.rb b/updater/lib/dependabot/api_client.rb new file mode 100644 index 000000000..8bfcb6739 --- /dev/null +++ b/updater/lib/dependabot/api_client.rb @@ -0,0 +1,200 @@ +# frozen_string_literal: true + +require "http" +require "dependabot/job" + +# Provides a client to access the internal Dependabot Service's API +# +# The Service acts as a relay to Core's GitHub API adapters while providing +# some co-ordination and enrichment functionality that is only relevant to +# the integrated service. +# +# This API is only available to Dependabot jobs being executed within our +# hosted infrastructure and is not open to integrators at this time. +# +module Dependabot + class ApiError < StandardError; end + + class ApiClient + def initialize(base_url, job_id, job_token) + @base_url = base_url + @job_id = job_id + @job_token = job_token + end + + # TODO: Make `base_commit_sha` part of Dependabot::DependencyChange + def create_pull_request(dependency_change, base_commit_sha) + api_url = "#{base_url}/update_jobs/#{job_id}/create_pull_request" + data = create_pull_request_data(dependency_change, base_commit_sha) + response = http_client.post(api_url, json: { data: data }) + raise ApiError, response.body if response.code >= 400 + rescue HTTP::ConnectionError, OpenSSL::SSL::SSLError + retry_count ||= 0 + retry_count += 1 + raise if retry_count > 3 + + sleep(rand(3.0..10.0)) && retry + end + + # TODO: Make `base_commit_sha` part of Dependabot::DependencyChange + # TODO: Determine if we should regenerate the PR message within core for updates + def update_pull_request(dependency_change, base_commit_sha) + api_url = "#{base_url}/update_jobs/#{job_id}/update_pull_request" + body = { + data: { + "dependency-names": dependency_change.updated_dependencies.map(&:name), + "updated-dependency-files": dependency_change.updated_dependency_files_hash, + "base-commit-sha": base_commit_sha + } + } + response = http_client.post(api_url, json: body) + raise ApiError, response.body if response.code >= 400 + rescue HTTP::ConnectionError, OpenSSL::SSL::SSLError + retry_count ||= 0 + retry_count += 1 + raise if retry_count > 3 + + sleep(rand(3.0..10.0)) && retry + end + + def close_pull_request(dependency_name, reason) + api_url = "#{base_url}/update_jobs/#{job_id}/close_pull_request" + body = { data: { "dependency-names": dependency_name, reason: reason } } + response = http_client.post(api_url, json: body) + raise ApiError, response.body if response.code >= 400 + rescue HTTP::ConnectionError, OpenSSL::SSL::SSLError + retry_count ||= 0 + retry_count += 1 + raise if retry_count > 3 + + sleep(rand(3.0..10.0)) && retry + end + + def record_update_job_error(error_type:, error_details:) + api_url = "#{base_url}/update_jobs/#{job_id}/record_update_job_error" + body = { + data: { + "error-type": error_type, + "error-details": error_details + } + } + response = http_client.post(api_url, json: body) + raise ApiError, response.body if response.code >= 400 + rescue HTTP::ConnectionError, OpenSSL::SSL::SSLError + retry_count ||= 0 + retry_count += 1 + raise if retry_count > 3 + + sleep(rand(3.0..10.0)) && retry + end + + def mark_job_as_processed(base_commit_sha) + api_url = "#{base_url}/update_jobs/#{job_id}/mark_as_processed" + body = { data: { "base-commit-sha": base_commit_sha } } + response = http_client.patch(api_url, json: body) + raise ApiError, response.body if response.code >= 400 + rescue HTTP::ConnectionError, OpenSSL::SSL::SSLError + retry_count ||= 0 + retry_count += 1 + raise if retry_count > 3 + + sleep(rand(3.0..10.0)) && retry + end + + def update_dependency_list(dependencies, dependency_files) + api_url = "#{base_url}/update_jobs/#{job_id}/update_dependency_list" + body = { + data: { + dependencies: dependencies, + dependency_files: dependency_files + } + } + response = http_client.post(api_url, json: body) + raise ApiError, response.body if response.code >= 400 + rescue HTTP::ConnectionError, OpenSSL::SSL::SSLError + retry_count ||= 0 + retry_count += 1 + raise if retry_count > 3 + + sleep(rand(3.0..10.0)) && retry + end + + def record_ecosystem_versions(ecosystem_versions) + api_url = "#{base_url}/update_jobs/#{job_id}/record_ecosystem_versions" + body = { + data: { ecosystem_versions: ecosystem_versions } + } + response = http_client.post(api_url, json: body) + raise ApiError, response.body if response.code >= 400 + rescue HTTP::ConnectionError, OpenSSL::SSL::SSLError + retry_count ||= 0 + retry_count += 1 + raise if retry_count > 3 + + sleep(rand(3.0..10.0)) && retry + end + + def increment_metric(metric, tags:) + api_url = "#{base_url}/update_jobs/#{job_id}/increment_metric" + body = { + data: { + metric: metric, + tags: tags + } + } + response = http_client.post(api_url, json: body) + # We treat metrics as fire-and-forget, so just warn if they fail. + Dependabot.logger.debug("Unable to report metric '#{metric}'.") if response.code >= 400 + rescue HTTP::ConnectionError, OpenSSL::SSL::SSLError + Dependabot.logger.debug("Unable to report metric '#{metric}'.") + end + + private + + attr_reader :base_url, :job_id, :job_token + + def http_client + client = HTTP.auth(job_token) + proxy = ENV["HTTPS_PROXY"] ? URI(ENV["HTTPS_PROXY"]) : URI(base_url).find_proxy + unless proxy.nil? + args = [proxy.host, proxy.port, proxy.user, proxy.password].compact + client = client.via(*args) + end + client + end + + def dependency_group_hash(dependency_change) + return {} unless dependency_change.grouped_update? + + # FIXME: We currently assumpt that _an attempt_ to send a DependencyGroup#id should + # result in the `grouped-update` flag being set, regardless of whether the + # DependencyGroup actually exists. + { "dependency-group": dependency_change.dependency_group.to_h }.compact + end + + def create_pull_request_data(dependency_change, base_commit_sha) + data = { + dependencies: dependency_change.updated_dependencies.map do |dep| + { + name: dep.name, + "previous-version": dep.previous_version, + requirements: dep.requirements, + "previous-requirements": dep.previous_requirements + }.merge({ + version: dep.version, + removed: dep.removed? ? true : nil + }.compact) + end, + "updated-dependency-files": dependency_change.updated_dependency_files_hash, + "base-commit-sha": base_commit_sha + }.merge(dependency_group_hash(dependency_change)) + + return data unless dependency_change.pr_message + + data["commit-message"] = dependency_change.pr_message.commit_message + data["pr-title"] = dependency_change.pr_message.pr_name + data["pr-body"] = dependency_change.pr_message.pr_message + data + end + end +end diff --git a/updater/lib/dependabot/base_command.rb b/updater/lib/dependabot/base_command.rb new file mode 100644 index 000000000..7222c2513 --- /dev/null +++ b/updater/lib/dependabot/base_command.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +require "raven" +require "dependabot/api_client" +require "dependabot/service" +require "dependabot/logger" +require "dependabot/logger/formats" +require "dependabot/python" +require "dependabot/terraform" +require "dependabot/elm" +require "dependabot/docker" +require "dependabot/git_submodules" +require "dependabot/github_actions" +require "dependabot/composer" +require "dependabot/nuget" +require "dependabot/gradle" +require "dependabot/maven" +require "dependabot/hex" +require "dependabot/cargo" +require "dependabot/go_modules" +require "dependabot/npm_and_yarn" +require "dependabot/bundler" +require "dependabot/pub" +require "dependabot/environment" + +module Dependabot + class RunFailure < StandardError; end + + class BaseCommand + # Implement in subclass + def perform_job + raise NotImplementedError + end + + # Implement in subclass + def job + raise NotImplementedError + end + + # Implement in subclass + def base_commit_sha + raise NotImplementedError + end + + # TODO: Avoid rescuing StandardError at this point in the code + # + # This means that exceptions in tests can occasionally be swallowed + # and we must rely on reading RSpec output to detect certain problems. + def run + Dependabot.logger.formatter = Dependabot::Logger::JobFormatter.new(job_id) + Dependabot.logger.info("Starting job processing") + perform_job + Dependabot.logger.info("Finished job processing") + rescue StandardError => e + handle_exception(e) + service.mark_job_as_processed(base_commit_sha) + ensure + Dependabot.logger.formatter = Dependabot::Logger::BasicFormatter.new + Dependabot.logger.info(service.summary) unless service.noop? + raise Dependabot::RunFailure if Dependabot::Environment.github_actions? && service.failure? + end + + def handle_exception(err) + Dependabot.logger.error(err.message) + err.backtrace.each { |line| Dependabot.logger.error(line) } + + service.capture_exception(error: err, job: job) + service.record_update_job_error(error_type: "unknown_error", error_details: { message: err.message }) + end + + def job_id + Environment.job_id + end + + def api_client + @api_client ||= Dependabot::ApiClient.new( + Environment.api_url, + job_id, + Environment.job_token + ) + end + + def service + @service ||= Dependabot::Service.new(client: api_client) + end + end +end diff --git a/updater/lib/dependabot/dependency_change.rb b/updater/lib/dependabot/dependency_change.rb new file mode 100644 index 000000000..262b22cde --- /dev/null +++ b/updater/lib/dependabot/dependency_change.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +# This class describes a change to the project's Dependencies which has been +# determined by a Dependabot operation. +# +# It includes a list of changed Dependabot::Dependency objects, an array of +# Dependabot::DependencyFile objects which contain the changes to be applied +# along with any Dependabot::DependencyGroup that was used to generate the change. +# +# This class provides methods for presenting the change set which can be used +# by adapters to create a Pull Request, apply the changes on disk, etc. +module Dependabot + class DependencyChange + attr_reader :job, :updated_dependencies, :updated_dependency_files, :dependency_group + + def initialize(job:, updated_dependencies:, updated_dependency_files:, dependency_group: nil) + @job = job + @updated_dependencies = updated_dependencies + @updated_dependency_files = updated_dependency_files + @dependency_group = dependency_group + end + + def pr_message + return @pr_message if defined?(@pr_message) + + case job.source&.provider + when "github" + pr_message_max_length = Dependabot::PullRequestCreator::Github::PR_DESCRIPTION_MAX_LENGTH + when "azure" + pr_message_max_length = Dependabot::PullRequestCreator::Azure::PR_DESCRIPTION_MAX_LENGTH + pr_message_encoding = Dependabot::PullRequestCreator::Azure::PR_DESCRIPTION_ENCODING + when "codecommit" + pr_message_max_length = Dependabot::PullRequestCreator::Codecommit::PR_DESCRIPTION_MAX_LENGTH + when "bitbucket" + pr_message_max_length = Dependabot::PullRequestCreator::Bitbucket::PR_DESCRIPTION_MAX_LENGTH + else + pr_message_max_length = Dependabot::PullRequestCreator::Github::PR_DESCRIPTION_MAX_LENGTH + end + + @pr_message = Dependabot::PullRequestCreator::MessageBuilder.new( + source: job.source, + dependencies: updated_dependencies, + files: updated_dependency_files, + credentials: job.credentials, + commit_message_options: job.commit_message_options, + dependency_group: dependency_group, + pr_message_max_length: pr_message_max_length, + pr_message_encoding: pr_message_encoding, + ignore_conditions: job.ignore_conditions + ).message + end + + def humanized + updated_dependencies.map do |dependency| + "#{dependency.name} ( from #{dependency.humanized_previous_version} to #{dependency.humanized_version} )" + end.join(", ") + end + + def updated_dependency_files_hash + updated_dependency_files.map(&:to_h) + end + + def grouped_update? + !!dependency_group + end + + # This method combines checking the job's `updating_a_pull_request` flag + # with verification the dependencies involved remain the same. + # + # If the dependencies involved have changed, we should close the old PR + # rather than supersede it as the new changes don't necessarily follow + # from the previous ones; dependencies could have been removed from the + # project, or pinned by other changes. + def should_replace_existing_pr? + return false unless job.updating_a_pull_request? + + # NOTE: Gradle, Maven and Nuget dependency names can be case-insensitive + # and the dependency name injected from a security advisory often doesn't + # match what users have specified in their manifest. + updated_dependencies.map { |x| x.name.downcase } != job.dependencies.map(&:downcase) + end + + def matches_existing_pr? + !!existing_pull_request + end + + private + + def existing_pull_request + if grouped_update? + # We only want PRs for the same group that have the same versions + job.existing_group_pull_requests.find do |pr| + pr["dependency-group-name"] == dependency_group.name && + Set.new(pr["dependencies"]) == updated_dependencies_set + end + else + job.existing_pull_requests.find { |pr| Set.new(pr) == updated_dependencies_set } + end + end + + def updated_dependencies_set + Set.new( + updated_dependencies.map do |dep| + { + "dependency-name" => dep.name, + "dependency-version" => dep.version, + "dependency-removed" => dep.removed? ? true : nil + }.compact + end + ) + end + end +end diff --git a/updater/lib/dependabot/dependency_change_builder.rb b/updater/lib/dependabot/dependency_change_builder.rb new file mode 100644 index 000000000..fba37abe4 --- /dev/null +++ b/updater/lib/dependabot/dependency_change_builder.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require "dependabot/dependency" +require "dependabot/dependency_change" +require "dependabot/file_updaters" +require "dependabot/dependency_group" + +# This class is responsible for generating a DependencyChange for a given +# set of dependencies and dependency files. +# +# This class should be used via the `create_from` method with the following +# arguments: +# - job: +# The Dependabot::Job object the change is originated by +# - dependency_files: +# The list dependency files we aim to modify as part of this change +# - updated_dependencies: +# The set of dependency updates to be applied to the dependency files +# - change_source: +# A change can be generated from either a single 'lead' Dependency or +# a DependencyGroup +module Dependabot + class DependencyChangeBuilder + def self.create_from(**kwargs) + new(**kwargs).run + end + + def initialize(job:, dependency_files:, updated_dependencies:, change_source:) + @job = job + @dependency_files = dependency_files + @updated_dependencies = updated_dependencies + @change_source = change_source + end + + def run + updated_files = generate_dependency_files + # Remove any unchanged dependencies from the updated list + updated_deps = updated_dependencies.reject do |d| + # Avoid rejecting the source dependency + next false if source_dependency_name && d.name == source_dependency_name + + next false if d.top_level? && d.requirements != d.previous_requirements + + d.version == d.previous_version + end + + Dependabot::DependencyChange.new( + job: job, + updated_dependencies: updated_deps, + updated_dependency_files: updated_files, + dependency_group: source_dependency_group + ) + end + + private + + attr_reader :job, :dependency_files, :updated_dependencies, :change_source + + def source_dependency_name + return nil unless change_source.is_a? Dependabot::Dependency + + change_source.name + end + + def source_dependency_group + return nil unless change_source.is_a? Dependabot::DependencyGroup + + change_source + end + + def generate_dependency_files + if updated_dependencies.count == 1 + updated_dependency = updated_dependencies.first + Dependabot.logger.info("Updating #{updated_dependency.name} from " \ + "#{updated_dependency.previous_version} to " \ + "#{updated_dependency.version}") + else + dependency_names = updated_dependencies.map(&:name) + Dependabot.logger.info("Updating #{dependency_names.join(', ')}") + end + + # Ignore dependencies that are tagged as information_only. These will be + # updated indirectly as a result of a parent dependency update and are + # only included here to be included in the PR info. + relevant_dependencies = updated_dependencies.reject(&:informational_only?) + file_updater_for(relevant_dependencies).updated_dependency_files + end + + def file_updater_for(dependencies) + Dependabot::FileUpdaters.for_package_manager(job.package_manager).new( + dependencies: dependencies, + dependency_files: dependency_files, + repo_contents_path: job.repo_contents_path, + credentials: job.credentials, + options: job.experiments + ) + end + end +end diff --git a/updater/lib/dependabot/dependency_group_engine.rb b/updater/lib/dependabot/dependency_group_engine.rb new file mode 100644 index 000000000..15aab090b --- /dev/null +++ b/updater/lib/dependabot/dependency_group_engine.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require "dependabot/dependency_group" + +# This class implements our strategy for keeping track of and matching dependency +# groups that are defined by users in their dependabot config file. +# +# We instantiate the DependencyGroupEngine after parsing dependencies, configuring +# any groups from the job's configuration before assigning the dependency list to +# the groups. +# +# We permit dependencies to be in more than one group and also track those which +# have zero matches so they may be updated individuall. +# +# **Note:** This is currently an experimental feature which is not supported +# in the service or as an integration point. +# +module Dependabot + class DependencyGroupEngine + class ConfigurationError < StandardError; end + + def self.from_job_config(job:) + groups = job.dependency_groups.map do |group| + Dependabot::DependencyGroup.new(name: group["name"], rules: group["rules"]) + end + + new(dependency_groups: groups) + end + + attr_reader :dependency_groups, :groups_calculated, :ungrouped_dependencies + + def find_group(name:) + dependency_groups.find { |group| group.name == name } + end + + def assign_to_groups!(dependencies:) + raise ConfigurationError, "dependency groups have already been configured!" if @groups_calculated + + if dependency_groups.any? + dependencies.each do |dependency| + matched_groups = @dependency_groups.each_with_object([]) do |group, matches| + next unless group.contains?(dependency) + + group.dependencies.push(dependency) + matches << group + end + + # If we had no matches, collect the dependency as ungrouped + @ungrouped_dependencies << dependency if matched_groups.empty? + end + else + @ungrouped_dependencies = dependencies + end + + validate_groups + @groups_calculated = true + end + + private + + def initialize(dependency_groups:) + @dependency_groups = dependency_groups + @ungrouped_dependencies = [] + @groups_calculated = false + end + + def validate_groups + empty_groups = dependency_groups.select { |group| group.dependencies.empty? } + warn_misconfigured_groups(empty_groups) if empty_groups.any? + end + + def warn_misconfigured_groups(groups) + Dependabot.logger.warn <<~WARN + Please check your configuration as there are groups where no dependencies match: + #{groups.map { |g| "- #{g.name}" }.join("\n")} + + This can happen if: + - the group's 'pattern' rules are mispelled + - your configuration's 'allow' rules do not permit any of the dependencies that match the group + - the dependencies that match the group rules have been removed from your project + WARN + end + end +end diff --git a/updater/lib/dependabot/dependency_snapshot.rb b/updater/lib/dependabot/dependency_snapshot.rb new file mode 100644 index 000000000..2331f65a9 --- /dev/null +++ b/updater/lib/dependabot/dependency_snapshot.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require "base64" +require "dependabot/file_parsers" + +# This class describes the dependencies obtained from a project at a specific commit SHA +# including both the Dependabot::DependencyFile objects at that reference as well as +# means to parse them into a set of Dependabot::Dependency objects. +# +# This class is the input for a Dependabot::Updater process with Dependabot::DependencyChange +# representing the output. +module Dependabot + class DependencySnapshot + def self.create_from_job_definition(job:, job_definition:) + decoded_dependency_files = job_definition.fetch("base64_dependency_files").map do |a| + file = Dependabot::DependencyFile.new(**a.transform_keys(&:to_sym)) + file.content = Base64.decode64(file.content).force_encoding("utf-8") unless file.binary? && !file.deleted? + file + end + + new( + job: job, + base_commit_sha: job_definition.fetch("base_commit_sha"), + dependency_files: decoded_dependency_files + ) + end + + attr_reader :base_commit_sha, :dependency_files, :dependencies + + # Returns the subset of all project dependencies which are permitted + # by the project configuration. + def allowed_dependencies + @allowed_dependencies ||= dependencies.select { |d| job.allowed_update?(d) } + end + + # Returns the subset of all project dependencies which are specifically + # requested to be updated by the job definition. + def job_dependencies + return [] unless job.dependencies&.any? + return @job_dependencies if defined? @job_dependencies + + # Gradle, Maven and Nuget dependency names can be case-insensitive and + # the dependency name in the security advisory often doesn't match what + # users have specified in their manifest. + # + # It's technically possibly to publish case-sensitive npm packages to a + # private registry but shouldn't cause problems here as job.dependencies + # is set either from an existing PR rebase/recreate or a security + # advisory. + job_dependency_names = job.dependencies.map(&:downcase) + @job_dependencies = dependencies.select do |dep| + job_dependency_names.include?(dep.name.downcase) + end + end + + # Returns just the group that is specifically requested to be updated by + # the job definition + def job_group + return nil unless Dependabot::Experiments.enabled?(:grouped_updates_prototype) + return nil unless job.dependency_group_to_refresh + return @job_group if defined?(@job_group) + + @job_group = @dependency_group_engine.find_group(name: job.dependency_group_to_refresh) + end + + def groups + return [] unless Dependabot::Experiments.enabled?(:grouped_updates_prototype) + + @dependency_group_engine.dependency_groups + end + + def ungrouped_dependencies + # If no groups are defined, all dependencies are ungrouped by default. + return allowed_dependencies unless groups.any? + + # Otherwise return dependencies that haven't been handled during the group update portion. + all_handled_dependencies = Set.new(groups.map { |g| g.handled_dependencies.to_a }.flatten) + allowed_dependencies.reject { |dep| all_handled_dependencies.include?(dep.name) } + end + + private + + def initialize(job:, base_commit_sha:, dependency_files:) + @job = job + @base_commit_sha = base_commit_sha + @dependency_files = dependency_files + + @dependencies = parse_files! + + return unless Dependabot::Experiments.enabled?(:grouped_updates_prototype) + + @dependency_group_engine = DependencyGroupEngine.from_job_config(job: job) + @dependency_group_engine.assign_to_groups!(dependencies: allowed_dependencies) + end + + attr_reader :job + + def parse_files! + dependency_file_parser.parse + end + + def dependency_file_parser + Dependabot::FileParsers.for_package_manager(job.package_manager).new( + dependency_files: dependency_files, + repo_contents_path: job.repo_contents_path, + source: job.source, + credentials: job.credentials, + reject_external_code: job.reject_external_code?, + options: job.experiments + ) + end + end +end diff --git a/updater/lib/dependabot/file_fetcher_command.rb b/updater/lib/dependabot/file_fetcher_command.rb new file mode 100644 index 000000000..6678e4512 --- /dev/null +++ b/updater/lib/dependabot/file_fetcher_command.rb @@ -0,0 +1,222 @@ +# frozen_string_literal: true + +require "base64" +require "dependabot/base_command" +require "dependabot/updater" +require "octokit" + +module Dependabot + class FileFetcherCommand < BaseCommand + # BaseCommand does not implement this method, so we should expose + # the instance variable for error handling to avoid raising a + # NotImplementedError if it is referenced + attr_reader :base_commit_sha + + def perform_job + @base_commit_sha = nil + + begin + connectivity_check if ENV["ENABLE_CONNECTIVITY_CHECK"] == "1" + clone_repo_contents + @base_commit_sha = file_fetcher.commit + raise "base commit SHA not found" unless @base_commit_sha + + # We don't set this flag in GHES because there's no point in recording versions since we can't access that data. + if Experiments.enabled?(:record_ecosystem_versions) + ecosystem_versions = file_fetcher.ecosystem_versions + api_client.record_ecosystem_versions(ecosystem_versions) unless ecosystem_versions.nil? + end + + dependency_files + rescue StandardError => e + @base_commit_sha ||= "unknown" + if Octokit::RATE_LIMITED_ERRORS.include?(e.class) + remaining = rate_limit_error_remaining(e) + Dependabot.logger.error("Repository is rate limited, attempting to retry in " \ + "#{remaining}s") + else + Dependabot.logger.error("Error during file fetching; aborting") + end + handle_file_fetcher_error(e) + service.mark_job_as_processed(@base_commit_sha) + return + end + + File.write(Environment.output_path, JSON.dump( + base64_dependency_files: base64_dependency_files.map(&:to_h), + base_commit_sha: @base_commit_sha + )) + + save_job_details + end + + private + + def save_job_details + # TODO: Use the Dependabot::Environment helper for this + return unless ENV["UPDATER_ONE_CONTAINER"] + + File.write(Environment.job_path, JSON.dump( + base64_dependency_files: base64_dependency_files.map(&:to_h), + base_commit_sha: @base_commit_sha, + job: Environment.job_definition["job"] + )) + end + + def dependency_files + file_fetcher.files + rescue Octokit::BadGateway + @file_fetcher_retries ||= 0 + @file_fetcher_retries += 1 + @file_fetcher_retries <= 2 ? retry : raise + end + + def clone_repo_contents + return unless job.clone? + + file_fetcher.clone_repo_contents + end + + def base64_dependency_files + dependency_files.map do |file| + base64_file = file.dup + base64_file.content = Base64.encode64(file.content) unless file.binary? + base64_file + end + end + + def job + @job ||= Job.new_fetch_job( + job_id: job_id, + job_definition: Environment.job_definition, + repo_contents_path: Environment.repo_contents_path + ) + end + + def file_fetcher + return @file_fetcher if defined? @file_fetcher + + args = { + source: job.source, + credentials: Environment.job_definition.fetch("credentials", []), + options: job.experiments + } + # This bypasses the `job.repo_contents_path` presenter to ensure we fetch + # from the file system if the repository contents are mounted even if + # cloning is disabled. + args[:repo_contents_path] = Environment.repo_contents_path if job.clone? || already_cloned? + @file_fetcher ||= Dependabot::FileFetchers.for_package_manager(job.package_manager).new(**args) + end + + def already_cloned? + return false unless Environment.repo_contents_path + + # For testing, the source repo may already be mounted. + @already_cloned ||= File.directory?(File.join(Environment.repo_contents_path, ".git")) + end + + # rubocop:disable Metrics/MethodLength + def handle_file_fetcher_error(error) + error_details = + case error + when Dependabot::BranchNotFound + { + "error-type": "branch_not_found", + "error-detail": { "branch-name": error.branch_name } + } + when Dependabot::RepoNotFound + # This happens if the repo gets removed after a job gets kicked off. + # This also happens when a configured personal access token is not authz'd to fetch files from the job repo. + { + "error-type": "job_repo_not_found", + "error-detail": {} + } + when Dependabot::DependencyFileNotParseable + { + "error-type": "dependency_file_not_parseable", + "error-detail": { + message: error.message, + "file-path": error.file_path + } + } + when Dependabot::DependencyFileNotFound + { + "error-type": "dependency_file_not_found", + "error-detail": { "file-path": error.file_path } + } + when Dependabot::OutOfDisk + { + "error-type": "out_of_disk", + "error-detail": {} + } + when Dependabot::PathDependenciesNotReachable + { + "error-type": "path_dependencies_not_reachable", + "error-detail": { dependencies: error.dependencies } + } + when Octokit::Unauthorized + { "error-type": "octokit_unauthorized" } + when Octokit::ServerError + # If we get a 500 from GitHub there's very little we can do about it, + # and responsibility for fixing it is on them, not us. As a result we + # quietly log these as errors + { "error-type": "unknown_error" } + when *Octokit::RATE_LIMITED_ERRORS + # If we get a rate-limited error we let dependabot-api handle the + # retry by re-enqueing the update job after the reset + { + "error-type": "octokit_rate_limited", + "error-detail": { + "rate-limit-reset": error.response_headers["X-RateLimit-Reset"] + } + } + else + Dependabot.logger.error(error.message) + error.backtrace.each { |line| Dependabot.logger.error line } + + service.capture_exception(error: error, job: job) + { "error-type": "unknown_error" } + end + + record_error(error_details) if error_details + end + + # rubocop:enable Metrics/MethodLength + def rate_limit_error_remaining(error) + # Time at which the current rate limit window resets in UTC epoch secs. + expires_at = error.response_headers["X-RateLimit-Reset"].to_i + remaining = Time.at(expires_at) - Time.now + remaining.positive? ? remaining : 0 + end + + def record_error(error_details) + service.record_update_job_error( + error_type: error_details.fetch(:"error-type"), + error_details: error_details[:"error-detail"] + ) + end + + # Perform a debug check of connectivity to GitHub/GHES. This also ensures + # connectivity through the proxy is established which can take 10-15s on + # the first request in some customer's environments. + def connectivity_check + Dependabot.logger.info("Connectivity check starting") + github_connectivity_client(job).repository(job.source.repo) + Dependabot.logger.info("Connectivity check successful") + rescue StandardError => e + Dependabot.logger.error("Connectivity check failed: #{e.message}") + end + + def github_connectivity_client(job) + Octokit::Client.new({ + api_endpoint: job.source.api_endpoint, + connection_options: { + request: { + open_timeout: 20, + timeout: 5 + } + } + }) + end + end +end diff --git a/updater/lib/dependabot/job.rb b/updater/lib/dependabot/job.rb new file mode 100644 index 000000000..da87e6798 --- /dev/null +++ b/updater/lib/dependabot/job.rb @@ -0,0 +1,341 @@ +# frozen_string_literal: true + +require "dependabot/config/ignore_condition" +require "dependabot/config/update_config" +require "dependabot/dependency_group_engine" +require "dependabot/experiments" +require "dependabot/source" +require "wildcard_matcher" + +# Describes a single Dependabot workload within the GitHub-integrated Service +# +# This primarily acts as a value class to hold inputs for various Core objects +# and is an approximate data structure for the 'job description file' used by +# the CLI tool. +# +# See: https://github.com/dependabot/cli#job-description-file +# +# This class should evenually be promoted to common/lib and augmented to +# validate job description files. +module Dependabot + class Job + TOP_LEVEL_DEPENDENCY_TYPES = %w(direct production development).freeze + PERMITTED_KEYS = %i( + allowed_updates + commit_message_options + dependencies + existing_pull_requests + existing_group_pull_requests + experiments + ignore_conditions + lockfile_only + package_manager + reject_external_code + repo_contents_path + requirements_update_strategy + security_advisories + security_updates_only + source + update_subdependencies + updating_a_pull_request + vendor_dependencies + dependency_groups + dependency_group_to_refresh + repo_private + ).freeze + + attr_reader :allowed_updates, + :credentials, + :dependencies, + :existing_pull_requests, + :existing_group_pull_requests, + :id, + :ignore_conditions, + :package_manager, + :requirements_update_strategy, + :security_advisories, + :security_updates_only, + :source, + :token, + :vendor_dependencies, + :dependency_groups, + :dependency_group_to_refresh + + def self.new_fetch_job(job_id:, job_definition:, repo_contents_path: nil) + attrs = standardise_keys(job_definition["job"]).slice(*PERMITTED_KEYS) + + new(attrs.merge(id: job_id, repo_contents_path: repo_contents_path)) + end + + def self.new_update_job(job_id:, job_definition:, repo_contents_path: nil) + job_hash = standardise_keys(job_definition["job"]) + attrs = job_hash.slice(*PERMITTED_KEYS) + attrs[:credentials] = job_hash[:credentials_metadata] || [] + + new(attrs.merge(id: job_id, repo_contents_path: repo_contents_path)) + end + + def self.standardise_keys(hash) + hash.transform_keys { |key| key.tr("-", "_").to_sym } + end + + # NOTE: "attributes" are fetched and injected at run time from + # dependabot-api using the UpdateJobPrivateSerializer + def initialize(attributes) + @id = attributes.fetch(:id) + @allowed_updates = attributes.fetch(:allowed_updates) + @commit_message_options = attributes.fetch(:commit_message_options, {}) + @credentials = attributes.fetch(:credentials, []) + @dependencies = attributes.fetch(:dependencies) + @existing_pull_requests = attributes.fetch(:existing_pull_requests) + # TODO: Make this hash required + # + # We will need to do a pass updating the CLI and smoke tests before this is possible, + # so let's consider it optional for now. If we get a nil value, let's force it to be + # an array. + @existing_group_pull_requests = attributes.fetch(:existing_group_pull_requests, []) || [] + @experiments = attributes.fetch(:experiments, {}) + @ignore_conditions = attributes.fetch(:ignore_conditions) + @package_manager = attributes.fetch(:package_manager) + @reject_external_code = attributes.fetch(:reject_external_code, false) + @repo_contents_path = attributes.fetch(:repo_contents_path, nil) + + @requirements_update_strategy = build_update_strategy( + **attributes.slice(:requirements_update_strategy, :lockfile_only) + ) + + @security_advisories = attributes.fetch(:security_advisories) + @security_updates_only = attributes.fetch(:security_updates_only) + @source = build_source(attributes.fetch(:source)) + @token = attributes.fetch(:token, nil) + @update_subdependencies = attributes.fetch(:update_subdependencies) + @updating_a_pull_request = attributes.fetch(:updating_a_pull_request) + @vendor_dependencies = attributes.fetch(:vendor_dependencies, false) + # TODO: Make this hash required + # + # We will need to do a pass updating the CLI and smoke tests before this is possible, + # so let's consider it optional for now. If we get a nil value, let's force it to be + # an array. + @dependency_groups = attributes.fetch(:dependency_groups, []) || [] + @dependency_group_to_refresh = attributes.fetch(:dependency_group_to_refresh, nil) + @repo_private = attributes.fetch(:repo_private, nil) + + register_experiments + end + + def clone? + vendor_dependencies? || + Dependabot::Utils.always_clone_for_package_manager?(@package_manager) + end + + # Some Core components test for a non-nil repo_contents_path as an implicit + # signal they should use cloning behaviour, so we present it as nil unless + # cloning is enabled to avoid unexpected behaviour. + def repo_contents_path + return nil unless clone? + + @repo_contents_path + end + + def repo_private? + @repo_private + end + + def updating_a_pull_request? + @updating_a_pull_request + end + + def update_subdependencies? + @update_subdependencies + end + + def security_updates_only? + @security_updates_only + end + + def vendor_dependencies? + @vendor_dependencies + end + + def reject_external_code? + @reject_external_code + end + + # TODO: Remove vulnerability checking + # + # This method does too much, let's make it focused on _just_ determining + # if the given dependency is within the configurations allowed_updates. + # + # The calling operation should be responsible for checking vulnerability + # separately, if required. + # + # rubocop:disable Metrics/PerceivedComplexity + def allowed_update?(dependency) + allowed_updates.any? do |update| + # Check the update-type (defaulting to all) + update_type = update.fetch("update-type", "all") + # NOTE: Preview supports specifying a "security" update type whereas + # native will say "security-updates-only" + security_update = update_type == "security" || security_updates_only? + next false if security_update && !vulnerable?(dependency) + + # Check the dependency-name (defaulting to matching) + condition_name = update.fetch("dependency-name", dependency.name) + next false unless name_match?(condition_name, dependency.name) + + # Check the dependency-type (defaulting to all) + dep_type = update.fetch("dependency-type", "all") + next false if dep_type == "indirect" && + dependency.requirements.any? + # In dependabot-api, dependency-type is defaulting to "direct" not "all". Ignoring + # that field for security updates, since it should probably be "all". + next false if !security_updates_only && + dependency.requirements.none? && + TOP_LEVEL_DEPENDENCY_TYPES.include?(dep_type) + next false if dependency.production? && dep_type == "development" + next false if !dependency.production? && dep_type == "production" + + true + end + end + # rubocop:enable Metrics/PerceivedComplexity + + def vulnerable?(dependency) + security_advisories = security_advisories_for(dependency) + return false if security_advisories.none? + + # Can't (currently) detect whether dependencies without a version + # (i.e., for repos without a lockfile) are vulnerable + return false unless dependency.version + + # Can't (currently) detect whether git dependencies are vulnerable + version_class = + Dependabot::Utils. + version_class_for_package_manager(dependency.package_manager) + return false unless version_class.correct?(dependency.version) + + all_versions = dependency.all_versions. + filter_map { |v| version_class.new(v) if version_class.correct?(v) } + security_advisories.any? { |a| all_versions.any? { |v| a.vulnerable?(v) } } + end + + def security_fix?(dependency) + security_advisories_for(dependency).any? { |a| a.fixed_by?(dependency) } + end + + def name_normaliser + Dependabot::Dependency.name_normaliser_for_package_manager(package_manager) + end + + def experiments + return {} unless @experiments + + self.class.standardise_keys(@experiments) + end + + def commit_message_options + return {} unless @commit_message_options + + self.class.standardise_keys(@commit_message_options).compact + end + + def security_advisories_for(dependency) + relevant_advisories = + security_advisories. + select { |adv| adv.fetch("dependency-name").casecmp(dependency.name).zero? } + + relevant_advisories.map do |adv| + vulnerable_versions = adv["affected-versions"] || [] + safe_versions = (adv["patched-versions"] || []) + + (adv["unaffected-versions"] || []) + + Dependabot::SecurityAdvisory.new( + dependency_name: dependency.name, + package_manager: package_manager, + vulnerable_versions: vulnerable_versions, + safe_versions: safe_versions + ) + end + end + + def ignore_conditions_for(dependency) + update_config.ignored_versions_for( + dependency, + security_updates_only: security_updates_only? + ) + end + + # TODO: Present Dependabot::Config::IgnoreCondition in calling code + # + # This is a workaround for our existing logging using the 'raw' + # ignore conditions passed into the job definition rather than + # the objects returned by `ignore_conditions_for`. + # + # The blocker on adopting Dependabot::Config::IgnoreCondition is + # that it does not have a 'source' attribute which we currently + # use to distinguish rules from the config file from those that + # were created via "@dependabot ignore version" commands + def log_ignore_conditions_for(dependency) + conditions = ignore_conditions.select { |ic| name_match?(ic["dependency-name"], dependency.name) } + return if conditions.empty? + + Dependabot.logger.info("Ignored versions:") + conditions.each do |ic| + unless ic["version-requirement"].nil? + Dependabot.logger.info(" #{ic['version-requirement']} - from #{ic['source']}") + end + + ic["update-types"]&.each do |update_type| + msg = " #{update_type} - from #{ic['source']}" + msg += " (doesn't apply to security update)" if security_updates_only? + Dependabot.logger.info(msg) + end + end + end + + private + + def register_experiments + experiments.each do |name, value| + Dependabot::Experiments.register(name, value) + end + end + + def name_match?(name1, name2) + WildcardMatcher.match?( + name_normaliser.call(name1), + name_normaliser.call(name2) + ) + end + + def build_update_strategy(requirements_update_strategy:, lockfile_only:) + return requirements_update_strategy unless requirements_update_strategy.nil? + + lockfile_only ? "lockfile_only" : nil + end + + def build_source(source_details) + Dependabot::Source.new( + **source_details.transform_keys { |k| k.tr("-", "_").to_sym } + ) + end + + # Provides a Dependabot::Config::UpdateConfig objected hydrated with + # relevant information obtained from the job definition. + # + # At present we only use this for ignore rules. + def update_config + return @update_config if defined? @update_config + + @update_config ||= Dependabot::Config::UpdateConfig.new( + ignore_conditions: ignore_conditions.map do |ic| + Dependabot::Config::IgnoreCondition.new( + dependency_name: ic["dependency-name"], + versions: [ic["version-requirement"]].compact, + update_types: ic["update-types"] + ) + end + ) + end + end +end diff --git a/updater/lib/dependabot/logger/formats.rb b/updater/lib/dependabot/logger/formats.rb new file mode 100644 index 000000000..95bd163e1 --- /dev/null +++ b/updater/lib/dependabot/logger/formats.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require "logger" + +# Provides Logger::Formatter classes specific to the Updater project to augment +# the global log helper defined in common/lib/dependabot/logger.rb +module Dependabot + module Logger + TIME_FORMAT = "%Y/%m/%d %H:%M:%S" + + class BasicFormatter < ::Logger::Formatter + def call(severity, _datetime, _progname, msg) + "#{Time.now.strftime(TIME_FORMAT)} #{severity} #{msg2str(msg)}\n" + end + end + + class JobFormatter < ::Logger::Formatter + CLI_ID = "cli" + UNKNOWN_ID = "unknown_id" + + def initialize(job_id) + @job_id = job_id + end + + def call(severity, _datetime, _progname, msg) + [ + Time.now.strftime(TIME_FORMAT), + severity, + job_prefix, + msg2str(msg) + ].compact.join(" ") + "\n" + end + + private + + def job_prefix + return @job_prefix if defined? @job_prefix + # The dependabot/cli tool uses a placeholder value since it does not + # have an actual Job ID issued by the service. + # + # Let's just omit the prefix if this is the case. + return @job_prefix = nil if @job_id == CLI_ID + + @job_prefix = "" + end + end + end +end diff --git a/updater/lib/dependabot/sentry.rb b/updater/lib/dependabot/sentry.rb new file mode 100644 index 000000000..7eb3d68c4 --- /dev/null +++ b/updater/lib/dependabot/sentry.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require "raven" + +# ExceptionSanitizer filters potential secrets/PII from exception payloads +class ExceptionSanitizer < Raven::Processor + REPO = %r{[\w.\-]+/([\w.\-]+)} + PATTERNS = { + auth_token: /(?:authorization|bearer):? (\w+)/i, + repo: %r{api\.github\.com/repos/#{REPO}|github\.com/#{REPO}} + }.freeze + + def process(data) + return data unless data[:exception] && data[:exception][:values] + + data[:exception][:values] = data[:exception][:values].map do |e| + PATTERNS.each do |key, regex| + next unless (matches = e[:value].scan(regex)) + + matches.flatten.compact.each do |match| + e[:value] = e[:value].gsub(match, "[FILTERED_#{key.to_s.upcase}]") + end + end + e + end + + data + end +end diff --git a/updater/lib/dependabot/service.rb b/updater/lib/dependabot/service.rb new file mode 100644 index 000000000..51a99f922 --- /dev/null +++ b/updater/lib/dependabot/service.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +require "raven" +require "terminal-table" +require "dependabot/api_client" + +# This class provides an output adapter for the Dependabot Service which manages +# communication with the private API as well as consolidated error handling. +# +# Currently this is the only output adapter available, but in future we may +# support others for use with the dependabot/cli project. +# +module Dependabot + class Service + extend Forwardable + attr_reader :pull_requests, :errors + + def initialize(client:) + @client = client + @pull_requests = [] + @errors = [] + end + + def_delegators :client, + :mark_job_as_processed, + :update_dependency_list, + :record_ecosystem_versions, + :increment_metric + + def create_pull_request(dependency_change, base_commit_sha) + client.create_pull_request(dependency_change, base_commit_sha) + @pull_requests << [dependency_change.humanized, :created] + end + + def update_pull_request(dependency_change, base_commit_sha) + client.update_pull_request(dependency_change, base_commit_sha) + @pull_requests << [dependency_change.humanized, :updated] + end + + def close_pull_request(dependencies, reason) + client.close_pull_request(dependencies, reason) + humanized_deps = dependencies.is_a?(String) ? dependencies : dependencies.join(",") + @pull_requests << [humanized_deps, "closed: #{reason}"] + end + + def record_update_job_error(error_type:, error_details:, dependency: nil) + @errors << [error_type.to_s, dependency] + client.record_update_job_error(error_type: error_type, error_details: error_details) + end + + def update_dependency_list(dependency_snapshot:) + dependency_payload = dependency_snapshot.dependencies.map do |dep| + { + name: dep.name, + version: dep.version, + requirements: dep.requirements + } + end + dependency_file_paths = dependency_snapshot.dependency_files.reject(&:support_file).map(&:path) + + client.update_dependency_list(dependency_payload, dependency_file_paths) + end + + # This method wraps the Raven client as the Application error tracker + # the service uses to notice errors. + # + # This should be called as an alternative/in addition to record_update_job_error + # for cases where an error could indicate a problem with the service. + def capture_exception(error:, job: nil, dependency: nil, dependency_group: nil, tags: {}, extra: {}) + Raven.capture_exception( + error, + { + tags: tags.merge({ + update_job_id: job&.id, + package_manager: job&.package_manager, + repo_private: job&.repo_private? + }.compact), + extra: extra.merge({ + dependency_name: dependency&.name, + dependency_group: dependency_group&.name + }.compact) + } + ) + end + + def noop? + pull_requests.empty? && errors.empty? + end + + def failure? + errors.any? + end + + # Example output: + # + # +----------------------------+-----------------------------------+ + # | Changes to Dependabot Pull Requests | + # +----------------------------+-----------------------------------+ + # | created | package-a ( from 1.0.0 to 1.0.1 ) | + # | updated | package-b ( from 1.1.0 to 1.2.1 ) | + # | closed:dependency-removed | package-c | + # +----------------------------+-----------------------------------+ + # + def summary + return if noop? + + [ + "Results:", + pull_request_summary, + error_summary, + job_error_type_summary, + dependency_error_summary + ].compact.join("\n") + end + + private + + attr_reader :client + + def pull_request_summary + return unless pull_requests.any? + + Terminal::Table.new do |t| + t.title = "Changes to Dependabot Pull Requests" + t.rows = pull_requests.map { |deps, action| [action, truncate(deps)] } + end + end + + def error_summary + return unless errors.any? + + "Dependabot encountered '#{errors.length}' error(s) during execution, please check the logs for more details." + end + + # Example output: + # + # +--------------------+ + # | Errors | + # +--------------------+ + # | job_repo_not_found | + # +--------------------+ + def job_error_type_summary + job_error_types = errors.filter_map { |error_type, dependency| [error_type] if dependency.nil? } + return if job_error_types.none? + + Terminal::Table.new do |t| + t.title = "Errors" + t.rows = job_error_types + end + end + + # Example output: + # + # +-------------------------------------+ + # | Dependencies failed to update | + # +---------------------+---------------+ + # | best_dependency_yay | unknown_error | + # +---------------------+---------------+ + def dependency_error_summary + dependency_errors = errors.filter_map do |error_type, dependency| + [dependency.name, error_type] unless dependency.nil? + end + return if dependency_errors.none? + + Terminal::Table.new do |t| + t.title = "Dependencies failed to update" + t.rows = dependency_errors + end + end + + def truncate(string, max: 120) + snip = max - 3 + string.length > max ? "#{string[0...snip]}..." : string + end + end +end diff --git a/updater/lib/dependabot/setup.rb b/updater/lib/dependabot/setup.rb new file mode 100644 index 000000000..d9395bffa --- /dev/null +++ b/updater/lib/dependabot/setup.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require "dependabot/logger" +require "dependabot/logger/formats" +require "dependabot/environment" + +Dependabot.logger = Logger.new($stdout).tap do |logger| + logger.level = Dependabot::Environment.log_level + logger.formatter = Dependabot::Logger::BasicFormatter.new +end + +require "dependabot/sentry" +Raven.configure do |config| + config.logger = Dependabot.logger + config.project_root = File.expand_path("../../..", __dir__) + + config.app_dirs_pattern = %r{( + dependabot-updater/bin| + dependabot-updater/config| + dependabot-updater/lib| + common| + python| + terraform| + elm| + docker| + git_submodules| + github_actions| + composer| + nuget| + gradle| + maven| + hex| + cargo| + go_modules| + npm_and_yarn| + bundler| + pub| + swift + )}x + + config.processors += [ExceptionSanitizer] +end + +# We configure `Dependabot::Utils.register_always_clone` for some ecosystems. In +# order for that configuration to take effect, we need to make sure that these +# registration commands have been executed. +require "dependabot/python" +require "dependabot/terraform" +require "dependabot/elm" +require "dependabot/docker" +require "dependabot/git_submodules" +require "dependabot/github_actions" +require "dependabot/composer" +require "dependabot/nuget" +require "dependabot/gradle" +require "dependabot/maven" +require "dependabot/hex" +require "dependabot/cargo" +require "dependabot/go_modules" +require "dependabot/npm_and_yarn" +require "dependabot/bundler" +require "dependabot/pub" +require "dependabot/swift" diff --git a/updater/lib/dependabot/update_files_command.rb b/updater/lib/dependabot/update_files_command.rb new file mode 100644 index 000000000..5b0504d3b --- /dev/null +++ b/updater/lib/dependabot/update_files_command.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +require "base64" +require "dependabot/base_command" +require "dependabot/dependency_snapshot" +require "dependabot/updater" + +module Dependabot + class UpdateFilesCommand < BaseCommand + def perform_job + # We expect the FileFetcherCommand to have been executed beforehand to place + # encoded files and commit information in the environment, so let's retrieve + # them, decode and parse them into an object that knows the current state + # of the project's dependencies. + begin + dependency_snapshot = Dependabot::DependencySnapshot.create_from_job_definition( + job: job, + job_definition: Environment.job_definition + ) + rescue StandardError => e + handle_parser_error(e) + # If dependency file parsing has failed, there's nothing more we can do, + # so let's mark the job as processed and stop. + return service.mark_job_as_processed(Environment.job_definition["base_commit_sha"]) + end + + # Update the service's metadata about this project + service.update_dependency_list(dependency_snapshot: dependency_snapshot) + + # TODO: Pull fatal error handling handling up into this class + # + # As above, we can remove the responsibility for handling fatal/job halting + # errors from Dependabot::Updater entirely. + Dependabot::Updater.new( + service: service, + job: job, + dependency_snapshot: dependency_snapshot + ).run + + # Finally, mark the job as processed. The Dependabot::Updater may have + # reported errors to the service, but we always consider the job as + # successfully processed unless it actually raises. + service.mark_job_as_processed(dependency_snapshot.base_commit_sha) + end + + private + + def job + @job ||= Job.new_update_job( + job_id: job_id, + job_definition: Environment.job_definition, + repo_contents_path: Environment.repo_contents_path + ) + end + + def base_commit_sha + Environment.job_definition["base_commit_sha"] + end + + # rubocop:disable Metrics/MethodLength + def handle_parser_error(error) + # This happens if the repo gets removed after a job gets kicked off. + # The service will handle the removal without any prompt from the updater, + # so no need to add an error to the errors array + return if error.is_a? Dependabot::RepoNotFound + + error_details = + case error + when Dependabot::DependencyFileNotEvaluatable + { + "error-type": "dependency_file_not_evaluatable", + "error-detail": { message: error.message } + } + when Dependabot::DependencyFileNotResolvable + { + "error-type": "dependency_file_not_resolvable", + "error-detail": { message: error.message } + } + when Dependabot::BranchNotFound + { + "error-type": "branch_not_found", + "error-detail": { "branch-name": error.branch_name } + } + when Dependabot::DependencyFileNotParseable + { + "error-type": "dependency_file_not_parseable", + "error-detail": { + message: error.message, + "file-path": error.file_path + } + } + when Dependabot::DependencyFileNotFound + { + "error-type": "dependency_file_not_found", + "error-detail": { "file-path": error.file_path } + } + when Dependabot::PathDependenciesNotReachable + { + "error-type": "path_dependencies_not_reachable", + "error-detail": { dependencies: error.dependencies } + } + when Dependabot::PrivateSourceAuthenticationFailure + { + "error-type": "private_source_authentication_failure", + "error-detail": { source: error.source } + } + when Dependabot::GitDependenciesNotReachable + { + "error-type": "git_dependencies_not_reachable", + "error-detail": { "dependency-urls": error.dependency_urls } + } + when Dependabot::NotImplemented + { + "error-type": "not_implemented", + "error-detail": { + message: error.message + } + } + when Octokit::ServerError + # If we get a 500 from GitHub there's very little we can do about it, + # and responsibility for fixing it is on them, not us. As a result we + # quietly log these as errors + { "error-type": "unknown_error" } + else + # Check if the error is a known "run halting" state we should handle + if (error_type = Updater::ErrorHandler::RUN_HALTING_ERRORS[error.class]) + { "error-type": error_type } + else + # If it isn't, then log all the details and let the application error + # tracker know about it + Dependabot.logger.error error.message + error.backtrace.each { |line| Dependabot.logger.error line } + + service.capture_exception(error: error, job: job) + + # Set an unknown error type to be added to the job + { "error-type": "unknown_error" } + end + end + + service.record_update_job_error( + error_type: error_details.fetch(:"error-type"), + error_details: error_details[:"error-detail"] + ) + end + # rubocop:enable Metrics/MethodLength + end +end diff --git a/updater/lib/dependabot/updater.rb b/updater/lib/dependabot/updater.rb new file mode 100644 index 000000000..846d64630 --- /dev/null +++ b/updater/lib/dependabot/updater.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +# Dependabot components +require "dependabot/dependency_change" +require "dependabot/dependency_change_builder" +require "dependabot/environment" +require "dependabot/experiments" +require "dependabot/file_fetchers" +require "dependabot/logger" +require "dependabot/security_advisory" +require "dependabot/update_checkers" + +# Ecosystems +require "dependabot/python" +require "dependabot/terraform" +require "dependabot/elm" +require "dependabot/docker" +require "dependabot/git_submodules" +require "dependabot/github_actions" +require "dependabot/composer" +require "dependabot/nuget" +require "dependabot/gradle" +require "dependabot/maven" +require "dependabot/hex" +require "dependabot/cargo" +require "dependabot/go_modules" +require "dependabot/npm_and_yarn" +require "dependabot/bundler" +require "dependabot/pub" +require "dependabot/swift" + +# Updater components +require "dependabot/updater/error_handler" +require "dependabot/updater/operations" +require "dependabot/updater/security_update_helpers" + +require "wildcard_matcher" + +module Dependabot + class Updater + # To do work, this class needs three arguments: + # - The Dependabot::Service to send events and outcomes to + # - The Dependabot::Job that describes the work to be done + # - The Dependabot::DependencySnapshot which encapsulates the starting state of the project + def initialize(service:, job:, dependency_snapshot:) + @service = service + @job = job + @dependency_snapshot = dependency_snapshot + @error_handler = ErrorHandler.new(service: service, job: job) + end + + def run + return unless job + raise Dependabot::NotImplemented unless (operation_class = Operations.class_for(job: job)) + + Dependabot.logger.debug("Performing job with #{operation_class}") + service.increment_metric("updater.started", tags: { operation: operation_class.tag_name }) + operation_class.new( + service: service, + job: job, + dependency_snapshot: dependency_snapshot, + error_handler: error_handler + ).perform + rescue *ErrorHandler::RUN_HALTING_ERRORS.keys => e + # TODO: Drop this into Security-specific operations + if e.is_a?(Dependabot::AllVersionsIgnored) && !job.security_updates_only? + error = StandardError.new( + "Dependabot::AllVersionsIgnored was unexpectedly raised for a non-security update job" + ) + error.set_backtrace(e.backtrace) + service.capture_exception(error: error, job: job) + return + end + + # OOM errors are special cased so that we stop the update run early + service.record_update_job_error( + error_type: ErrorHandler::RUN_HALTING_ERRORS.fetch(e.class), + error_details: nil + ) + end + + private + + attr_reader :service, :job, :dependency_snapshot, :error_handler + end +end diff --git a/updater/lib/dependabot/updater/dependency_group_change_batch.rb b/updater/lib/dependabot/updater/dependency_group_change_batch.rb new file mode 100644 index 000000000..af8028a49 --- /dev/null +++ b/updater/lib/dependabot/updater/dependency_group_change_batch.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +# This class is responsible for aggregating individual DependencyChange objects +# by tracking changes to individual files and the overall dependency list. +module Dependabot + class Updater + class DependencyGroupChangeBatch + attr_reader :updated_dependencies + + def initialize(initial_dependency_files:) + @updated_dependencies = [] + + @dependency_file_batch = initial_dependency_files.each_with_object({}) do |file, hsh| + hsh[file.path] = { file: file, changed: false, changes: 0 } + end + + @vendored_dependency_batch = {} + + Dependabot.logger.debug("Starting with '#{@dependency_file_batch.count}' dependency files:") + debug_current_file_state + end + + # Returns an array of DependencyFile objects for the current state + def current_dependency_files + @dependency_file_batch.map do |_path, data| + data[:file] + end + end + + # Returns an array of DependencyFile objects for dependency files that have changed at least once merged with + # and changes we've collected to vendored dependencies + def updated_dependency_files + @dependency_file_batch.filter_map { |_path, data| data[:file] if data[:changed] } + + @vendored_dependency_batch.map { |_path, data| data[:file] } + end + + def merge(dependency_change) + merge_dependency_changes(dependency_change.updated_dependencies) + merge_file_changes(dependency_change.updated_dependency_files) + + Dependabot.logger.debug("Dependencies updated:") + debug_updated_dependencies + + Dependabot.logger.debug("Dependency files updated:") + debug_current_file_state + end + + private + + # We should retain a list of all dependencies that we change, in future we may need to account for the folder + # in which these changes are made to permit-cross folder updates of the same dependency. + # + # This list may contain duplicates if we make iterative updates to a Dependency within a single group, but + # rather than re-write the Dependency objects to account for the changes from the lowest previous version + # to the final version, we should defer it to the Dependabot::PullRequestCreator::MessageBuilder as a + # presentation concern. + def merge_dependency_changes(updated_dependencies) + @updated_dependencies.concat(updated_dependencies) + end + + def merge_file_changes(updated_dependency_files) + updated_dependency_files.each do |updated_file| + if updated_file.vendored_file? + merge_file_to_batch(updated_file, @vendored_dependency_batch) + else + merge_file_to_batch(updated_file, @dependency_file_batch) + end + end + end + + def merge_file_to_batch(file, batch) + change_count = if (existing_file = batch[file.path]) + existing_file.fetch(:change_count, 0) + else + # The file is newly encountered + Dependabot.logger.debug("File #{file.operation}d: '#{file.path}'") + 0 + end + + batch[file.path] = { file: file, changed: true, changes: change_count + 1 } + end + + def debug_updated_dependencies + return unless Dependabot.logger.debug? + + @updated_dependencies.each do |dependency| + version_change = "#{dependency.humanized_previous_version} to #{dependency.humanized_version}" + Dependabot.logger.debug(" - #{dependency.name} ( #{version_change} )") + end + end + + def debug_current_file_state + return unless Dependabot.logger.debug? + + @dependency_file_batch.each { |path, data| debug_file_hash(path, data) } + + return unless @vendored_dependency_batch.any? + + Dependabot.logger.debug("Vendored dependency changes:") + @vendored_dependency_batch.each { |path, data| debug_file_hash(path, data) } + end + + def debug_file_hash(path, data) + changed_string = data[:changed] ? "( Changed #{data[:changes]} times )" : "" + Dependabot.logger.debug(" - #{path} #{changed_string}") + end + end + end +end diff --git a/updater/lib/dependabot/updater/error_handler.rb b/updater/lib/dependabot/updater/error_handler.rb new file mode 100644 index 000000000..0f27d1ff0 --- /dev/null +++ b/updater/lib/dependabot/updater/error_handler.rb @@ -0,0 +1,210 @@ +# frozen_string_literal: true + +require "dependabot/updater/errors" + +# This class is responsible for determining how to present a Dependabot::Error +# to the Service and Logger. +# +# TODO: Iterate further on leaner error handling +# +# This class is a coarse abstraction of some shared logic that has several flags +# against it from Rubocop we aren't addressing right now. +# +# It feels like this concern could be slimmed down if each Dependabot::Error +# class implemented a "presenter" method to generate it's own `error-type` and +# `error-detail` since this never draws attributes from the Updater context. +# +# For now, let's just extract it and set it aside as a tangent from the critical +# path. +module Dependabot + class Updater + class ErrorHandler + # These are errors that halt the update run and are handled in the main + # backend. They do *not* raise a sentry. + RUN_HALTING_ERRORS = { + Dependabot::OutOfDisk => "out_of_disk", + Dependabot::OutOfMemory => "out_of_memory", + Dependabot::AllVersionsIgnored => "all_versions_ignored", + Dependabot::UnexpectedExternalCode => "unexpected_external_code", + Errno::ENOSPC => "out_of_disk", + Octokit::Unauthorized => "octokit_unauthorized" + }.freeze + + def initialize(service:, job:) + @service = service + @job = job + end + + # This method handles errors where there is a dependency in the current + # context. This should be used by preference where possible. + def handle_dependency_error(error:, dependency:, dependency_group: nil) + # If the error is fatal for the run, we should re-raise it rather than + # pass it back to the service. + raise error if RUN_HALTING_ERRORS.keys.any? { |err| error.is_a?(err) } + + error_details = error_details_for(error, dependency: dependency, dependency_group: dependency_group) + service.record_update_job_error( + error_type: error_details.fetch(:"error-type"), + error_details: error_details[:"error-detail"], + dependency: dependency + ) + + log_dependency_error( + dependency: dependency, + error: error, + error_type: error_details.fetch(:"error-type"), + error_detail: error_details.fetch(:"error-detail", nil) + ) + end + + # Provides logging for errors that occur when processing a dependency + def log_dependency_error(dependency:, error:, error_type:, error_detail: nil) + if error_type == "unknown_error" + Dependabot.logger.error "Error processing #{dependency.name} (#{error.class.name})" + log_unknown_error_with_backtrace(error) + else + Dependabot.logger.info( + "Handled error whilst updating #{dependency.name}: #{error_type} #{error_detail}" + ) + end + end + + # This method handles errors where there is no dependency in the current + # context. + def handle_job_error(error:, dependency_group: nil) + # If the error is fatal for the run, we should re-raise it rather than + # pass it back to the service. + raise error if RUN_HALTING_ERRORS.keys.any? { |err| error.is_a?(err) } + + error_details = error_details_for(error, dependency_group: dependency_group) + service.record_update_job_error( + error_type: error_details.fetch(:"error-type"), + error_details: error_details[:"error-detail"] + ) + log_job_error( + error: error, + error_type: error_details.fetch(:"error-type"), + error_detail: error_details.fetch(:"error-detail", nil) + ) + end + + # Provides logging for errors that occur outside of a dependency context + def log_job_error(error:, error_type:, error_detail: nil) + if error_type == "unknown_error" + Dependabot.logger.error "Error processing job (#{error.class.name})" + log_unknown_error_with_backtrace(error) + else + Dependabot.logger.info( + "Handled error whilst processing job: #{error_type} #{error_detail}" + ) + end + end + + private + + attr_reader :service, :job + + # This method accepts an error class and returns an appropriate `error_details` hash + # to be reported to the backend service. + # + # For some specific errors, it also passes additional information to the + # exception service to aid in debugging, the optional arguments provide + # context to pass through in these cases. + def error_details_for(error, dependency: nil, dependency_group: nil) # rubocop:disable Metrics/MethodLength + case error + when Dependabot::DependencyFileNotResolvable + { + "error-type": "dependency_file_not_resolvable", + "error-detail": { message: error.message } + } + when Dependabot::DependencyFileNotEvaluatable + { + "error-type": "dependency_file_not_evaluatable", + "error-detail": { message: error.message } + } + when Dependabot::GitDependenciesNotReachable + { + "error-type": "git_dependencies_not_reachable", + "error-detail": { "dependency-urls": error.dependency_urls } + } + when Dependabot::GitDependencyReferenceNotFound + { + "error-type": "git_dependency_reference_not_found", + "error-detail": { dependency: error.dependency } + } + when Dependabot::PrivateSourceAuthenticationFailure + { + "error-type": "private_source_authentication_failure", + "error-detail": { source: error.source } + } + when Dependabot::PrivateSourceTimedOut + { + "error-type": "private_source_timed_out", + "error-detail": { source: error.source } + } + when Dependabot::PrivateSourceCertificateFailure + { + "error-type": "private_source_certificate_failure", + "error-detail": { source: error.source } + } + when Dependabot::MissingEnvironmentVariable + { + "error-type": "missing_environment_variable", + "error-detail": { + "environment-variable": error.environment_variable + } + } + when Dependabot::GoModulePathMismatch + { + "error-type": "go_module_path_mismatch", + "error-detail": { + "declared-path": error.declared_path, + "discovered-path": error.discovered_path, + "go-mod": error.go_mod + } + } + when Dependabot::NotImplemented + { + "error-type": "not_implemented", + "error-detail": { + message: error.message + } + } + when Dependabot::SharedHelpers::HelperSubprocessFailed + # If a helper subprocess has failed the error may include sensitive + # info such as file contents or paths. This information is already + # in the job logs, so we send a breadcrumb to Sentry to retrieve those + # instead. + msg = "Subprocess #{error.raven_context[:fingerprint]} failed to run. Check the job logs for error messages" + sanitized_error = SubprocessFailed.new(msg, raven_context: error.raven_context) + sanitized_error.set_backtrace(error.backtrace) + service.capture_exception(error: sanitized_error, job: job) + + { "error-type": "unknown_error" } + when *Octokit::RATE_LIMITED_ERRORS + # If we get a rate-limited error we let dependabot-api handle the + # retry by re-enqueing the update job after the reset + { + "error-type": "octokit_rate_limited", + "error-detail": { + "rate-limit-reset": error.response_headers["X-RateLimit-Reset"] + } + } + else + service.capture_exception( + error: error, + job: job, + dependency: dependency, + dependency_group: dependency_group + ) + { "error-type": "unknown_error" } + end + end + + def log_unknown_error_with_backtrace(error) + Dependabot.logger.error error.message + error.backtrace.each { |line| Dependabot.logger.error line } + end + end + end +end diff --git a/updater/lib/dependabot/updater/errors.rb b/updater/lib/dependabot/updater/errors.rb new file mode 100644 index 000000000..0407391a1 --- /dev/null +++ b/updater/lib/dependabot/updater/errors.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Dependabot + class Updater + class SubprocessFailed < StandardError + attr_reader :raven_context + + def initialize(message, raven_context:) + super(message) + + @raven_context = raven_context + end + end + end +end diff --git a/updater/lib/dependabot/updater/group_update_creation.rb b/updater/lib/dependabot/updater/group_update_creation.rb new file mode 100644 index 000000000..ff6666929 --- /dev/null +++ b/updater/lib/dependabot/updater/group_update_creation.rb @@ -0,0 +1,305 @@ +# frozen_string_literal: true + +require "dependabot/dependency_change_builder" +require "dependabot/updater/dependency_group_change_batch" +require "dependabot/workspace" + +# This module contains the methods required to build a DependencyChange for +# a single DependencyGroup. +# +# When included in an Operation it expects the following to be available: +# - job: the current Dependabot::Job object +# - dependency_snapshot: the Dependabot::DependencySnapshot of the current state +# - error_handler: a Dependabot::UpdaterErrorHandler to report any problems to +# +module Dependabot + class Updater + module GroupUpdateCreation + # Returns a Dependabot::DependencyChange object that encapsulates the + # outcome of attempting to update every dependency iteratively which + # can be used for PR creation. + def compile_all_dependency_changes_for(group) + prepare_workspace + + group_changes = Dependabot::Updater::DependencyGroupChangeBatch.new( + initial_dependency_files: dependency_snapshot.dependency_files + ) + + group.dependencies.each do |dependency| + # Get the current state of the dependency files for use in this iteration + dependency_files = group_changes.current_dependency_files + + # Reparse the current files + reparsed_dependencies = dependency_file_parser(dependency_files).parse + dependency = reparsed_dependencies.find { |d| d.name == dependency.name } + + # If the dependency can not be found in the reparsed files then it was likely removed by a previous + # dependency update + next if dependency.nil? + + updated_dependencies = compile_updates_for(dependency, dependency_files, group) + + next unless updated_dependencies.any? + + lead_dependency = updated_dependencies.find do |dep| + dep.name.casecmp(dependency.name).zero? + end + + dependency_change = create_change_for(lead_dependency, updated_dependencies, dependency_files, group) + + # Move on to the next dependency using the existing files if we + # could not create a change for any reason + next unless dependency_change + + # Store the updated files for the next loop + group_changes.merge(dependency_change) + store_changes(dependency) + end + + # Create a single Dependabot::DependencyChange that aggregates everything we've updated + # into a single object we can pass to PR creation. + Dependabot::DependencyChange.new( + job: job, + updated_dependencies: group_changes.updated_dependencies, + updated_dependency_files: group_changes.updated_dependency_files, + dependency_group: group + ) + ensure + cleanup_workspace + end + + def dependency_file_parser(dependency_files) + Dependabot::FileParsers.for_package_manager(job.package_manager).new( + dependency_files: dependency_files, + repo_contents_path: job.repo_contents_path, + source: job.source, + credentials: job.credentials, + reject_external_code: job.reject_external_code?, + options: job.experiments + ) + end + + # This method generates a DependencyChange from the current files and + # list of dependencies to be updated + # + # This method **must** return false in the event of an error + def create_change_for(lead_dependency, updated_dependencies, dependency_files, dependency_group) + Dependabot::DependencyChangeBuilder.create_from( + job: job, + dependency_files: dependency_files, + updated_dependencies: updated_dependencies, + change_source: dependency_group + ) + rescue Dependabot::InconsistentRegistryResponse => e + error_handler.log_dependency_error( + dependency: lead_dependency, + error: e, + error_type: "inconsistent_registry_response", + error_detail: e.message + ) + + false + rescue StandardError => e + error_handler.handle_dependency_error(error: e, dependency: lead_dependency, dependency_group: dependency_group) + + false + end + + # This method determines which dependencies must change given a target + # 'lead' dependency we want to update. + # + # This may return more than 1 dependency since the ecosystem-specific + # tooling may find collaborators which need to be updated in lock-step. + # + # This method **must** must return an Array when it errors + # + def compile_updates_for(dependency, dependency_files, group) # rubocop:disable Metrics/MethodLength + checker = update_checker_for( + dependency, + dependency_files, + group, + raise_on_ignored: raise_on_ignored?(dependency) + ) + + log_checking_for_update(dependency) + + return [] if all_versions_ignored?(dependency, checker) + return [] unless semver_rules_allow_grouping?(group, dependency, checker) + + # Consider the dependency handled so no individual PR is raised since it is in this group. + # Even if update is not possible, etc. + group.add_to_handled(dependency) + + if checker.up_to_date? + log_up_to_date(dependency) + return [] + end + + requirements_to_unlock = requirements_to_unlock(checker) + log_requirements_for_update(requirements_to_unlock, checker) + + if requirements_to_unlock == :update_not_possible + Dependabot.logger.info( + "No update possible for #{dependency.name} #{dependency.version}" + ) + return [] + end + + checker.updated_dependencies( + requirements_to_unlock: requirements_to_unlock + ) + rescue Dependabot::InconsistentRegistryResponse => e + group.add_to_handled(dependency) + error_handler.log_dependency_error( + dependency: dependency, + error: e, + error_type: "inconsistent_registry_response", + error_detail: e.message + ) + [] # return an empty set + rescue StandardError => e + # If there was an error we might not be able to determine if the dependency is in this + # group due to semver grouping, so we consider it handled to avoid raising an individual PR. + group.add_to_handled(dependency) + error_handler.handle_dependency_error(error: e, dependency: dependency, dependency_group: group) + [] # return an empty set + end + + def log_up_to_date(dependency) + Dependabot.logger.info( + "No update needed for #{dependency.name} #{dependency.version}" + ) + end + + def raise_on_ignored?(dependency) + job.ignore_conditions_for(dependency).any? + end + + def update_checker_for(dependency, dependency_files, dependency_group, raise_on_ignored:) + Dependabot::UpdateCheckers.for_package_manager(job.package_manager).new( + dependency: dependency, + dependency_files: dependency_files, + repo_contents_path: job.repo_contents_path, + credentials: job.credentials, + ignored_versions: job.ignore_conditions_for(dependency), + security_advisories: [], # FIXME: Version updates do not use advisory data for now + raise_on_ignored: raise_on_ignored, + requirements_update_strategy: job.requirements_update_strategy, + dependency_group: dependency_group, + options: job.experiments + ) + end + + def log_checking_for_update(dependency) + Dependabot.logger.info( + "Checking if #{dependency.name} #{dependency.version} needs updating" + ) + job.log_ignore_conditions_for(dependency) + end + + def all_versions_ignored?(dependency, checker) + Dependabot.logger.info("Latest version is #{checker.latest_version}") + false + rescue Dependabot::AllVersionsIgnored + Dependabot.logger.info("All updates for #{dependency.name} were ignored") + true + end + + # This method applies "SemVer Grouping" rules: if the latest update is greater than the update-types, + # then it should not be in the group, but be an individual PR, or in another group that fits it. + # SemVer Grouping rules have to be applied after we have a checker, because we need to know the latest version. + # Other rules are applied earlier in the process. + def semver_rules_allow_grouping?(group, dependency, checker) + # There are no group rules defined, so this dependency can be included in the group. + return true unless group.rules["update-types"] + + # git dependencies are not SemVer compatible so we cannot include them in the group + return false if git_dependency?(dependency) + + version = Dependabot::Utils.version_class_for_package_manager(job.package_manager).new(dependency.version.to_s) + # Not every version class implements .major, .minor, .patch so we calculate it here from the segments + latest = semver_segments(checker.latest_version) + current = semver_segments(version) + return group.rules["update-types"].include?("major") if latest[:major] > current[:major] + return group.rules["update-types"].include?("minor") if latest[:minor] > current[:minor] + return group.rules["update-types"].include?("patch") if latest[:patch] > current[:patch] + + # some ecosystems don't do semver exactly, so anything lower gets individual for now + false + end + + def semver_segments(version) + { + major: version.segments[0] || 0, + minor: version.segments[1] || 0, + patch: version.segments[2] || 0 + } + end + + def requirements_to_unlock(checker) + if !checker.requirements_unlocked_or_can_be? + if checker.can_update?(requirements_to_unlock: :none) then :none + else + :update_not_possible + end + elsif checker.can_update?(requirements_to_unlock: :own) then :own + elsif checker.can_update?(requirements_to_unlock: :all) then :all + else + :update_not_possible + end + end + + def git_dependency?(dependency) + GitCommitChecker.new( + dependency: dependency, + credentials: job.credentials + ).git_dependency? + end + + def log_requirements_for_update(requirements_to_unlock, checker) + Dependabot.logger.info("Requirements to unlock #{requirements_to_unlock}") + + return unless checker.respond_to?(:requirements_update_strategy) + + Dependabot.logger.info( + "Requirements update strategy #{checker.requirements_update_strategy}" + ) + end + + def warn_group_is_empty(group) + Dependabot.logger.warn( + "Skipping update group for '#{group.name}' as it does not match any allowed dependencies." + ) + + return unless Dependabot.logger.debug? + + Dependabot.logger.debug(<<~DEBUG.chomp) + The configuration for this group is: + + #{group.to_config_yaml} + DEBUG + end + + def prepare_workspace + return unless job.clone? && job.repo_contents_path + + Dependabot::Workspace.setup( + repo_contents_path: job.repo_contents_path, + directory: Pathname.new(job.source.directory || "/").cleanpath + ) + end + + def store_changes(dependency) + return unless job.clone? && job.repo_contents_path + + Dependabot::Workspace.store_change(memo: "Updating #{dependency.name}") + end + + def cleanup_workspace + return unless job.clone? && job.repo_contents_path + + Dependabot::Workspace.cleanup! + end + end + end +end diff --git a/updater/lib/dependabot/updater/operations.rb b/updater/lib/dependabot/updater/operations.rb new file mode 100644 index 000000000..a8759a433 --- /dev/null +++ b/updater/lib/dependabot/updater/operations.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require "dependabot/updater/operations/create_security_update_pull_request" +require "dependabot/updater/operations/group_update_all_versions" +require "dependabot/updater/operations/refresh_group_update_pull_request" +require "dependabot/updater/operations/refresh_security_update_pull_request" +require "dependabot/updater/operations/refresh_version_update_pull_request" +require "dependabot/updater/operations/update_all_versions" + +# This module is responsible for determining which Operation a Job is requesting +# the Updater to perform. +# +# The design goal for this module is to make these classes easy to understand, +# maintain and extend so we can eventually support community-contributed +# alternatives or ecosystem-specific implementations. +# +# Consider the following guidelines when working on Operation classes: +# +# - Operations *should not have optional parameters*, prefer to create a new +# class instead of adding branching to an existing one. +# +# - Operations should prefer to share logic by composition, there is no base +# class. We want to avoid implicit or indirect behaviour as much as possible. +# +module Dependabot + class Updater + module Operations + # We check if each operation ::applies_to? a given job, returning the first + # that does, so these Operations should be ordered so that those with most + # specific preconditions go before those with more permissive checks. + OPERATIONS = [ + CreateSecurityUpdatePullRequest, + RefreshSecurityUpdatePullRequest, + RefreshGroupUpdatePullRequest, + RefreshVersionUpdatePullRequest, + GroupUpdateAllVersions, + UpdateAllVersions + ].freeze + + def self.class_for(job:) + # Let's not bother generating the string if debug is disabled + if Dependabot.logger.debug? + update_type = job.security_updates_only? ? "security" : "version" + update_verb = job.updating_a_pull_request? ? "refresh" : "create" + update_deps = job.dependencies&.any? ? job.dependencies.count : "all" + + Dependabot.logger.debug( + "Finding operation for a #{update_type} to #{update_verb} a Pull Request for #{update_deps} dependencies" + ) + end + + raise ArgumentError, "Expected Dependabot::Job, got #{job.class}" unless job.is_a?(Dependabot::Job) + + OPERATIONS.find { |op| op.applies_to?(job: job) } + end + end + end +end diff --git a/updater/lib/dependabot/updater/operations/create_group_update_pull_request.rb b/updater/lib/dependabot/updater/operations/create_group_update_pull_request.rb new file mode 100644 index 000000000..28205eb3a --- /dev/null +++ b/updater/lib/dependabot/updater/operations/create_group_update_pull_request.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require "dependabot/updater/group_update_creation" + +# This class implements our strategy for creating a single Pull Request which +# updates all outdated Dependencies within a specific project folder that match +# a specificed Dependency Group. +# +# This will always post a new Pull Request to Dependabot API and does not check +# to see if any exists for the group or any of the dependencies involved. +# +module Dependabot + class Updater + module Operations + class CreateGroupUpdatePullRequest + include GroupUpdateCreation + + # We do not invoke this class directly for any jobs, so let's return false in the event this + # check is called. + def self.applies_to?(*) + false + end + + def self.tag_name + :create_version_group_pr + end + + # Since this class is not invoked generically based on the job definition, this class accepts a `group` argument + # which is expected to be a prepopulated DependencyGroup object. + def initialize(service:, job:, dependency_snapshot:, error_handler:, group:) + @service = service + @job = job + @dependency_snapshot = dependency_snapshot + @error_handler = error_handler + @group = group + end + + def perform + return warn_group_is_empty(group) if group.dependencies.empty? + + Dependabot.logger.info("Starting update group for '#{group.name}'") + + dependency_change = compile_all_dependency_changes_for(group) + + if dependency_change.updated_dependencies.any? + Dependabot.logger.info("Creating a pull request for '#{group.name}'") + begin + service.create_pull_request(dependency_change, dependency_snapshot.base_commit_sha) + rescue StandardError => e + error_handler.handle_job_error(error: e, dependency_group: group) + end + else + Dependabot.logger.info("Nothing to update for Dependency Group: '#{group.name}'") + end + + dependency_change + end + + private + + attr_reader :job, + :service, + :dependency_snapshot, + :error_handler, + :group + end + end + end +end diff --git a/updater/lib/dependabot/updater/operations/create_security_update_pull_request.rb b/updater/lib/dependabot/updater/operations/create_security_update_pull_request.rb new file mode 100644 index 000000000..d5666897d --- /dev/null +++ b/updater/lib/dependabot/updater/operations/create_security_update_pull_request.rb @@ -0,0 +1,270 @@ +# frozen_string_literal: true + +require "dependabot/updater/security_update_helpers" + +# This class implements our strategy for updating a single, insecure dependency +# to a secure version. We attempt to make the smallest version update possible, +# i.e. semver patch-level increase is preferred over minor-level increase. +module Dependabot + class Updater + module Operations + class CreateSecurityUpdatePullRequest + include SecurityUpdateHelpers + + def self.applies_to?(job:) + return false if job.updating_a_pull_request? + # If we haven't been given data for the vulnerable dependency, + # this strategy cannot act. + return false unless job.dependencies&.any? + + job.security_updates_only? + end + + def self.tag_name + :create_security_pr + end + + def initialize(service:, job:, dependency_snapshot:, error_handler:) + @service = service + @job = job + @dependency_snapshot = dependency_snapshot + @error_handler = error_handler + # TODO: Collect @created_pull_requests on the Job object? + @created_pull_requests = [] + end + + # TODO: We currently tolerate multiple dependencies for this operation + # but in reality, we only expect a single dependency per job. + # + # Changing this contract now without some safety catches introduces + # risk, so we'll maintain the interface as-is for now, but this is + # something we should make much more intentional in future. + def perform + Dependabot.logger.info("Starting security update job for #{job.source.repo}") + + target_dependencies = dependency_snapshot.job_dependencies + + if target_dependencies.empty? + record_security_update_dependency_not_found + else + target_dependencies.each { |dep| check_and_create_pr_with_error_handling(dep) } + end + end + + private + + attr_reader :job, + :service, + :dependency_snapshot, + :error_handler, + :created_pull_requests + + def check_and_create_pr_with_error_handling(dependency) + check_and_create_pull_request(dependency) + rescue Dependabot::InconsistentRegistryResponse => e + error_handler.log_dependency_error( + dependency: dependency, + error: e, + error_type: "inconsistent_registry_response", + error_detail: e.message + ) + rescue StandardError => e + error_handler.handle_dependency_error(error: e, dependency: dependency) + end + + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/PerceivedComplexity + # rubocop:disable Metrics/MethodLength + def check_and_create_pull_request(dependency) + checker = update_checker_for(dependency) + + log_checking_for_update(dependency) + + Dependabot.logger.info("Latest version is #{checker.latest_version}") + + unless checker.vulnerable? + # The current dependency isn't vulnerable if the version is correct and + # can be matched against the advisories affected versions + if checker.version_class.correct?(checker.dependency.version) + return record_security_update_not_needed_error(checker) + end + + return record_dependency_file_not_supported_error(checker) + end + + return record_security_update_ignored(checker) unless job.allowed_update?(dependency) + + # The current version is still vulnerable and Dependabot can't find a + # published or compatible non-vulnerable version, this can happen if the + # fixed version hasn't been published yet or the published version isn't + # compatible with the current enviroment (e.g. python version) or + # version (uses a different version suffix for gradle/maven) + return record_security_update_not_found(checker) if checker.up_to_date? + + if pr_exists_for_latest_version?(checker) + Dependabot.logger.info( + "Pull request already exists for #{checker.dependency.name} " \ + "with latest version #{checker.latest_version}" + ) + return record_pull_request_exists_for_latest_version(checker) + end + + requirements_to_unlock = requirements_to_unlock(checker) + log_requirements_for_update(requirements_to_unlock, checker) + return record_security_update_not_possible_error(checker) if requirements_to_unlock == :update_not_possible + + updated_deps = checker.updated_dependencies( + requirements_to_unlock: requirements_to_unlock + ) + + # Prevent updates that don't end up fixing any security advisories, + # blocking any updates where dependabot-core updates to a vulnerable + # version. This happens for npm/yarn subdendencies where Dependabot has no + # control over the target version. Related issue: + # https://github.com/github/dependabot-api/issues/905 + return record_security_update_not_possible_error(checker) if updated_deps.none? { |d| job.security_fix?(d) } + + if (existing_pr = existing_pull_request(updated_deps)) + # Create a update job error to prevent dependabot-api from creating a + # update_not_possible error, this is likely caused by a update job retry + # so should be invisible to users (as the first job completed with a pull + # request) + record_pull_request_exists_for_security_update(existing_pr) + + deps = existing_pr.map do |dep| + if dep.fetch("dependency-removed", false) + "#{dep.fetch('dependency-name')}@removed" + else + "#{dep.fetch('dependency-name')}@#{dep.fetch('dependency-version')}" + end + end + + return Dependabot.logger.info( + "Pull request already exists for #{deps.join(', ')}" + ) + end + + dependency_change = Dependabot::DependencyChangeBuilder.create_from( + job: job, + dependency_files: dependency_snapshot.dependency_files, + updated_dependencies: updated_deps, + change_source: checker.dependency + ) + create_pull_request(dependency_change) + rescue Dependabot::AllVersionsIgnored + Dependabot.logger.info("All updates for #{dependency.name} were ignored") + # Report this error to the backend to create an update job error + raise + end + # rubocop:enable Metrics/MethodLength + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/PerceivedComplexity + + def update_checker_for(dependency) + Dependabot::UpdateCheckers.for_package_manager(job.package_manager).new( + dependency: dependency, + dependency_files: dependency_snapshot.dependency_files, + repo_contents_path: job.repo_contents_path, + credentials: job.credentials, + ignored_versions: job.ignore_conditions_for(dependency), + security_advisories: job.security_advisories_for(dependency), + raise_on_ignored: true, # always true for security updates + requirements_update_strategy: job.requirements_update_strategy, + options: job.experiments + ) + end + + def log_checking_for_update(dependency) + Dependabot.logger.info( + "Checking if #{dependency.name} #{dependency.version} needs updating" + ) + job.log_ignore_conditions_for(dependency) + end + + def log_up_to_date(dependency) + Dependabot.logger.info( + "No update needed for #{dependency.name} #{dependency.version}" + ) + end + + def log_requirements_for_update(requirements_to_unlock, checker) + Dependabot.logger.info("Requirements to unlock #{requirements_to_unlock}") + + return unless checker.respond_to?(:requirements_update_strategy) + + Dependabot.logger.info( + "Requirements update strategy #{checker.requirements_update_strategy}" + ) + end + + def pr_exists_for_latest_version?(checker) + latest_version = checker.latest_version&.to_s + return false if latest_version.nil? + + job.existing_pull_requests. + select { |pr| pr.count == 1 }. + map(&:first). + select { |pr| pr.fetch("dependency-name") == checker.dependency.name }. + any? { |pr| pr.fetch("dependency-version", nil) == latest_version } + end + + def existing_pull_request(updated_dependencies) + new_pr_set = Set.new( + updated_dependencies.map do |dep| + { + "dependency-name" => dep.name, + "dependency-version" => dep.version, + "dependency-removed" => dep.removed? ? true : nil + }.compact + end + ) + + job.existing_pull_requests.find { |pr| Set.new(pr) == new_pr_set } || + created_pull_requests.find { |pr| Set.new(pr) == new_pr_set } + end + + def requirements_to_unlock(checker) + if !checker.requirements_unlocked_or_can_be? + if checker.can_update?(requirements_to_unlock: :none) then :none + else + :update_not_possible + end + elsif checker.can_update?(requirements_to_unlock: :own) then :own + elsif checker.can_update?(requirements_to_unlock: :all) then :all + else + :update_not_possible + end + end + + def create_pull_request(dependency_change) + Dependabot.logger.info("Submitting #{dependency_change.updated_dependencies.map(&:name).join(', ')} " \ + "pull request for creation") + + service.create_pull_request(dependency_change, dependency_snapshot.base_commit_sha) + + created_pull_requests << dependency_change.updated_dependencies.map do |dep| + { + "dependency-name" => dep.name, + "dependency-version" => dep.version, + "dependency-removed" => dep.removed? ? true : nil + }.compact + end + end + + def update_pull_request(dependency_change) + Dependabot.logger.info("Submitting #{dependency_change.updated_dependencies.map(&:name).join(', ')} " \ + "pull request for update") + + service.update_pull_request(dependency_change, dependency_snapshot.base_commit_sha) + end + + def close_pull_request(reason:) + reason_string = reason.to_s.tr("_", " ") + Dependabot.logger.info("Telling backend to close pull request for " \ + "#{job.dependencies.join(', ')} - #{reason_string}") + service.close_pull_request(job.dependencies, reason) + end + end + end + end +end diff --git a/updater/lib/dependabot/updater/operations/group_update_all_versions.rb b/updater/lib/dependabot/updater/operations/group_update_all_versions.rb new file mode 100644 index 000000000..fdb0d3447 --- /dev/null +++ b/updater/lib/dependabot/updater/operations/group_update_all_versions.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +require "dependabot/updater/operations/create_group_update_pull_request" +require "dependabot/updater/operations/update_all_versions" + +# This class is responsible for coordinating the creation and upkeep of Pull Requests for +# a given folder's defined DependencyGroups. +# +# - If there is no Pull Request already open for a DependencyGroup, it will be delegated +# to Dependabot::Updater::Operations::CreateGroupUpdatePullRequest. +# - If there is an open Pull Request for a DependencyGroup, it will skip over that group +# as the service is responsible for refreshing it in a separate job. +# - Any ungrouped Dependencies will be handled individually by delegation to +# Dependabot::Updater::Operations::UpdateAllVersions. +# +module Dependabot + class Updater + module Operations + class GroupUpdateAllVersions + def self.applies_to?(job:) + return false if job.security_updates_only? + return false if job.updating_a_pull_request? + return false if job.dependencies&.any? + + job.dependency_groups&.any? && Dependabot::Experiments.enabled?(:grouped_updates_prototype) + end + + def self.tag_name + :group_update_all_versions + end + + def initialize(service:, job:, dependency_snapshot:, error_handler:) + @service = service + @job = job + @dependency_snapshot = dependency_snapshot + @error_handler = error_handler + @dependencies_handled = Set.new + end + + def perform + if dependency_snapshot.groups.any? + run_grouped_dependency_updates + else + # We shouldn't have selected this operation if no groups were defined + # due to the rules in `::applies_to?`, but if it happens it isn't + # enough reasons to fail the job. + Dependabot.logger.warn( + "No dependency groups defined!" + ) + + # We should warn our exception tracker in case this represents an + # unexpected problem hydrating groups we have swallowed and then + # delegate everything to run_ungrouped_dependency_updates. + service.capture_exception( + error: DependabotError.new("Attempted a grouped update with no groups defined."), + job: job + ) + end + + run_ungrouped_dependency_updates + end + + private + + attr_reader :job, + :service, + :dependency_snapshot, + :error_handler + + def run_grouped_dependency_updates + Dependabot.logger.info("Starting grouped update job for #{job.source.repo}") + Dependabot.logger.info("Found #{dependency_snapshot.groups.count} group(s).") + + dependency_snapshot.groups.each do |group| + # If this group does not use update-types, then consider all dependencies as grouped. + # This will prevent any failures from creating individual PRs erroneously. + group.add_all_to_handled unless group.rules&.key?("update-types") + + if pr_exists_for_dependency_group?(group) + Dependabot.logger.info("Detected existing pull request for '#{group.name}'.") + Dependabot.logger.info( + "Deferring creation of a new pull request. The existing pull request will update in a separate job." + ) + # add the dependencies in the group so individual updates don't try to update them + group.add_all_to_handled + next + end + + result = run_update_for(group) + group.add_to_handled(*result.updated_dependencies) if result + end + end + + def pr_exists_for_dependency_group?(group) + job.existing_group_pull_requests&.any? { |pr| pr["dependency-group-name"] == group.name } + end + + def run_update_for(group) + Dependabot::Updater::Operations::CreateGroupUpdatePullRequest.new( + service: service, + job: job, + dependency_snapshot: dependency_snapshot, + error_handler: error_handler, + group: group + ).perform + end + + def run_ungrouped_dependency_updates + return if dependency_snapshot.ungrouped_dependencies.empty? + + Dependabot::Updater::Operations::UpdateAllVersions.new( + service: service, + job: job, + dependency_snapshot: dependency_snapshot, + error_handler: error_handler + ).perform + end + end + end + end +end diff --git a/updater/lib/dependabot/updater/operations/refresh_group_update_pull_request.rb b/updater/lib/dependabot/updater/operations/refresh_group_update_pull_request.rb new file mode 100644 index 000000000..b243153d4 --- /dev/null +++ b/updater/lib/dependabot/updater/operations/refresh_group_update_pull_request.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +require "dependabot/updater/group_update_creation" + +# This class implements our strategy for refreshing a single Pull Request which +# updates all outdated Dependencies within a specific project folder that match +# a specificed Dependency Group. +# +# Refreshing a Dependency Group pull request essentially has two outcomes, we +# either update or supersede the existing PR. +# +# To decide which strategy to use, we recompute the DependencyChange on the +# current head of the target branch and: +# - determine that all the same dependencies change to the same versions +# - in this case we update the existing PR +# - determine that one or more dependencies are now involved or removed +# - in this case we close the existing PR and create a new one +# - determine that all the dependencies are the same, but versions have changed +# -in this case we close the existing PR and create a new one +module Dependabot + class Updater + module Operations + class RefreshGroupUpdatePullRequest + include GroupUpdateCreation + + def self.applies_to?(job:) + return false if job.security_updates_only? + # If we haven't been given metadata about the dependencies present + # in the pull request and the Dependency Group that originally created + # it, this strategy cannot act. + return false unless job.dependencies&.any? + return false unless job.dependency_group_to_refresh + + job.updating_a_pull_request? && Dependabot::Experiments.enabled?(:grouped_updates_prototype) + end + + def self.tag_name + :update_version_group_pr + end + + def initialize(service:, job:, dependency_snapshot:, error_handler:) + @service = service + @job = job + @dependency_snapshot = dependency_snapshot + @error_handler = error_handler + end + + def perform + # This guards against any jobs being performed where the data is malformed, this should not happen unless + # there was is defect in the service and we emitted a payload where the job and configuration data objects + # were out of sync. + unless dependency_snapshot.job_group + Dependabot.logger.warn( + "The '#{job.dependency_group_to_refresh || 'unknown'}' group has been removed from the update config." + ) + + service.capture_exception( + error: DependabotError.new("Attempted to refresh a missing group."), + job: job + ) + return + end + + Dependabot.logger.info("Starting PR update job for #{job.source.repo}") + + if dependency_snapshot.job_group.dependencies.empty? + # If the group is empty that means any Dependencies that did match this group + # have been removed from the project or are no longer allowed by the config. + # + # Let's warn that the group is empty and then signal the PR should be closed + # so users are informed this group is no longer actionable by Dependabot. + warn_group_is_empty(dependency_snapshot.job_group) + close_pull_request(reason: :dependency_group_empty) + else + Dependabot.logger.info("Updating the '#{dependency_snapshot.job_group.name}' group") + + dependency_change = compile_all_dependency_changes_for(dependency_snapshot.job_group) + + upsert_pull_request_with_error_handling(dependency_change) + end + end + + private + + attr_reader :job, + :service, + :dependency_snapshot, + :error_handler + + def upsert_pull_request_with_error_handling(dependency_change) + if dependency_change.updated_dependencies.any? + upsert_pull_request(dependency_change) + else + Dependabot.logger.info("Dependencies are up to date, closing existing Pull Request") + close_pull_request(reason: :up_to_date) + end + rescue StandardError => e + error_handler.handle_job_error(error: e, group: job_group) + end + + # Having created the dependency_change, we need to determine the right strategy to apply it to the project: + # - Replace existing PR if the dependencies involved have changed + # - Update the existing PR if the dependencies and the target versions remain the same + # - Supersede the existing PR if the dependencies are the same but the target versions have changed + def upsert_pull_request(dependency_change) + if dependency_change.should_replace_existing_pr? + Dependabot.logger.info("Dependencies have changed, closing existing Pull Request") + close_pull_request(reason: :dependencies_changed) + Dependabot.logger.info("Creating a new pull request for '#{dependency_snapshot.job_group.name}'") + service.create_pull_request(dependency_change, dependency_snapshot.base_commit_sha) + elsif dependency_change.matches_existing_pr? + Dependabot.logger.info("Updating pull request for '#{dependency_snapshot.job_group.name}'") + service.update_pull_request(dependency_change, dependency_snapshot.base_commit_sha) + else + # If the changes do not match an existing PR, then we should open a new pull request and leave it to + # the backend to close the existing pull request with a comment that it has been superseded. + Dependabot.logger.info("Target versions have changed, existing Pull Request should be superseded") + Dependabot.logger.info("Creating a new pull request for '#{dependency_snapshot.job_group.name}'") + service.create_pull_request(dependency_change, dependency_snapshot.base_commit_sha) + end + end + + def close_pull_request(reason:) + reason_string = reason.to_s.tr("_", " ") + Dependabot.logger.info( + "Telling backend to close pull request for the " \ + "#{dependency_snapshot.job_group.name} group " \ + "(#{job.dependencies.join(', ')}) - #{reason_string}" + ) + + service.close_pull_request(job.dependencies, reason) + end + end + end + end +end diff --git a/updater/lib/dependabot/updater/operations/refresh_security_update_pull_request.rb b/updater/lib/dependabot/updater/operations/refresh_security_update_pull_request.rb new file mode 100644 index 000000000..bf410557a --- /dev/null +++ b/updater/lib/dependabot/updater/operations/refresh_security_update_pull_request.rb @@ -0,0 +1,233 @@ +# frozen_string_literal: true + +# This class implements our strategy for 'refreshing' an existing Pull Request +# that updates an insecure dependency. +# +# TODO: copyedit +# +# It will determine if the existing diff is still relevant, in which case it +# functions similar to a "rebase", but in the case where the project folder's +# dependencies have changed or a newer version is available, it will supersede +# the existing pull request with a new one for clarity. +module Dependabot + class Updater + module Operations + class RefreshSecurityUpdatePullRequest + include SecurityUpdateHelpers + + def self.applies_to?(job:) + return false unless job.security_updates_only? + # If we haven't been given metadata about the dependencies present + # in the pull request, this strategy cannot act. + return false if job.dependencies&.none? + + job.updating_a_pull_request? + end + + def self.tag_name + :update_security_pr + end + + def initialize(service:, job:, dependency_snapshot:, error_handler:) + @service = service + @job = job + @dependency_snapshot = dependency_snapshot + @error_handler = error_handler + end + + def perform + dependency = dependencies.last + check_and_update_pull_request(dependencies) + rescue StandardError => e + error_handler.handle_dependency_error(error: e, dependency: dependency) + end + + private + + attr_reader :job, + :service, + :dependency_snapshot, + :error_handler + + def dependencies + dependency_snapshot.job_dependencies + end + + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/PerceivedComplexity + # rubocop:disable Metrics/MethodLength + def check_and_update_pull_request(dependencies) + if dependencies.count != job.dependencies.count + # If the job dependencies mismatch the parsed dependencies, then + # we should close the PR as at least one thing we changed has been + # removed from the project. + close_pull_request(reason: :dependency_removed) + return + end + + # NOTE: Prevent security only updates from turning into latest version + # updates if the current version is no longer vulnerable. This happens + # when a security update is applied by the user directly and the existing + # pull request is rebased. + if dependencies.none? { |d| job.allowed_update?(d) } + lead_dependency = dependencies.first + if job.vulnerable?(lead_dependency) + Dependabot.logger.info( + "Dependency no longer allowed to update #{lead_dependency.name} #{lead_dependency.version}" + ) + else + Dependabot.logger.info("No longer vulnerable #{lead_dependency.name} #{lead_dependency.version}") + end + close_pull_request(reason: :up_to_date) + return + end + + # The first dependency is the "lead" dependency in a multi-dependency + # update - i.e., the one we're trying to update. + # + # Note: Gradle, Maven and Nuget dependency names can be case-insensitive + # and the dependency name in the security advisory often doesn't match + # what users have specified in their manifest. + lead_dep_name = job.dependencies.first.downcase + lead_dependency = dependencies.find do |dep| + dep.name.downcase == lead_dep_name + end + checker = update_checker_for(lead_dependency) + log_checking_for_update(lead_dependency) + + Dependabot.logger.info("Latest version is #{checker.latest_version}") + + return close_pull_request(reason: :up_to_date) if checker.up_to_date? + + requirements_to_unlock = requirements_to_unlock(checker) + log_requirements_for_update(requirements_to_unlock, checker) + + if requirements_to_unlock == :update_not_possible + return close_pull_request(reason: :update_no_longer_possible) + end + + updated_deps = checker.updated_dependencies( + requirements_to_unlock: requirements_to_unlock + ) + + dependency_change = Dependabot::DependencyChangeBuilder.create_from( + job: job, + dependency_files: dependency_snapshot.dependency_files, + updated_dependencies: updated_deps, + change_source: checker.dependency + ) + + # NOTE: Gradle, Maven and Nuget dependency names can be case-insensitive + # and the dependency name in the security advisory often doesn't match + # what users have specified in their manifest. + job_dependencies = job.dependencies.map(&:downcase) + if dependency_change.updated_dependencies.map { |x| x.name.downcase } != job_dependencies + # The dependencies being updated have changed. Close the existing + # multi-dependency PR and try creating a new one. + close_pull_request(reason: :dependencies_changed) + create_pull_request(dependency_change) + elsif existing_pull_request(dependency_change.updated_dependencies) + # The existing PR is for this version. Update it. + update_pull_request(dependency_change) + else + # The existing PR is for a previous version. Supersede it. + create_pull_request(dependency_change) + end + rescue Dependabot::AllVersionsIgnored + Dependabot.logger.info("All updates for #{dependency.name} were ignored") + + # Report this error to the backend to create an update job error + raise + end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/PerceivedComplexity + # rubocop:enable Metrics/MethodLength + + def requirements_to_unlock(checker) + if !checker.requirements_unlocked_or_can_be? + if checker.can_update?(requirements_to_unlock: :none) then :none + else + :update_not_possible + end + elsif checker.can_update?(requirements_to_unlock: :own) then :own + elsif checker.can_update?(requirements_to_unlock: :all) then :all + else + :update_not_possible + end + end + + def update_checker_for(dependency) + Dependabot::UpdateCheckers.for_package_manager(job.package_manager).new( + dependency: dependency, + dependency_files: dependency_snapshot.dependency_files, + repo_contents_path: job.repo_contents_path, + credentials: job.credentials, + ignored_versions: job.ignore_conditions_for(dependency), + security_advisories: job.security_advisories_for(dependency), + raise_on_ignored: true, + requirements_update_strategy: job.requirements_update_strategy, + options: job.experiments + ) + end + + def log_checking_for_update(dependency) + Dependabot.logger.info( + "Checking if #{dependency.name} #{dependency.version} needs updating" + ) + job.log_ignore_conditions_for(dependency) + end + + def log_up_to_date(dependency) + Dependabot.logger.info( + "No update needed for #{dependency.name} #{dependency.version}" + ) + end + + def log_requirements_for_update(requirements_to_unlock, checker) + Dependabot.logger.info("Requirements to unlock #{requirements_to_unlock}") + + return unless checker.respond_to?(:requirements_update_strategy) + + Dependabot.logger.info( + "Requirements update strategy #{checker.requirements_update_strategy}" + ) + end + + def existing_pull_request(updated_dependencies) + new_pr_set = Set.new( + updated_dependencies.map do |dep| + { + "dependency-name" => dep.name, + "dependency-version" => dep.version, + "dependency-removed" => dep.removed? ? true : nil + }.compact + end + ) + + job.existing_pull_requests.find { |pr| Set.new(pr) == new_pr_set } + end + + def create_pull_request(dependency_change) + Dependabot.logger.info("Submitting #{dependency_change.updated_dependencies.map(&:name).join(', ')} " \ + "pull request for creation") + + service.create_pull_request(dependency_change, dependency_snapshot.base_commit_sha) + end + + def update_pull_request(dependency_change) + Dependabot.logger.info("Submitting #{dependency_change.updated_dependencies.map(&:name).join(', ')} " \ + "pull request for update") + + service.update_pull_request(dependency_change, dependency_snapshot.base_commit_sha) + end + + def close_pull_request(reason:) + reason_string = reason.to_s.tr("_", " ") + Dependabot.logger.info("Telling backend to close pull request for " \ + "#{job.dependencies.join(', ')} - #{reason_string}") + service.close_pull_request(job.dependencies, reason) + end + end + end + end +end diff --git a/updater/lib/dependabot/updater/operations/refresh_version_update_pull_request.rb b/updater/lib/dependabot/updater/operations/refresh_version_update_pull_request.rb new file mode 100644 index 000000000..045490b2c --- /dev/null +++ b/updater/lib/dependabot/updater/operations/refresh_version_update_pull_request.rb @@ -0,0 +1,213 @@ +# frozen_string_literal: true + +# This class implements our strategy for 'refreshing' an existing Pull Request +# that updates a dependnency to the latest permitted version. +# +# It will determine if the existing diff is still relevant, in which case it +# functions similar to a "rebase", but in the case where the project folder's +# dependencies have changed or a newer version is available, it will supersede +# the existing pull request with a new one for clarity. +module Dependabot + class Updater + module Operations + class RefreshVersionUpdatePullRequest + def self.applies_to?(job:) + return false if job.security_updates_only? + # If we haven't been given metadata about the dependencies present + # in the pull request, this strategy cannot act. + return false if job.dependencies&.none? + + job.updating_a_pull_request? + end + + def self.tag_name + :update_version_pr + end + + def initialize(service:, job:, dependency_snapshot:, error_handler:) + @service = service + @job = job + @dependency_snapshot = dependency_snapshot + @error_handler = error_handler + end + + def perform + Dependabot.logger.info("Starting PR update job for #{job.source.repo}") + dependency = dependencies.last + check_and_update_pull_request(dependencies) + rescue StandardError => e + error_handler.handle_dependency_error(error: e, dependency: dependency) + end + + private + + attr_reader :job, + :service, + :dependency_snapshot, + :error_handler, + :created_pull_requests + + def dependencies + dependency_snapshot.job_dependencies + end + + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/PerceivedComplexity + def check_and_update_pull_request(dependencies) + if dependencies.count != job.dependencies.count + # If the job dependencies mismatch the parsed dependencies, then + # we should close the PR as at least one thing we changed has been + # removed from the project. + close_pull_request(reason: :dependency_removed) + return + end + + # The first dependency is the "lead" dependency in a multi-dependency + # update - i.e., the one we're trying to update. + # + # Note: Gradle, Maven and Nuget dependency names can be case-insensitive + # and the dependency name in the security advisory often doesn't match + # what users have specified in their manifest. + lead_dep_name = job.dependencies.first.downcase + lead_dependency = dependencies.find do |dep| + dep.name.downcase == lead_dep_name + end + checker = update_checker_for(lead_dependency, raise_on_ignored: raise_on_ignored?(lead_dependency)) + log_checking_for_update(lead_dependency) + + return if all_versions_ignored?(lead_dependency, checker) + + return close_pull_request(reason: :up_to_date) if checker.up_to_date? + + requirements_to_unlock = requirements_to_unlock(checker) + log_requirements_for_update(requirements_to_unlock, checker) + + if requirements_to_unlock == :update_not_possible + return close_pull_request(reason: :update_no_longer_possible) + end + + updated_deps = checker.updated_dependencies( + requirements_to_unlock: requirements_to_unlock + ) + + dependency_change = Dependabot::DependencyChangeBuilder.create_from( + job: job, + dependency_files: dependency_snapshot.dependency_files, + updated_dependencies: updated_deps, + change_source: checker.dependency + ) + + # NOTE: Gradle, Maven and Nuget dependency names can be case-insensitive + # and the dependency name in the security advisory often doesn't match + # what users have specified in their manifest. + job_dependencies = job.dependencies.map(&:downcase) + if dependency_change.updated_dependencies.map { |x| x.name.downcase } != job_dependencies + # The dependencies being updated have changed. Close the existing + # multi-dependency PR and try creating a new one. + close_pull_request(reason: :dependencies_changed) + create_pull_request(dependency_change) + elsif existing_pull_request(dependency_change.updated_dependencies) + # The existing PR is for this version. Update it. + update_pull_request(dependency_change) + else + # The existing PR is for a previous version. Supersede it. + create_pull_request(dependency_change) + end + end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/PerceivedComplexity + + def create_pull_request(dependency_change) + Dependabot.logger.info("Submitting #{dependency_change.updated_dependencies.map(&:name).join(', ')} " \ + "pull request for creation") + + service.create_pull_request(dependency_change, dependency_snapshot.base_commit_sha) + end + + def update_pull_request(dependency_change) + Dependabot.logger.info("Submitting #{dependency_change.updated_dependencies.map(&:name).join(', ')} " \ + "pull request for update") + + service.update_pull_request(dependency_change, dependency_snapshot.base_commit_sha) + end + + def close_pull_request(reason:) + reason_string = reason.to_s.tr("_", " ") + Dependabot.logger.info("Telling backend to close pull request for " \ + "#{job.dependencies.join(', ')} - #{reason_string}") + service.close_pull_request(job.dependencies, reason) + end + + def raise_on_ignored?(dependency) + job.ignore_conditions_for(dependency).any? + end + + def update_checker_for(dependency, raise_on_ignored:) + Dependabot::UpdateCheckers.for_package_manager(job.package_manager).new( + dependency: dependency, + dependency_files: dependency_snapshot.dependency_files, + repo_contents_path: job.repo_contents_path, + credentials: job.credentials, + ignored_versions: job.ignore_conditions_for(dependency), + security_advisories: job.security_advisories_for(dependency), + raise_on_ignored: raise_on_ignored, + requirements_update_strategy: job.requirements_update_strategy, + options: job.experiments + ) + end + + def log_checking_for_update(dependency) + Dependabot.logger.info( + "Checking if #{dependency.name} #{dependency.version} needs updating" + ) + job.log_ignore_conditions_for(dependency) + end + + def all_versions_ignored?(dependency, checker) + Dependabot.logger.info("Latest version is #{checker.latest_version}") + false + rescue Dependabot::AllVersionsIgnored + Dependabot.logger.info("All updates for #{dependency.name} were ignored") + true + end + + def requirements_to_unlock(checker) + if !checker.requirements_unlocked_or_can_be? + if checker.can_update?(requirements_to_unlock: :none) then :none + else + :update_not_possible + end + elsif checker.can_update?(requirements_to_unlock: :own) then :own + elsif checker.can_update?(requirements_to_unlock: :all) then :all + else + :update_not_possible + end + end + + def log_requirements_for_update(requirements_to_unlock, checker) + Dependabot.logger.info("Requirements to unlock #{requirements_to_unlock}") + + return unless checker.respond_to?(:requirements_update_strategy) + + Dependabot.logger.info( + "Requirements update strategy #{checker.requirements_update_strategy}" + ) + end + + def existing_pull_request(updated_dependencies) + new_pr_set = Set.new( + updated_dependencies.map do |dep| + { + "dependency-name" => dep.name, + "dependency-version" => dep.version, + "dependency-removed" => dep.removed? ? true : nil + }.compact + end + ) + + job.existing_pull_requests.find { |pr| Set.new(pr) == new_pr_set } + end + end + end + end +end diff --git a/updater/lib/dependabot/updater/operations/update_all_versions.rb b/updater/lib/dependabot/updater/operations/update_all_versions.rb new file mode 100644 index 000000000..76fe8972f --- /dev/null +++ b/updater/lib/dependabot/updater/operations/update_all_versions.rb @@ -0,0 +1,257 @@ +# frozen_string_literal: true + +# This class implements our strategy for iterating over all of the dependencies +# for a specific project folder to find those that are out of date and create +# a single PR per Dependency. +module Dependabot + class Updater + module Operations + class UpdateAllVersions + def self.applies_to?(job:) + return false if job.security_updates_only? + return false if job.updating_a_pull_request? + return false if job.dependencies&.any? + + true + end + + def self.tag_name + :update_all_versions + end + + def initialize(service:, job:, dependency_snapshot:, error_handler:) + @service = service + @job = job + @dependency_snapshot = dependency_snapshot + @error_handler = error_handler + # TODO: Collect @created_pull_requests on the Job object? + @created_pull_requests = [] + end + + def perform + Dependabot.logger.info("Starting update job for #{job.source.repo}") + Dependabot.logger.info("Checking all dependencies for version updates...") + dependencies.each { |dep| check_and_create_pr_with_error_handling(dep) } + end + + private + + attr_reader :job, + :service, + :dependency_snapshot, + :error_handler, + :created_pull_requests + + def dependencies + if dependency_snapshot.dependencies.any? && dependency_snapshot.allowed_dependencies.none? + Dependabot.logger.info("Found no dependencies to update after filtering allowed updates") + return [] + end + + if Environment.deterministic_updates? + dependency_snapshot.ungrouped_dependencies + else + dependency_snapshot.ungrouped_dependencies.shuffle + end + end + + def check_and_create_pr_with_error_handling(dependency) + check_and_create_pull_request(dependency) + rescue Dependabot::InconsistentRegistryResponse => e + error_handler.log_dependency_error( + dependency: dependency, + error: e, + error_type: "inconsistent_registry_response", + error_detail: e.message + ) + rescue StandardError => e + error_handler.handle_dependency_error(error: e, dependency: dependency) + end + + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/MethodLength + def check_and_create_pull_request(dependency) + checker = update_checker_for(dependency, raise_on_ignored: raise_on_ignored?(dependency)) + + log_checking_for_update(dependency) + + return if all_versions_ignored?(dependency, checker) + return log_up_to_date(dependency) if checker.up_to_date? + + if pr_exists_for_latest_version?(checker) + return Dependabot.logger.info( + "Pull request already exists for #{checker.dependency.name} " \ + "with latest version #{checker.latest_version}" + ) + end + + requirements_to_unlock = requirements_to_unlock(checker) + log_requirements_for_update(requirements_to_unlock, checker) + + if requirements_to_unlock == :update_not_possible + return Dependabot.logger.info( + "No update possible for #{dependency.name} #{dependency.version}" + ) + end + + updated_deps = checker.updated_dependencies( + requirements_to_unlock: requirements_to_unlock + ) + + if (existing_pr = existing_pull_request(updated_deps)) + deps = existing_pr.map do |dep| + if dep.fetch("dependency-removed", false) + "#{dep.fetch('dependency-name')}@removed" + else + "#{dep.fetch('dependency-name')}@#{dep.fetch('dependency-version')}" + end + end + + return Dependabot.logger.info( + "Pull request already exists for #{deps.join(', ')}" + ) + end + + if peer_dependency_should_update_instead?(checker.dependency.name, updated_deps) + return Dependabot.logger.info( + "No update possible for #{dependency.name} #{dependency.version} " \ + "(peer dependency can be updated)" + ) + end + + dependency_change = Dependabot::DependencyChangeBuilder.create_from( + job: job, + dependency_files: dependency_snapshot.dependency_files, + updated_dependencies: updated_deps, + change_source: checker.dependency + ) + create_pull_request(dependency_change) + end + # rubocop:enable Metrics/MethodLength + # rubocop:enable Metrics/AbcSize + + def log_up_to_date(dependency) + Dependabot.logger.info( + "No update needed for #{dependency.name} #{dependency.version}" + ) + end + + def raise_on_ignored?(dependency) + job.ignore_conditions_for(dependency).any? + end + + def update_checker_for(dependency, raise_on_ignored:) + Dependabot::UpdateCheckers.for_package_manager(job.package_manager).new( + dependency: dependency, + dependency_files: dependency_snapshot.dependency_files, + repo_contents_path: job.repo_contents_path, + credentials: job.credentials, + ignored_versions: job.ignore_conditions_for(dependency), + security_advisories: job.security_advisories_for(dependency), + raise_on_ignored: raise_on_ignored, + requirements_update_strategy: job.requirements_update_strategy, + options: job.experiments + ) + end + + def log_checking_for_update(dependency) + Dependabot.logger.info( + "Checking if #{dependency.name} #{dependency.version} needs updating" + ) + job.log_ignore_conditions_for(dependency) + end + + def all_versions_ignored?(dependency, checker) + Dependabot.logger.info("Latest version is #{checker.latest_version}") + false + rescue Dependabot::AllVersionsIgnored + Dependabot.logger.info("All updates for #{dependency.name} were ignored") + true + end + + def pr_exists_for_latest_version?(checker) + latest_version = checker.latest_version&.to_s + return false if latest_version.nil? + + job.existing_pull_requests. + select { |pr| pr.count == 1 }. + map(&:first). + select { |pr| pr.fetch("dependency-name") == checker.dependency.name }. + any? { |pr| pr.fetch("dependency-version", nil) == latest_version } + end + + def existing_pull_request(updated_dependencies) + new_pr_set = Set.new( + updated_dependencies.map do |dep| + { + "dependency-name" => dep.name, + "dependency-version" => dep.version, + "dependency-removed" => dep.removed? ? true : nil + }.compact + end + ) + + job.existing_pull_requests.find { |pr| Set.new(pr) == new_pr_set } || + created_pull_requests.find { |pr| Set.new(pr) == new_pr_set } + end + + def requirements_to_unlock(checker) + if !checker.requirements_unlocked_or_can_be? + if checker.can_update?(requirements_to_unlock: :none) then :none + else + :update_not_possible + end + elsif checker.can_update?(requirements_to_unlock: :own) then :own + elsif checker.can_update?(requirements_to_unlock: :all) then :all + else + :update_not_possible + end + end + + def log_requirements_for_update(requirements_to_unlock, checker) + Dependabot.logger.info("Requirements to unlock #{requirements_to_unlock}") + + return unless checker.respond_to?(:requirements_update_strategy) + + Dependabot.logger.info( + "Requirements update strategy #{checker.requirements_update_strategy}" + ) + end + + # If a version update for a peer dependency is possible we should + # defer to the PR that will be created for it to avoid duplicate PRs. + def peer_dependency_should_update_instead?(dependency_name, updated_deps) + updated_deps. + reject { |dep| dep.name == dependency_name }. + any? do |dep| + next true if existing_pull_request([dep]) + + original_peer_dep = ::Dependabot::Dependency.new( + name: dep.name, + version: dep.previous_version, + requirements: dep.previous_requirements, + package_manager: dep.package_manager + ) + update_checker_for(original_peer_dep, raise_on_ignored: false). + can_update?(requirements_to_unlock: :own) + end + end + + def create_pull_request(dependency_change) + Dependabot.logger.info("Submitting #{dependency_change.updated_dependencies.map(&:name).join(', ')} " \ + "pull request for creation") + + service.create_pull_request(dependency_change, dependency_snapshot.base_commit_sha) + + created_pull_requests << dependency_change.updated_dependencies.map do |dep| + { + "dependency-name" => dep.name, + "dependency-version" => dep.version, + "dependency-removed" => dep.removed? ? true : nil + }.compact + end + end + end + end + end +end diff --git a/updater/lib/dependabot/updater/security_update_helpers.rb b/updater/lib/dependabot/updater/security_update_helpers.rb new file mode 100644 index 000000000..fdffe8a34 --- /dev/null +++ b/updater/lib/dependabot/updater/security_update_helpers.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: true + +# This module extracts all helpers required to perform additional update job +# error recording and logging for Security Updates since they are shared +# between a few operations. +module Dependabot + class Updater + module SecurityUpdateHelpers + def record_security_update_not_needed_error(checker) + Dependabot.logger.info( + "no security update needed as #{checker.dependency.name} " \ + "is no longer vulnerable" + ) + + service.record_update_job_error( + error_type: "security_update_not_needed", + error_details: { + "dependency-name": checker.dependency.name + } + ) + end + + def record_security_update_ignored(checker) + Dependabot.logger.info( + "Dependabot cannot update to the required version as all versions " \ + "were ignored for #{checker.dependency.name}" + ) + + service.record_update_job_error( + error_type: "all_versions_ignored", + error_details: { + "dependency-name": checker.dependency.name + } + ) + end + + def record_dependency_file_not_supported_error(checker) + Dependabot.logger.info( + "Dependabot can't update vulnerable dependencies for projects " \ + "without a lockfile or pinned version requirement as the currently " \ + "installed version of #{checker.dependency.name} isn't known." + ) + + service.record_update_job_error( + error_type: "dependency_file_not_supported", + error_details: { + "dependency-name": checker.dependency.name + } + ) + end + + def record_security_update_not_possible_error(checker) + latest_allowed_version = + (checker.lowest_resolvable_security_fix_version || + checker.dependency.version)&.to_s + lowest_non_vulnerable_version = + checker.lowest_security_fix_version&.to_s + conflicting_dependencies = checker.conflicting_dependencies + + Dependabot.logger.info( + security_update_not_possible_message(checker, latest_allowed_version, conflicting_dependencies) + ) + Dependabot.logger.info( + earliest_fixed_version_message(lowest_non_vulnerable_version) + ) + + service.record_update_job_error( + error_type: "security_update_not_possible", + error_details: { + "dependency-name": checker.dependency.name, + "latest-resolvable-version": latest_allowed_version, + "lowest-non-vulnerable-version": lowest_non_vulnerable_version, + "conflicting-dependencies": conflicting_dependencies + } + ) + end + + def record_security_update_not_found(checker) + Dependabot.logger.info( + "Dependabot can't find a published or compatible non-vulnerable " \ + "version for #{checker.dependency.name}. " \ + "The latest available version is #{checker.dependency.version}" + ) + + service.record_update_job_error( + error_type: "security_update_not_found", + error_details: { + "dependency-name": checker.dependency.name, + "dependency-version": checker.dependency.version + }, + dependency: checker.dependency + ) + end + + def record_pull_request_exists_for_latest_version(checker) + service.record_update_job_error( + error_type: "pull_request_exists_for_latest_version", + error_details: { + "dependency-name": checker.dependency.name, + "dependency-version": checker.latest_version&.to_s + }, + dependency: checker.dependency + ) + end + + def record_pull_request_exists_for_security_update(existing_pull_request) + updated_dependencies = existing_pull_request.map do |dep| + { + "dependency-name": dep.fetch("dependency-name"), + "dependency-version": dep.fetch("dependency-version", nil), + "dependency-removed": dep.fetch("dependency-removed", nil) + }.compact + end + + service.record_update_job_error( + error_type: "pull_request_exists_for_security_update", + error_details: { + "updated-dependencies": updated_dependencies + } + ) + end + + def record_security_update_dependency_not_found + service.record_update_job_error( + error_type: "security_update_dependency_not_found", + error_details: {} + ) + end + + def earliest_fixed_version_message(lowest_non_vulnerable_version) + if lowest_non_vulnerable_version + "The earliest fixed version is #{lowest_non_vulnerable_version}." + else + "Dependabot could not find a non-vulnerable version" + end + end + + def security_update_not_possible_message(checker, latest_allowed_version, + conflicting_dependencies) + if conflicting_dependencies.any? + dep_messages = conflicting_dependencies.map do |dep| + " #{dep['explanation']}" + end.join("\n") + + dependencies_pluralized = + conflicting_dependencies.count > 1 ? "dependencies" : "dependency" + + "The latest possible version that can be installed is " \ + "#{latest_allowed_version} because of the following " \ + "conflicting #{dependencies_pluralized}:\n\n#{dep_messages}" + else + "The latest possible version of #{checker.dependency.name} that can " \ + "be installed is #{latest_allowed_version}" + end + end + end + end +end diff --git a/updater/spec/dependabot/api_client_spec.rb b/updater/spec/dependabot/api_client_spec.rb new file mode 100644 index 000000000..ca0797aa9 --- /dev/null +++ b/updater/spec/dependabot/api_client_spec.rb @@ -0,0 +1,436 @@ +# frozen_string_literal: true + +require "spec_helper" +require "dependabot/dependency" +require "dependabot/dependency_change" +require "dependabot/api_client" + +RSpec.describe Dependabot::ApiClient do + subject(:client) { Dependabot::ApiClient.new("http://example.com", 1, "token") } + let(:headers) { { "Content-Type" => "application/json" } } + + describe "create_pull_request" do + let(:dependency_change) do + Dependabot::DependencyChange.new( + job: job, + updated_dependencies: dependencies, + updated_dependency_files: dependency_files + ) + end + let(:job) do + instance_double(Dependabot::Job, + source: nil, + credentials: [], + commit_message_options: [], + updating_a_pull_request?: false, + ignore_conditions: []) + end + let(:dependencies) do + [dependency] + end + let(:dependency) do + Dependabot::Dependency.new( + name: "business", + package_manager: "bundler", + version: "1.8.0", + previous_version: "1.7.0", + requirements: [ + { file: "Gemfile", requirement: "~> 1.8.0", groups: [], source: nil } + ], + previous_requirements: [ + { file: "Gemfile", requirement: "~> 1.7.0", groups: [], source: nil } + ] + ) + end + let(:dependency_files) do + [ + Dependabot::DependencyFile.new( + name: "Gemfile", + content: "some things", + directory: "/" + ), + Dependabot::DependencyFile.new( + name: "Gemfile.lock", + content: "more things", + directory: "/" + ) + ] + end + let(:create_pull_request_url) do + "http://example.com/update_jobs/1/create_pull_request" + end + let(:base_commit) { "sha" } + let(:message) do + Dependabot::PullRequestCreator::Message.new( + pr_name: "PR name", + pr_message: "PR message", + commit_message: "Commit message" + ) + end + + before do + allow(Dependabot::PullRequestCreator::MessageBuilder).to receive_message_chain(:new, :message).and_return(message) + + stub_request(:post, create_pull_request_url). + to_return(status: 204, headers: headers) + end + + it "hits the correct endpoint" do + client.create_pull_request(dependency_change, base_commit) + + expect(WebMock). + to have_requested(:post, create_pull_request_url). + with(headers: { "Authorization" => "token" }) + end + + it "encodes the payload correctly fields" do + client.create_pull_request(dependency_change, base_commit) + + expect(WebMock).to(have_requested(:post, create_pull_request_url).with do |req| + data = JSON.parse(req.body)["data"] + + expect(data["dependencies"]).to eq([ + { + "name" => "business", + "previous-requirements" => + [ + { + "file" => "Gemfile", + "groups" => [], + "requirement" => "~> 1.7.0", + "source" => nil + } + ], + "previous-version" => "1.7.0", + "requirements" => + [ + { + "file" => "Gemfile", + "groups" => [], + "requirement" => "~> 1.8.0", + "source" => nil + } + ], + "version" => "1.8.0" + } + ]) + expect(data["updated-dependency-files"]).to eql([ + { + "content" => "some things", + "content_encoding" => "utf-8", + "deleted" => false, + "directory" => "/", + "mode" => "100644", + "name" => "Gemfile", + "operation" => "update", + "support_file" => false, + "type" => "file" + }, + { "content" => "more things", + "content_encoding" => "utf-8", + "deleted" => false, + "directory" => "/", + "mode" => "100644", + "name" => "Gemfile.lock", + "operation" => "update", + "support_file" => false, + "type" => "file" } + ]) + expect(data["base-commit-sha"]).to eql("sha") + expect(data["commit-message"]).to eq("Commit message") + expect(data["pr-title"]).to eq("PR name") + expect(data["pr-body"]).to eq("PR message") + end) + end + + context "with a removed dependency" do + let(:removed_dependency) do + Dependabot::Dependency.new( + name: "removed", + package_manager: "bundler", + previous_version: "1.7.0", + requirements: [], + previous_requirements: [], + removed: true + ) + end + + let(:dependencies) do + [removed_dependency, dependency] + end + + it "encodes fields" do + client.create_pull_request(dependency_change, base_commit) + expect(WebMock). + to(have_requested(:post, create_pull_request_url). + with(headers: { "Authorization" => "token" }). + with do |req| + data = JSON.parse(req.body)["data"] + expect(data["dependencies"].first["removed"]).to eq(true) + expect(data["dependencies"].first.key?("version")).to eq(false) + expect(data["dependencies"].last.key?("removed")).to eq(false) + expect(data["dependencies"].last["version"]).to eq("1.8.0") + true + end) + end + end + + context "grouped updates" do + it "does not include the dependency-group key by default" do + client.create_pull_request(dependency_change, base_commit) + + expect(WebMock). + to(have_requested(:post, create_pull_request_url). + with do |req| + expect(req.body).not_to include("dependency-group") + end) + end + + it "flags the PR as having dependency-groups if the dependency change has a dependency group assigned" do + group = Dependabot::DependencyGroup.new(name: "dummy-group-name", rules: { patterns: ["*"] }) + + grouped_dependency_change = Dependabot::DependencyChange.new( + job: job, + updated_dependencies: dependencies, + updated_dependency_files: dependency_files, + dependency_group: group + ) + + client.create_pull_request(grouped_dependency_change, base_commit) + + expect(WebMock). + to(have_requested(:post, create_pull_request_url). + with do |req| + data = JSON.parse(req.body)["data"] + expect(data["dependency-group"]).to eq({ "name" => "dummy-group-name" }) + end) + end + end + end + + describe "update_pull_request" do + let(:dependency_change) do + Dependabot::DependencyChange.new( + job: job, + updated_dependencies: [dependency], + updated_dependency_files: dependency_files + ) + end + let(:job) do + instance_double(Dependabot::Job, + source: nil, + credentials: [], + commit_message_options: [], + updating_a_pull_request?: true) + end + let(:dependency) do + Dependabot::Dependency.new( + name: "business", + package_manager: "bundler", + version: "1.8.0", + previous_version: "1.7.0", + requirements: [ + { file: "Gemfile", requirement: "~> 1.8.0", groups: [], source: nil } + ], + previous_requirements: [ + { file: "Gemfile", requirement: "~> 1.7.0", groups: [], source: nil } + ] + ) + end + let(:dependency_files) do + [ + Dependabot::DependencyFile.new( + name: "Gemfile", + content: "some things", + directory: "/" + ), + Dependabot::DependencyFile.new( + name: "Gemfile.lock", + content: "more things", + directory: "/" + ) + ] + end + let(:update_pull_request_url) do + "http://example.com/update_jobs/1/update_pull_request" + end + let(:base_commit) { "sha" } + + before do + stub_request(:post, update_pull_request_url). + to_return(status: 204, headers: headers) + end + + it "hits the correct endpoint" do + client.update_pull_request(dependency_change, base_commit) + + expect(WebMock). + to have_requested(:post, update_pull_request_url). + with(headers: { "Authorization" => "token" }) + end + + it "does not encode the pull request fields" do + expect(Dependabot::PullRequestCreator::MessageBuilder).not_to receive(:new) + + client.update_pull_request(dependency_change, base_commit) + + expect(WebMock). + to(have_requested(:post, update_pull_request_url).with do |req| + data = JSON.parse(req.body)["data"] + + expect(data["dependency-names"]).to eq(["business"]) + expect(data["updated-dependency-files"]).to eql([ + { + "content" => "some things", + "content_encoding" => "utf-8", + "deleted" => false, + "directory" => "/", + "mode" => "100644", + "name" => "Gemfile", + "operation" => "update", + "support_file" => false, + "type" => "file" + }, + { "content" => "more things", + "content_encoding" => "utf-8", + "deleted" => false, + "directory" => "/", + "mode" => "100644", + "name" => "Gemfile.lock", + "operation" => "update", + "support_file" => false, + "type" => "file" } + ]) + expect(data["base-commit-sha"]).to eql("sha") + expect(data).not_to have_key("commit-message") + expect(data).not_to have_key("pr-title") + expect(data).not_to have_key("pr-body") + expect(data).not_to have_key("grouped-update") + end) + end + end + + describe "close_pull_request" do + let(:dependency_name) { "business" } + let(:close_pull_request_url) do + "http://example.com/update_jobs/1/close_pull_request" + end + + before do + stub_request(:post, close_pull_request_url). + to_return(status: 204, headers: headers) + end + + it "hits the correct endpoint" do + client.close_pull_request(dependency_name, :dependency_removed) + + expect(WebMock). + to have_requested(:post, close_pull_request_url). + with(headers: { "Authorization" => "token" }) + end + end + + describe "record_update_job_error" do + let(:url) { "http://example.com/update_jobs/1/record_update_job_error" } + let(:error_type) { "dependency_file_not_evaluatable" } + let(:error_detail) { { "message" => "My message" } } + before { stub_request(:post, url).to_return(status: 204) } + + it "hits the correct endpoint" do + client.record_update_job_error( + error_type: error_type, + error_details: error_detail + ) + + expect(WebMock). + to have_requested(:post, url). + with(headers: { "Authorization" => "token" }) + end + end + + describe "mark_job_as_processed" do + let(:url) { "http://example.com/update_jobs/1/mark_as_processed" } + let(:base_commit) { "sha" } + before { stub_request(:patch, url).to_return(status: 204) } + + it "hits the correct endpoint" do + client.mark_job_as_processed(base_commit) + + expect(WebMock). + to have_requested(:patch, url). + with(headers: { "Authorization" => "token" }) + end + end + + describe "update_dependency_list" do + let(:url) { "http://example.com/update_jobs/1/update_dependency_list" } + let(:dependency) do + Dependabot::Dependency.new( + name: "business", + package_manager: "bundler", + version: "1.8.0", + requirements: [ + { file: "Gemfile", requirement: "~> 1.8.0", groups: [], source: nil } + ] + ) + end + before { stub_request(:post, url).to_return(status: 204) } + + it "hits the correct endpoint" do + client.update_dependency_list([dependency], ["Gemfile"]) + + expect(WebMock). + to have_requested(:post, url). + with(headers: { "Authorization" => "token" }) + end + end + + describe "ecosystem_versions" do + let(:url) { "http://example.com/update_jobs/1/record_ecosystem_versions" } + before { stub_request(:post, url).to_return(status: 204) } + + it "hits the correct endpoint" do + client.record_ecosystem_versions({ "ruby" => { "min" => 3, "max" => 3.2 } }) + + expect(WebMock). + to have_requested(:post, url). + with(headers: { "Authorization" => "token" }) + end + end + + describe "increment_metric" do + let(:url) { "http://example.com/update_jobs/1/increment_metric" } + before { stub_request(:post, url).to_return(status: 204) } + + context "when successful" do + before { stub_request(:post, url).to_return(status: 204) } + + it "hits the expected endpoint" do + client.increment_metric("apples", tags: { red: 1, green: 2 }) + + expect(WebMock). + to have_requested(:post, url). + with(headers: { "Authorization" => "token" }) + end + end + + context "when unsuccessful" do + before do + stub_request(:post, url).to_return(status: 401) + allow(Dependabot.logger).to receive(:debug) + end + + it "logs a debug notice" do + client.increment_metric("apples", tags: { red: 1, green: 2 }) + + expect(WebMock). + to have_requested(:post, url). + with(headers: { "Authorization" => "token" }) + + expect(Dependabot.logger).to have_received(:debug).with( + "Unable to report metric 'apples'." + ) + end + end + end +end diff --git a/updater/spec/dependabot/dependency_change_spec.rb b/updater/spec/dependabot/dependency_change_spec.rb new file mode 100644 index 000000000..484dc92d5 --- /dev/null +++ b/updater/spec/dependabot/dependency_change_spec.rb @@ -0,0 +1,159 @@ +# frozen_string_literal: true + +require "spec_helper" +require "dependabot/dependency_change" +require "dependabot/job" + +RSpec.describe Dependabot::DependencyChange do + subject(:dependency_change) do + described_class.new( + job: job, + updated_dependencies: updated_dependencies, + updated_dependency_files: updated_dependency_files + ) + end + + let(:job) do + instance_double(Dependabot::Job, ignore_conditions: []) + end + + let(:updated_dependencies) do + [ + Dependabot::Dependency.new( + name: "business", + package_manager: "bundler", + version: "1.8.0", + previous_version: "1.7.0", + requirements: [ + { file: "Gemfile", requirement: "~> 1.8.0", groups: [], source: nil } + ], + previous_requirements: [ + { file: "Gemfile", requirement: "~> 1.7.0", groups: [], source: nil } + ] + ) + ] + end + + let(:updated_dependency_files) do + [ + Dependabot::DependencyFile.new( + name: "Gemfile", + content: fixture("bundler/original/Gemfile"), + directory: "/" + ), + Dependabot::DependencyFile.new( + name: "Gemfile.lock", + content: fixture("bundler/original/Gemfile.lock"), + directory: "/" + ) + ] + end + + describe "#pr_message" do + let(:github_source) do + Dependabot::Source.new( + provider: "github", + repo: "dependabot-fixtures/dependabot-test-ruby-package", + directory: "/", + branch: nil, + api_endpoint: "https://api.github.com/", + hostname: "github.com" + ) + end + + let(:job_credentials) do + [ + { + "type" => "git_source", + "host" => "github.com", + "username" => "x-access-token", + "password" => "github-token" + }, + { "type" => "random", "secret" => "codes" } + ] + end + + let(:commit_message_options) do + { + include_scope: true, + prefix: "[bump]", + prefix_development: "[bump-dev]" + } + end + + let(:message_builder_mock) do + instance_double(Dependabot::PullRequestCreator::MessageBuilder, message: "Hello World!") + end + + before do + allow(job).to receive(:source).and_return(github_source) + allow(job).to receive(:credentials).and_return(job_credentials) + allow(job).to receive(:commit_message_options).and_return(commit_message_options) + allow(Dependabot::PullRequestCreator::MessageBuilder).to receive(:new).and_return(message_builder_mock) + end + + it "delegates to the Dependabot::PullRequestCreator::MessageBuilder with the correct configuration" do + expect(Dependabot::PullRequestCreator::MessageBuilder). + to receive(:new).with( + source: github_source, + files: updated_dependency_files, + dependencies: updated_dependencies, + credentials: job_credentials, + commit_message_options: commit_message_options, + dependency_group: nil, + pr_message_encoding: nil, + pr_message_max_length: 65_535, + ignore_conditions: [] + ) + + expect(dependency_change.pr_message).to eql("Hello World!") + end + + context "when a dependency group is assigned" do + it "delegates to the Dependabot::PullRequestCreator::MessageBuilder with the group included" do + group = Dependabot::DependencyGroup.new(name: "foo", rules: { patterns: ["*"] }) + + dependency_change = described_class.new( + job: job, + updated_dependencies: updated_dependencies, + updated_dependency_files: updated_dependency_files, + dependency_group: group + ) + + expect(Dependabot::PullRequestCreator::MessageBuilder). + to receive(:new).with( + source: github_source, + files: updated_dependency_files, + dependencies: updated_dependencies, + credentials: job_credentials, + commit_message_options: commit_message_options, + dependency_group: group, + pr_message_encoding: nil, + pr_message_max_length: 65_535, + ignore_conditions: [] + ) + + expect(dependency_change.pr_message).to eql("Hello World!") + end + end + end + + describe "#grouped_update?" do + it "is false by default" do + expect(dependency_change.grouped_update?).to be false + end + + context "when a dependency group is assigned" do + it "is true" do + dependency_change = described_class.new( + job: job, + updated_dependencies: updated_dependencies, + updated_dependency_files: updated_dependency_files, + dependency_group: Dependabot::DependencyGroup.new(name: "foo", rules: { patterns: ["*"] }) + ) + + expect(dependency_change.grouped_update?).to be true + end + end + end +end diff --git a/updater/spec/dependabot/dependency_group_engine_spec.rb b/updater/spec/dependabot/dependency_group_engine_spec.rb new file mode 100644 index 000000000..f1744b256 --- /dev/null +++ b/updater/spec/dependabot/dependency_group_engine_spec.rb @@ -0,0 +1,246 @@ +# frozen_string_literal: true + +require "spec_helper" +require "support/dependency_file_helpers" + +require "dependabot/dependency_group_engine" +require "dependabot/dependency_snapshot" +require "dependabot/job" + +RSpec.describe Dependabot::DependencyGroupEngine do + include DependencyFileHelpers + + let(:dependency_group_engine) { described_class.from_job_config(job: job) } + + let(:job) do + instance_double(Dependabot::Job, dependency_groups: dependency_groups_config) + end + + let(:dummy_pkg_a) do + Dependabot::Dependency.new( + name: "dummy-pkg-a", + package_manager: "bundler", + version: "1.1.0", + requirements: [ + { + file: "Gemfile", + requirement: "~> 1.1.0", + groups: ["default"], + source: nil + } + ] + ) + end + + let(:dummy_pkg_b) do + Dependabot::Dependency.new( + name: "dummy-pkg-b", + package_manager: "bundler", + version: "1.1.0", + requirements: [ + { + file: "Gemfile", + requirement: "~> 1.1.0", + groups: ["default"], + source: nil + } + ] + ) + end + + let(:dummy_pkg_c) do + Dependabot::Dependency.new( + name: "dummy-pkg-c", + package_manager: "bundler", + version: "1.1.0", + requirements: [ + { + file: "Gemfile", + requirement: "~> 1.1.0", + groups: ["default"], + source: nil + } + ] + ) + end + + let(:ungrouped_pkg) do + Dependabot::Dependency.new( + name: "ungrouped_pkg", + package_manager: "bundler", + version: "1.1.0", + requirements: [ + { + file: "Gemfile", + requirement: "~> 1.1.0", + groups: ["default"], + source: nil + } + ] + ) + end + + context "when a job has groups configured" do + let(:dependency_groups_config) do + [ + { + "name" => "group-a", + "rules" => { + "patterns" => ["dummy-pkg-*"], + "exclude-patterns" => ["dummy-pkg-b"] + } + }, + { + "name" => "group-b", + "rules" => { + "patterns" => %w(dummy-pkg-b dummy-pkg-c) + } + } + ] + end + + describe "::from_job_config" do + it "registers the dependency groups" do + expect(dependency_group_engine.dependency_groups.length).to eql(2) + expect(dependency_group_engine.dependency_groups.map(&:name)).to eql(%w(group-a group-b)) + expect(dependency_group_engine.dependency_groups.map(&:dependencies)).to all(be_empty) + end + end + + describe "#find_group" do + it "retrieves a defined group by name" do + group_a = dependency_group_engine.find_group(name: "group-a") + expect(group_a.rules).to eql({ + "patterns" => ["dummy-pkg-*"], + "exclude-patterns" => ["dummy-pkg-b"] + }) + end + + it "returns nil if the group does not exist" do + expect(dependency_group_engine.find_group(name: "no-such-thing")).to be_nil + end + end + + describe "#assign_to_groups!" do + context "when all groups have at least one dependency that matches" do + let(:dependencies) { [dummy_pkg_a, dummy_pkg_b, dummy_pkg_c, ungrouped_pkg] } + + before do + dependency_group_engine.assign_to_groups!(dependencies: dependencies) + end + + it "adds dependencies to every group they match" do + group_a = dependency_group_engine.find_group(name: "group-a") + expect(group_a.dependencies).to eql([dummy_pkg_a, dummy_pkg_c]) + + group_b = dependency_group_engine.find_group(name: "group-b") + expect(group_b.dependencies).to eql([dummy_pkg_b, dummy_pkg_c]) + end + + it "keeps a list of any dependencies that do not match any groups" do + expect(dependency_group_engine.ungrouped_dependencies).to eql([ungrouped_pkg]) + end + + it "raises an exception if it is called a second time" do + expect { dependency_group_engine.assign_to_groups!(dependencies: dependencies) }. + to raise_error(described_class::ConfigurationError, "dependency groups have already been configured!") + end + end + + context "when one group has no matching dependencies" do + let(:dependencies) { [dummy_pkg_a] } + + it "warns that the group is misconfigured" do + expect(Dependabot.logger).to receive(:warn).with( + <<~WARN + Please check your configuration as there are groups where no dependencies match: + - group-b + + This can happen if: + - the group's 'pattern' rules are mispelled + - your configuration's 'allow' rules do not permit any of the dependencies that match the group + - the dependencies that match the group rules have been removed from your project + WARN + ) + + dependency_group_engine.assign_to_groups!(dependencies: dependencies) + end + end + + context "when no groups have any matching dependencies" do + let(:dependencies) { [ungrouped_pkg] } + + it "warns that the groups are misconfigured" do + expect(Dependabot.logger).to receive(:warn).with( + <<~WARN + Please check your configuration as there are groups where no dependencies match: + - group-a + - group-b + + This can happen if: + - the group's 'pattern' rules are mispelled + - your configuration's 'allow' rules do not permit any of the dependencies that match the group + - the dependencies that match the group rules have been removed from your project + WARN + ) + + dependency_group_engine.assign_to_groups!(dependencies: dependencies) + end + end + end + + context "when a job has no groups configured" do + let(:dependency_groups_config) { [] } + + describe "::from_job_config" do + it "registers no dependency groups" do + expect(dependency_group_engine.dependency_groups).to be_empty + end + end + + describe "#assign_to_groups!" do + let(:dummy_pkg_a) do + Dependabot::Dependency.new( + name: "dummy-pkg-a", + package_manager: "bundler", + version: "1.1.0", + requirements: [ + { + file: "Gemfile", + requirement: "~> 1.1.0", + groups: ["default"], + source: nil + } + ] + ) + end + + let(:dummy_pkg_b) do + Dependabot::Dependency.new( + name: "dummy-pkg-b", + package_manager: "bundler", + version: "1.1.0", + requirements: [ + { + file: "Gemfile", + requirement: "~> 1.1.0", + groups: ["default"], + source: nil + } + ] + ) + end + + let(:dependencies) { [dummy_pkg_a, dummy_pkg_b] } + + before do + dependency_group_engine.assign_to_groups!(dependencies: dependencies) + end + + it "lists all dependencies as ungrouped" do + expect(dependency_group_engine.ungrouped_dependencies).to eql(dependencies) + end + end + end + end +end diff --git a/updater/spec/dependabot/job_spec.rb b/updater/spec/dependabot/job_spec.rb new file mode 100644 index 000000000..6757f6d36 --- /dev/null +++ b/updater/spec/dependabot/job_spec.rb @@ -0,0 +1,501 @@ +# frozen_string_literal: true + +require "spec_helper" +require "dependabot/job" +require "dependabot/dependency" +require "dependabot/bundler" + +RSpec.describe Dependabot::Job do + subject(:job) { described_class.new(attributes) } + + let(:attributes) do + { + id: 1, + token: "token", + dependencies: dependencies, + allowed_updates: allowed_updates, + existing_pull_requests: [], + ignore_conditions: [], + security_advisories: security_advisories, + package_manager: package_manager, + source: { + "provider" => "github", + "repo" => "dependabot-fixtures/dependabot-test-ruby-package", + "directory" => "/", + "api-endpoint" => "https://api.github.com/", + "hostname" => "github.com", + "branch" => nil + }, + credentials: [{ + "type" => "git_source", + "host" => "github.com", + "username" => "x-access-token", + "password" => "github-token" + }], + lockfile_only: lockfile_only, + requirements_update_strategy: nil, + update_subdependencies: false, + updating_a_pull_request: false, + vendor_dependencies: vendor_dependencies, + experiments: experiments, + commit_message_options: commit_message_options, + security_updates_only: security_updates_only, + dependency_groups: dependency_groups, + repo_private: repo_private + } + end + + let(:dependencies) { nil } + let(:security_advisories) { [] } + let(:package_manager) { "bundler" } + let(:lockfile_only) { false } + let(:security_updates_only) { false } + let(:allowed_updates) do + [ + { + "dependency-type" => "direct", + "update-type" => "all" + }, + { + "dependency-type" => "indirect", + "update-type" => "security" + } + ] + end + let(:experiments) { nil } + let(:commit_message_options) { nil } + let(:vendor_dependencies) { false } + let(:dependency_groups) { [] } + let(:repo_private) { false } + + describe "::new_update_job" do + let(:job_json) { fixture("jobs/job_with_credentials.json") } + + let(:new_update_job) do + described_class.new_update_job( + job_id: anything, + job_definition: JSON.parse(job_json), + repo_contents_path: anything + ) + end + + it "correctly replaces the credentials with the credential-metadata" do + expect(new_update_job.credentials.length).to eql(2) + + git_credential = new_update_job.credentials.find { |creds| creds["type"] == "git_source" } + expect(git_credential["host"]).to eql("github.com") + expect(git_credential.keys).not_to include("username", "password") + + ruby_credential = new_update_job.credentials.find { |creds| creds["type"] == "rubygems_index" } + expect(ruby_credential["host"]).to eql("my.rubygems-host.org") + expect(ruby_credential.keys).not_to include("token") + end + end + + context "when lockfile_only is passed as true" do + let(:lockfile_only) { true } + + it "infers a lockfile_only requirements_update_strategy" do + expect(subject.requirements_update_strategy).to eq("lockfile_only") + end + end + + describe "#allowed_update?" do + subject { job.allowed_update?(dependency) } + let(:dependency) do + Dependabot::Dependency.new( + name: dependency_name, + package_manager: "bundler", + version: "1.8.0", + requirements: requirements + ) + end + let(:dependency_name) { "business" } + let(:requirements) do + [{ file: "Gemfile", requirement: "~> 1.8.0", groups: [], source: nil }] + end + + context "with default allowed updates on a dependency with no requirements" do + let(:allowed_updates) do + [ + { + "dependency-type" => "direct", + "update-type" => "all" + } + ] + end + let(:security_advisories) do + [ + { + "dependency-name" => dependency_name, + "affected-versions" => [], + "patched-versions" => ["~> 1.11.0"], + "unaffected-versions" => [] + } + ] + end + let(:dependency) do + Dependabot::Dependency.new( + name: dependency_name, + package_manager: "bundler", + version: "1.8.0", + requirements: [] + ) + end + it { is_expected.to eq(false) } + + context "for a security update" do + let(:security_updates_only) { true } + it { is_expected.to eq(true) } + end + end + + context "with a top-level dependency" do + let(:requirements) do + [{ file: "Gemfile", requirement: "~> 1.8.0", groups: [], source: nil }] + end + + it { is_expected.to eq(true) } + end + + context "with a sub-dependency" do + let(:requirements) { [] } + it { is_expected.to eq(false) } + + context "that is insecure" do + let(:security_advisories) do + [ + { + "dependency-name" => "business", + "affected-versions" => [], + "patched-versions" => ["~> 1.11.0"], + "unaffected-versions" => [] + } + ] + end + + it { is_expected.to eq(true) } + end + end + + context "when only security fixes are allowed" do + let(:security_updates_only) { true } + it { is_expected.to eq(false) } + + context "for a security fix" do + let(:security_advisories) do + [ + { + "dependency-name" => "business", + "affected-versions" => [], + "patched-versions" => ["~> 1.11.0"], + "unaffected-versions" => [] + } + ] + end + + it { is_expected.to eq(true) } + end + + context "for a security fix that doesn't apply" do + let(:security_advisories) do + [ + { + "dependency-name" => "business", + "affected-versions" => ["> 1.8.0"], + "patched-versions" => [], + "unaffected-versions" => [] + } + ] + end + + it { is_expected.to eq(false) } + end + + context "for a security fix that doesn't apply to some versions" do + let(:security_advisories) do + [ + { + "dependency-name" => "business", + "affected-versions" => ["> 1.8.0"], + "patched-versions" => [], + "unaffected-versions" => [] + } + ] + end + + it "should be allowed" do + dependency.metadata[:all_versions] = [ + Dependabot::Dependency.new( + name: dependency_name, + package_manager: "bundler", + version: "1.8.0", + requirements: [] + ), + Dependabot::Dependency.new( + name: dependency_name, + package_manager: "bundler", + version: "1.9.0", + requirements: [] + ) + ] + + is_expected.to eq(true) + end + end + end + + context "and a dependency whitelist that includes the dependency" do + let(:allowed_updates) { [{ "dependency-name" => "business" }] } + it { is_expected.to eq(true) } + + context "with a dependency whitelist that uses a wildcard" do + let(:allowed_updates) { [{ "dependency-name" => "bus*" }] } + it { is_expected.to eq(true) } + end + end + + context "and a dependency whitelist that excludes the dependency" do + let(:allowed_updates) { [{ "dependency-name" => "rails" }] } + it { is_expected.to eq(false) } + + context "that would match if we were sloppy about substrings" do + let(:allowed_updates) { [{ "dependency-name" => "bus" }] } + it { is_expected.to eq(false) } + end + + context "with a dependency whitelist that uses a wildcard" do + let(:allowed_updates) { [{ "dependency-name" => "b.ness*" }] } + it { is_expected.to eq(false) } + end + + context "when security fixes are also allowed" do + let(:allowed_updates) do + [ + { "dependency-name" => "rails" }, + { "update-type" => "security" } + ] + end + + it { is_expected.to eq(false) } + + context "for a security fix" do + let(:security_advisories) do + [ + { + "dependency-name" => "business", + "affected-versions" => [], + "patched-versions" => ["~> 1.11.0"], + "unaffected-versions" => [] + } + ] + end + + it { is_expected.to eq(true) } + end + end + end + + context "with dev dependencies during a security update while allowed: production is in effect" do + let(:package_manager) { "npm_and_yarn" } + let(:security_updates_only) { true } + let(:dependency) do + Dependabot::Dependency.new( + name: "ansi-regex", + package_manager: "npm_and_yarn", + version: "6.0.0", + requirements: [ + { + file: "package.json", + requirement: "^6.0.0", + groups: ["devDependencies"], + source: { + type: "registry", + url: "https://registry.npmjs.org" + } + } + ] + ) + end + let(:security_advisories) do + [ + { + "dependency-name" => "ansi-regex", + "affected-versions" => [ + ">= 3.0.0 < 3.0.1", + ">= 4.0.0 < 4.1.1", + ">= 5.0.0 < 5.0.1", + ">= 6.0.0 < 6.0.1" + ], + "patched-versions" => [], + "unaffected-versions" => [] + } + ] + end + let(:allowed_updates) do + [{ "dependency-type" => "production" }] + end + it { is_expected.to eq(false) } + end + end + + describe "#security_updates_only?" do + subject { job.security_updates_only? } + + it { is_expected.to eq(false) } + + context "with security only allowed updates" do + let(:security_updates_only) { true } + + it { is_expected.to eq(true) } + end + end + + describe "#experiments" do + it "handles nil values" do + expect(job.experiments).to eq({}) + end + + context "with experiments" do + let(:experiments) { { "simple" => false, "kebab-case" => true } } + + it "transforms the keys" do + expect(job.experiments).to eq(simple: false, kebab_case: true) + end + + it "registers the experiments with Dependabot::Experiments" do + job + expect(Dependabot::Experiments.enabled?(:kebab_case)).to be_truthy + expect(Dependabot::Experiments.enabled?(:simpe)).to be_falsey + end + end + + context "with experimental values" do + let(:experiments) { { "timeout_per_operation_seconds" => 600 } } + + it "preserves the values" do + expect(job.experiments).to eq(timeout_per_operation_seconds: 600) + end + end + end + + describe "#commit_message_options" do + it "handles nil values" do + expect(job.commit_message_options).to eq({}) + end + + context "with commit_message_options" do + let(:commit_message_options) do + { + "prefix" => "[dev]", + "prefix-development" => "[bump-dev]", + "include-scope" => true + } + end + + it "transforms the keys" do + expect(job.commit_message_options[:prefix]).to eq("[dev]") + expect(job.commit_message_options[:prefix_development]).to eq("[bump-dev]") + expect(job.commit_message_options[:include_scope]).to eq(true) + end + end + + context "with partial commit_message_options" do + let(:commit_message_options) do + { + "prefix" => "[dev]" + } + end + + it "transforms the keys" do + expect(job.commit_message_options[:prefix]).to eq("[dev]") + expect(job.commit_message_options).not_to have_key(:prefix_development) + expect(job.commit_message_options).not_to have_key(:include_scope) + end + end + end + + describe "#clone?" do + subject { job.clone? } + + it { is_expected.to eq(false) } + + context "with vendoring configuration enabled" do + let(:vendor_dependencies) { true } + + it { is_expected.to eq(true) } + end + + context "for ecosystems that always clone" do + let(:vendor_dependencies) { false } + let(:dependencies) do + [ + Dependabot::Dependency.new( + name: "github.com/pkg/errors", + package_manager: "go_modules", + version: "v1.8.0", + requirements: [ + { + file: "go.mod", + requirement: "v1.8.0", + groups: [], + source: nil + } + ] + ) + ] + end + let(:package_manager) { "go_modules" } + + it { is_expected.to eq(true) } + end + end + + describe "#security_fix?" do + subject { job.security_fix?(dependency) } + + let(:dependency) do + Dependabot::Dependency.new( + package_manager: "bundler", + name: "business", + version: dependency_version, + previous_version: dependency_previous_version, + requirements: [], + previous_requirements: [] + ) + end + let(:dependency_version) { "1.11.1" } + let(:dependency_previous_version) { "0.7.1" } + let(:security_advisories) do + [ + { + "dependency-name" => "business", + "affected-versions" => [], + "patched-versions" => ["~> 1.11.0"], + "unaffected-versions" => [] + } + ] + end + + it { is_expected.to eq(true) } + + context "when the update hasn't been patched" do + let(:dependency_version) { "1.10.0" } + + it { is_expected.to eq(false) } + end + end + + describe "#reject_external_code?" do + it "defaults to false" do + expect(job.reject_external_code?).to eq(false) + end + + it "can be enabled by job attributes" do + attrs = attributes + attrs[:reject_external_code] = true + job = Dependabot::Job.new(attrs) + expect(job.reject_external_code?).to eq(true) + end + end +end diff --git a/updater/spec/dependabot/sentry_spec.rb b/updater/spec/dependabot/sentry_spec.rb new file mode 100644 index 000000000..126060339 --- /dev/null +++ b/updater/spec/dependabot/sentry_spec.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +require "dependabot/sentry" +require "spec_helper" + +RSpec.describe ExceptionSanitizer do + let(:message) { "kaboom" } + let(:data) do + { + environment: "default", + extra: {}, + exception: { + values: [ + { type: "StandardError", value: message } + ] + } + } + end + + it "does not filter messages by default" do + expect(sanitized_message(data)).to eq(message) + end + + context "with exception containing Bearer token" do + let(:message) { "Bearer SECRET_TOKEN is bad and you should feel bad" } + + it "filters sensitive messages" do + expect(sanitized_message(data)).to eq( + "Bearer [FILTERED_AUTH_TOKEN] is bad and you should feel bad" + ) + end + end + + context "with exception containing Authorization: header" do + let(:message) { "Authorization: SECRET_TOKEN is bad" } + + it "filters sensitive messages" do + expect(sanitized_message(data)).to eq( + "Authorization: [FILTERED_AUTH_TOKEN] is bad" + ) + end + end + + context "with exception containing authorization value" do + let(:message) { "authorization SECRET_TOKEN invalid" } + + it "filters sensitive messages" do + expect(sanitized_message(data)).to eq( + "authorization [FILTERED_AUTH_TOKEN] invalid" + ) + end + end + + context "with exception secret token without an indicator" do + let(:message) { "SECRET_TOKEN is not filtered" } + + it "filters sensitive messages" do + expect(sanitized_message(data)).to eq("SECRET_TOKEN is not filtered") + end + end + + context "with api repo NWO" do + let(:message) { "https://api.github.com/repos/foo/bar is bad" } + + it "filters repo name from an api request" do + expect(sanitized_message(data)).to eq( + "https://api.github.com/repos/foo/[FILTERED_REPO] is bad" + ) + end + end + + context "with regular repo NWO" do + let(:message) { "https://github.com/foo/bar is bad" } + + it "filters repo name from an api request" do + expect(sanitized_message(data)).to eq( + "https://github.com/foo/[FILTERED_REPO] is bad" + ) + end + end + + context "with multiple repo NWO" do + let(:message) do + "https://api.github.com/repos/foo/bar is bad, " \ + "https://github.com/foo/baz is bad" + end + + it "filters repo name from an api request" do + expect(sanitized_message(data)).to eq( + "https://api.github.com/repos/foo/[FILTERED_REPO] is bad, " \ + "https://github.com/foo/[FILTERED_REPO] is bad" + ) + end + end + + private + + def sanitized_message(data) + filtered = ExceptionSanitizer.new.process(data) + filtered[:exception][:values].first[:value] + end +end diff --git a/updater/spec/dependabot/service_spec.rb b/updater/spec/dependabot/service_spec.rb new file mode 100644 index 000000000..44eb583a8 --- /dev/null +++ b/updater/spec/dependabot/service_spec.rb @@ -0,0 +1,518 @@ +# frozen_string_literal: true + +require "spec_helper" +require "dependabot/api_client" +require "dependabot/dependency_change" +require "dependabot/dependency_snapshot" +require "dependabot/errors" +require "dependabot/service" + +RSpec.describe Dependabot::Service do + let(:base_sha) { "mock-sha" } + + let(:mock_client) do + instance_double(Dependabot::ApiClient, { + create_pull_request: nil, + update_pull_request: nil, + close_pull_request: nil, + record_update_job_error: nil + }) + end + subject(:service) { described_class.new(client: mock_client) } + + shared_context :a_pr_was_created do + let(:dependency_change) do + Dependabot::DependencyChange.new( + job: instance_double(Dependabot::Job, source: nil, credentials: [], commit_message_options: []), + updated_dependencies: dependencies, + updated_dependency_files: dependency_files + ) + end + + let(:pr_message) { "update all the things" } + let(:dependencies) do + [ + Dependabot::Dependency.new( + name: "dependabot-fortran", + package_manager: "bundler", + version: "1.8.0", + previous_version: "1.7.0", + requirements: [ + { file: "Gemfile", requirement: "~> 1.8.0", groups: [], source: nil } + ], + previous_requirements: [ + { file: "Gemfile", requirement: "~> 1.7.0", groups: [], source: nil } + ] + ), + Dependabot::Dependency.new( + name: "dependabot-pascal", + package_manager: "bundler", + version: "2.8.0", + previous_version: "2.7.0", + requirements: [ + { file: "Gemfile", requirement: "~> 2.8.0", groups: [], source: nil } + ], + previous_requirements: [ + { file: "Gemfile", requirement: "~> 2.7.0", groups: [], source: nil } + ] + ) + ] + end + + let(:dependency_files) do + [ + { name: "Gemfile", content: "some gems" } + ] + end + + before do + allow(Dependabot::PullRequestCreator::MessageBuilder). + to receive_message_chain(:new, :message).and_return(pr_message) + + service.create_pull_request(dependency_change, base_sha) + end + end + + shared_context :a_pr_was_updated do + let(:dependency_change) do + Dependabot::DependencyChange.new( + job: anything, + updated_dependencies: dependencies, + updated_dependency_files: dependency_files + ) + end + + let(:dependencies) do + [ + Dependabot::Dependency.new( + name: "dependabot-cobol", + package_manager: "bundler", + version: "3.8.0", + previous_version: "3.7.0", + requirements: [ + { file: "Gemfile", requirement: "~> 3.8.0", groups: [], source: nil } + ], + previous_requirements: [ + { file: "Gemfile", requirement: "~> 3.7.0", groups: [], source: nil } + ] + ) + ] + end + + let(:dependency_files) do + [ + { name: "Gemfile", content: "some gems" } + ] + end + + before do + service.update_pull_request(dependency_change, base_sha) + end + end + + shared_context :a_pr_was_closed do + let(:dependency_name) { "dependabot-fortran" } + let(:reason) { :dependency_removed } + + before do + service.close_pull_request(dependency_name, reason) + end + end + + shared_context :an_error_was_reported do + before do + service.record_update_job_error( + error_type: :epoch_error, + error_details: { + message: "What is fortran doing here?!" + } + ) + end + end + + shared_context :a_dependency_error_was_reported do + let(:dependency) do + Dependabot::Dependency.new( + name: "dependabot-cobol", + package_manager: "bundler", + version: "3.8.0", + previous_version: "3.7.0", + requirements: [ + { file: "Gemfile", requirement: "~> 3.8.0", groups: [], source: nil } + ], + previous_requirements: [ + { file: "Gemfile", requirement: "~> 3.7.0", groups: [], source: nil } + ] + ) + end + + before do + service.record_update_job_error( + error_type: :unknown_error, + error_details: { + message: "0001 Undefined error. Inform Technical Support" + }, + dependency: dependency + ) + end + end + + describe "Instance methods delegated to @client" do + { + mark_job_as_processed: %w(mock_sha), + record_ecosystem_versions: %w(mock_ecosystem_versions) + }.each do |method, arguments| + before { allow(mock_client).to receive(method) } + + it "delegates #{method}" do + service.send(method, *arguments) + + expect(mock_client).to have_received(method).with(*arguments) + end + end + + it "delegates increment_metric" do + allow(mock_client).to receive(:increment_metric) + + service.increment_metric("apples", tags: { green: 1, red: 2 }) + + expect(mock_client).to have_received(:increment_metric).with("apples", tags: { green: 1, red: 2 }) + end + end + + describe "#create_pull_request" do + include_context :a_pr_was_created + + it "delegates to @client" do + expect(mock_client). + to have_received(:create_pull_request).with(dependency_change, base_sha) + end + + it "memoizes a shorthand summary of the PR" do + expect(service.pull_requests). + to eql([["dependabot-fortran ( from 1.7.0 to 1.8.0 ), dependabot-pascal ( from 2.7.0 to 2.8.0 )", :created]]) + end + end + + describe "#update_pull_request" do + include_context :a_pr_was_updated + + it "delegates to @client" do + expect(mock_client).to have_received(:update_pull_request).with(dependency_change, base_sha) + end + + it "memoizes a shorthand summary of the PR" do + expect(service.pull_requests).to eql([["dependabot-cobol ( from 3.7.0 to 3.8.0 )", :updated]]) + end + end + + describe "#close_pull_request" do + include_context :a_pr_was_closed + + it "delegates to @client" do + expect(mock_client).to have_received(:close_pull_request).with(dependency_name, reason) + end + + it "memoizes a shorthand summary of the reason for closing PRs for a dependency" do + expect(service.pull_requests).to eql([["dependabot-fortran", "closed: dependency_removed"]]) + end + end + + describe "#record_update_job_error" do + include_context :an_error_was_reported + + it "delegates to @client" do + expect(mock_client).to have_received(:record_update_job_error).with( + { + error_type: :epoch_error, + error_details: { + message: "What is fortran doing here?!" + } + } + ) + end + + it "memoizes a shorthand summary of the error" do + expect(service.errors).to eql([["epoch_error", nil]]) + end + end + + describe "#capture_exception" do + before do + allow(Raven).to receive(:capture_exception) + end + + let(:error) do + Dependabot::DependabotError.new("Something went wrong") + end + + it "delegates error capture to Sentry (Raven)" do + service.capture_exception(error: error, tags: { foo: "bar" }, extra: { baz: "qux" }) + + expect(Raven).to have_received(:capture_exception).with(error, tags: { foo: "bar" }, extra: { baz: "qux" }) + end + + it "extracts information from a job if provided" do + job = OpenStruct.new(id: 1234, package_manager: "bundler", repo_private?: false) + service.capture_exception(error: error, job: job) + + expect(Raven).to have_received(:capture_exception). + with(error, + tags: { + update_job_id: 1234, + package_manager: "bundler", + repo_private: false + }, + extra: {}) + end + + it "extracts information from a dependency if provided" do + dependency = OpenStruct.new(name: "lodash") + service.capture_exception(error: error, dependency: dependency) + + expect(Raven).to have_received(:capture_exception). + with(error, + tags: {}, + extra: { + dependency_name: "lodash" + }) + end + + it "extracts information from a dependency_group if provided" do + dependency_group = OpenStruct.new(name: "all-the-things") + service.capture_exception(error: error, dependency_group: dependency_group) + + expect(Raven).to have_received(:capture_exception). + with(error, + tags: {}, + extra: { + dependency_group: "all-the-things" + }) + end + end + + describe "#update_dependency_list" do + let(:dependency_snapshot) do + instance_double(Dependabot::DependencySnapshot, + dependencies: [ + Dependabot::Dependency.new( + name: "dummy-pkg-a", + package_manager: "bundler", + version: "2.0.0", + requirements: [ + { file: "Gemfile", requirement: "~> 2.0.0", groups: [:default], source: nil } + ] + ), + Dependabot::Dependency.new( + name: "dummy-pkg-b", + package_manager: "bundler", + version: "1.1.0", + requirements: [ + { file: "Gemfile", requirement: "~> 1.1.0", groups: [:default], source: nil } + ] + ) + ], + dependency_files: [ + Dependabot::DependencyFile.new( + name: "Gemfile", + content: fixture("bundler/original/Gemfile"), + directory: "/" + ), + Dependabot::DependencyFile.new( + name: "Gemfile.lock", + content: fixture("bundler/original/Gemfile.lock"), + directory: "/" + ) + ]) + end + + let(:expected_dependency_payload) do + [ + { + name: "dummy-pkg-a", + version: "2.0.0", + requirements: [ + { + file: "Gemfile", + requirement: "~> 2.0.0", + groups: [:default], + source: nil + } + ] + }, + { + name: "dummy-pkg-b", + version: "1.1.0", + requirements: [ + { + file: "Gemfile", + requirement: "~> 1.1.0", + groups: [:default], + source: nil + } + ] + } + ] + end + let(:expected_file_paths) do + ["/Gemfile", "/Gemfile.lock"] + end + + it "extracts a payload from the DependencySnapshot and delegates to the client" do + expect(mock_client).to receive(:update_dependency_list).with(expected_dependency_payload, expected_file_paths) + + service.update_dependency_list(dependency_snapshot: dependency_snapshot) + end + end + + describe "#noop?" do + it "is true by default" do + expect(service).to be_noop + end + + it "is false if there has been an event" do + service.record_update_job_error( + error_type: :epoch_error, + error_details: { + message: "What is fortran doing here?!" + } + ) + + expect(service).not_to be_noop + end + + it "is false if there has been a pull request change" do + service.close_pull_request("dependabot-cobol", "legacy code removed") + + expect(service).not_to be_failure + end + end + + describe "#failure?" do + it "is false by default" do + expect(service).not_to be_failure + end + + it "is true if there has been an error" do + service.record_update_job_error( + error_type: :epoch_error, + error_details: { + message: "What is fortran doing here?!" + } + ) + + expect(service).to be_failure + end + end + + describe "#summary" do + context "when there were no service events" do + it "is empty" do + expect(service.summary).to be_nil + end + end + + context "when a pr was created" do + include_context :a_pr_was_created + + it "includes the summary of the created PR" do + expect(service.summary). + to include("created", "dependabot-fortran ( from 1.7.0 to 1.8.0 ), dependabot-pascal ( from 2.7.0 to 2.8.0 )") + end + end + + context "when a pr was updated" do + include_context :a_pr_was_updated + + it "includes the summary of the updated PR" do + expect(service.summary). + to include("updated", "dependabot-cobol ( from 3.7.0 to 3.8.0 )") + end + end + + context "when a pr was closed" do + include_context :a_pr_was_closed + + it "includes the summary of the closed PR" do + expect(service.summary). + to include("closed: dependency_removed", "dependabot-fortran") + end + end + + context "when there was an error" do + include_context :an_error_was_reported + + it "includes an error count" do + expect(service.summary). + to include("Dependabot encountered '1' error(s) during execution") + end + + it "includes an error summary" do + expect(service.summary). + to include("epoch_error") + end + end + + context "when there was an dependency error" do + include_context :a_dependency_error_was_reported + + it "includes an error count" do + expect(service.summary). + to include("Dependabot encountered '1' error(s) during execution") + end + + it "includes an error summary" do + expect(service.summary). + to include("unknown_error") + expect(service.summary). + to include("dependabot-cobol") + end + end + + context "when there was a mix of pr activity" do + include_context :a_pr_was_updated + include_context :a_pr_was_closed + + it "includes the summary of the updated PR" do + expect(service.summary). + to include("updated", "dependabot-cobol ( from 3.7.0 to 3.8.0 )") + end + + it "includes the summary of the closed PR" do + expect(service.summary). + to include("closed: dependency_removed", "dependabot-fortran") + end + end + + context "when there was a mix of pr and error activity" do + include_context :a_pr_was_created + include_context :a_pr_was_closed + include_context :an_error_was_reported + include_context :a_dependency_error_was_reported + + it "includes the summary of the created PR" do + expect(service.summary). + to include("created", "dependabot-fortran ( from 1.7.0 to 1.8.0 ), dependabot-pascal ( from 2.7.0 to 2.8.0 )") + end + + it "includes the summary of the closed PR" do + expect(service.summary). + to include("closed: dependency_removed", "dependabot-fortran") + end + + it "includes an error count" do + expect(service.summary). + to include("Dependabot encountered '2' error(s) during execution") + end + + it "includes an error summary" do + expect(service.summary). + to include("epoch_error") + expect(service.summary). + to include("unknown_error") + expect(service.summary). + to include("dependabot-fortran") + end + end + end +end diff --git a/updater/spec/dependabot/updater/error_handler_spec.rb b/updater/spec/dependabot/updater/error_handler_spec.rb new file mode 100644 index 000000000..1def349fe --- /dev/null +++ b/updater/spec/dependabot/updater/error_handler_spec.rb @@ -0,0 +1,223 @@ +# frozen_string_literal: true + +require "spec_helper" + +require "dependabot/dependency" +require "dependabot/dependency_group" +require "dependabot/job" +require "dependabot/service" +require "dependabot/updater/error_handler" + +RSpec.describe Dependabot::Updater::ErrorHandler do + subject(:error_handler) do + described_class.new( + service: mock_service, + job: mock_job + ) + end + + let(:mock_service) do + instance_double(Dependabot::Service) + end + + let(:mock_job) do + instance_double(Dependabot::Job) + end + + describe "#handle_dependency_error" do + let(:dependency) do + instance_double(Dependabot::Dependency, name: "broken-biscuits") + end + + let(:handle_dependency_error) do + error_handler.handle_dependency_error(error: error, dependency: dependency) + end + + context "with a handled known error" do + let(:error) do + Dependabot::DependencyFileNotResolvable.new("The file is full of bees") + end + + it "records the error with the service and logs it out" do + expect(mock_service).to receive(:record_update_job_error).with( + error_type: "dependency_file_not_resolvable", + error_details: { message: "The file is full of bees" }, + dependency: dependency + ) + + expect(Dependabot.logger).to receive(:info).with( + a_string_starting_with("Handled error whilst updating broken-biscuits:") + ) + + handle_dependency_error + end + end + + context "with a handled unknown error" do + let(:error) do + StandardError.new("There are bees everywhere").tap do |err| + err.set_backtrace ["bees.rb:5:in `buzz`"] + end + end + + it "records the error with the service, logs the backtrace and captures the exception" do + expect(mock_service).to receive(:record_update_job_error).with( + error_type: "unknown_error", + error_details: nil, + dependency: dependency + ) + + expect(mock_service).to receive(:capture_exception).with( + error: error, + job: mock_job, + dependency: dependency, + dependency_group: nil + ) + + expect(Dependabot.logger).to receive(:error).with( + "Error processing broken-biscuits (StandardError)" + ) + expect(Dependabot.logger).to receive(:error).with( + "There are bees everywhere" + ) + expect(Dependabot.logger).to receive(:error).with( + "bees.rb:5:in `buzz`" + ) + + handle_dependency_error + end + end + + context "with a job-halting error" do + let(:error) do + Dependabot::OutOfDisk.new("The disk is full of bees") + end + + it "re-raises the error" do + expect { handle_dependency_error }.to raise_error(error) + end + end + + context "with a subprocess failure error" do + let(:error_context) do + { bumblebees: "many", honeybees: "few", wasps: "none", fingerprint: "123456789" } + end + + let(:error) do + Dependabot::SharedHelpers::HelperSubprocessFailed.new(message: "the kernal is full of bees", + error_context: error_context).tap do |err| + err.set_backtrace ["****** ERROR 8335 -- 101"] + end + end + + it "records the error with the service and logs the backtrace" do + expect(mock_service).to receive(:record_update_job_error).with( + error_type: "unknown_error", + error_details: nil, + dependency: dependency + ) + + expect(mock_service).to receive(:capture_exception) + + expect(Dependabot.logger).to receive(:error).with( + "Error processing broken-biscuits (Dependabot::SharedHelpers::HelperSubprocessFailed)" + ) + expect(Dependabot.logger).to receive(:error).with( + "the kernal is full of bees" + ) + expect(Dependabot.logger).to receive(:error).with( + "****** ERROR 8335 -- 101" + ) + + handle_dependency_error + end + + it "sanitizes the error and captures it" do + allow(Dependabot.logger).to receive(:error) + allow(mock_service).to receive(:record_update_job_error) + expect(mock_service).to receive(:capture_exception).with( + error: an_instance_of(Dependabot::Updater::SubprocessFailed), job: mock_job + ) do |args| + expect(args[:error].message). + to eq('Subprocess ["123456789"] failed to run. Check the job logs for error messages') + expect(args[:error].raven_context). + to eq(fingerprint: ["123456789"], + extra: { + bumblebees: "many", honeybees: "few", wasps: "none" + }) + end + + handle_dependency_error + end + end + end + + describe "handle_job_error" do + let(:handle_job_error) do + error_handler.handle_job_error(error: error) + end + + context "with a handled known error" do + let(:error) do + Dependabot::DependencyFileNotResolvable.new("The file is full of bees") + end + + it "records the error with the service and logs it out" do + expect(mock_service).to receive(:record_update_job_error).with( + error_type: "dependency_file_not_resolvable", + error_details: { message: "The file is full of bees" } + ) + + expect(Dependabot.logger).to receive(:info).with( + a_string_starting_with("Handled error whilst processing job:") + ) + + handle_job_error + end + end + + context "with a handled unknown error" do + let(:error) do + StandardError.new("There are bees everywhere").tap do |err| + err.set_backtrace ["bees.rb:5:in `buzz`"] + end + end + + it "records the error with the service, logs the backtrace and captures the exception" do + expect(mock_service).to receive(:record_update_job_error).with( + error_type: "unknown_error", + error_details: nil + ) + + expect(mock_service).to receive(:capture_exception).with( + error: error, + job: mock_job, + dependency: nil, + dependency_group: nil + ) + + expect(Dependabot.logger).to receive(:error).with( + "Error processing job (StandardError)" + ) + expect(Dependabot.logger).to receive(:error).with( + "There are bees everywhere" + ) + expect(Dependabot.logger).to receive(:error).with( + "bees.rb:5:in `buzz`" + ) + + handle_job_error + end + end + + context "with a job-halting error" do + let(:error) do + Dependabot::OutOfDisk.new("The disk is full of bees") + end + + it "re-raises the error" do + expect { handle_job_error }.to raise_error(error) + end + end + end +end diff --git a/updater/spec/dependabot/updater/operations_spec.rb b/updater/spec/dependabot/updater/operations_spec.rb new file mode 100644 index 000000000..4d33f7f9a --- /dev/null +++ b/updater/spec/dependabot/updater/operations_spec.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require "dependabot/job" +require "dependabot/updater/operations" + +require "spec_helper" + +RSpec.describe Dependabot::Updater::Operations do + describe "::class_for" do + after do + Dependabot::Experiments.reset! + end + + it "returns nil if no operation matches" do + # We always expect jobs that update a pull request to specify their + # existing dependency changes, a job with this set of conditions + # should never exist. + job = instance_double(Dependabot::Job, + security_updates_only?: false, + updating_a_pull_request?: true, + dependencies: [], + dependency_groups: [], + is_a?: true) + + expect(described_class.class_for(job: job)).to be_nil + end + + it "returns the UpdateAllVersions class when the Job is for a fresh, non-security update with no dependencies" do + job = instance_double(Dependabot::Job, + security_updates_only?: false, + updating_a_pull_request?: false, + dependencies: [], + dependency_groups: [], + is_a?: true) + + expect(described_class.class_for(job: job)).to be(Dependabot::Updater::Operations::UpdateAllVersions) + end + + context "the grouped update experiment is enabled" do + it "returns the GroupUpdateAllVersions class when the Job is for a fresh, version update with no dependencies" do + Dependabot::Experiments.register("grouped_updates_prototype", true) + + job = instance_double(Dependabot::Job, + security_updates_only?: false, + updating_a_pull_request?: false, + dependencies: [], + dependency_groups: [anything], + is_a?: true) + + expect(described_class.class_for(job: job)).to be(Dependabot::Updater::Operations::GroupUpdateAllVersions) + + Dependabot::Experiments.reset! + end + + it "returns the RefreshGroupUpdatePullRequest class when the Job is for an existing group update" do + Dependabot::Experiments.register("grouped_updates_prototype", true) + + job = instance_double(Dependabot::Job, + security_updates_only?: false, + updating_a_pull_request?: true, + dependencies: [anything], + dependency_group_to_refresh: anything, + dependency_groups: [anything], + is_a?: true) + + expect(described_class.class_for(job: job)). + to be(Dependabot::Updater::Operations::RefreshGroupUpdatePullRequest) + + Dependabot::Experiments.reset! + end + end + + it "returns the RefreshVersionUpdatePullRequest class when the Job is for an existing dependency version update" do + job = instance_double(Dependabot::Job, + security_updates_only?: false, + updating_a_pull_request?: true, + dependencies: [anything], + dependency_group_to_refresh: nil, + dependency_groups: [anything], + is_a?: true) + + expect(described_class.class_for(job: job)). + to be(Dependabot::Updater::Operations::RefreshVersionUpdatePullRequest) + end + + it "returns the CreateSecurityUpdatePullRequest class when the Job is for a new security update for a dependency" do + job = instance_double(Dependabot::Job, + security_updates_only?: true, + updating_a_pull_request?: false, + dependencies: [anything], + dependency_groups: [anything], + is_a?: true) + + expect(described_class.class_for(job: job)). + to be(Dependabot::Updater::Operations::CreateSecurityUpdatePullRequest) + end + + it "returns the RefreshSecurityUpdatePullRequest class when the Job is for an existing security update" do + job = instance_double(Dependabot::Job, + security_updates_only?: true, + updating_a_pull_request?: true, + dependencies: [anything], + dependency_groups: [anything], + is_a?: true) + + expect(described_class.class_for(job: job)). + to be(Dependabot::Updater::Operations::RefreshSecurityUpdatePullRequest) + end + + it "raises an argument error with anything other than a Dependabot::Job" do + expect { described_class.class_for(job: Object.new) }.to raise_error(ArgumentError) + end + end +end diff --git a/updater/spec/fixtures/bundler/original/Gemfile b/updater/spec/fixtures/bundler/original/Gemfile new file mode 100644 index 000000000..ecc7a043d --- /dev/null +++ b/updater/spec/fixtures/bundler/original/Gemfile @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "dummy-pkg-a", "~> 2.0.0" +gem "dummy-pkg-b", "~> 1.1.0" diff --git a/updater/spec/fixtures/bundler/original/Gemfile.lock b/updater/spec/fixtures/bundler/original/Gemfile.lock new file mode 100644 index 000000000..4fd8f37aa --- /dev/null +++ b/updater/spec/fixtures/bundler/original/Gemfile.lock @@ -0,0 +1,16 @@ +GEM + remote: https://rubygems.org/ + specs: + dummy-pkg-a (2.0.0) + dummy-pkg-b (1.1.0) + dummy-pkg-a (~> 2.0) + +PLATFORMS + ruby + +DEPENDENCIES + dummy-pkg-a (~> 2.0.0) + dummy-pkg-b (~> 1.1.0) + +BUNDLED WITH + 1.14.6 diff --git a/updater/spec/fixtures/bundler/updated/Gemfile b/updater/spec/fixtures/bundler/updated/Gemfile new file mode 100644 index 000000000..607828c5f --- /dev/null +++ b/updater/spec/fixtures/bundler/updated/Gemfile @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "dummy-pkg-a", "~> 2.0.0" +gem "dummy-pkg-b", "~> 1.2.0" diff --git a/updater/spec/fixtures/bundler/updated/Gemfile.lock b/updater/spec/fixtures/bundler/updated/Gemfile.lock new file mode 100644 index 000000000..0e7697ff9 --- /dev/null +++ b/updater/spec/fixtures/bundler/updated/Gemfile.lock @@ -0,0 +1,16 @@ +GEM + remote: https://rubygems.org/ + specs: + dummy-pkg-a (2.0.0) + dummy-pkg-b (1.2.0) + dummy-pkg-a (~> 2.0) + +PLATFORMS + ruby + +DEPENDENCIES + dummy-pkg-a (~> 2.0.0) + dummy-pkg-b (~> 1.2.0) + +BUNDLED WITH + 1.14.6 diff --git a/updater/spec/fixtures/bundler_gemspec/original/Gemfile b/updater/spec/fixtures/bundler_gemspec/original/Gemfile new file mode 100644 index 000000000..b4e2a20bb --- /dev/null +++ b/updater/spec/fixtures/bundler_gemspec/original/Gemfile @@ -0,0 +1,3 @@ +source "https://rubygems.org" + +gemspec diff --git a/updater/spec/fixtures/bundler_gemspec/original/Gemfile.lock b/updater/spec/fixtures/bundler_gemspec/original/Gemfile.lock new file mode 100644 index 000000000..ce275f42b --- /dev/null +++ b/updater/spec/fixtures/bundler_gemspec/original/Gemfile.lock @@ -0,0 +1,39 @@ +PATH + remote: . + specs: + library (1.0.0) + rack (~> 2.1.4) + rubocop (~> 0.76.0) + toml-rb (~> 2.2.0) + +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.2) + citrus (3.0.2) + jaro_winkler (1.5.4) + parallel (1.23.0) + parser (3.2.2.1) + ast (~> 2.4.1) + rack (2.1.4.3) + rainbow (3.1.1) + rubocop (0.76.0) + jaro_winkler (~> 1.5.1) + parallel (~> 1.10) + parser (>= 2.6) + rainbow (>= 2.2.2, < 4.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 1.4.0, < 1.7) + ruby-progressbar (1.13.0) + toml-rb (2.2.0) + citrus (~> 3.0, > 3.0) + unicode-display_width (1.6.1) + +PLATFORMS + ruby + +DEPENDENCIES + library! + +BUNDLED WITH + 2.4.11 diff --git a/updater/spec/fixtures/bundler_gemspec/original/library.gemspec b/updater/spec/fixtures/bundler_gemspec/original/library.gemspec new file mode 100644 index 000000000..c3cb6f5fc --- /dev/null +++ b/updater/spec/fixtures/bundler_gemspec/original/library.gemspec @@ -0,0 +1,12 @@ + +Gem::Specification.new do |s| + s.name = "library" + s.summary = "A Library" + s.version = "1.0.0" + s.homepage = "https://github.com/dependabot/dependabot-core" + s.authors = %w[monalisa] + + s.add_runtime_dependency "rubocop", "~> 0.76.0" + s.add_runtime_dependency "toml-rb", "~> 2.2.0" + s.add_runtime_dependency "rack", "~> 2.1.4" +end diff --git a/updater/spec/fixtures/bundler_git/original/Gemfile b/updater/spec/fixtures/bundler_git/original/Gemfile new file mode 100644 index 000000000..d27ea892f --- /dev/null +++ b/updater/spec/fixtures/bundler_git/original/Gemfile @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "dummy-git-dependency", git: "git@github.com:dependabot-fixtures/ruby-dummy-git-dependency.git", ref: "v1.0.0" diff --git a/updater/spec/fixtures/bundler_git/original/Gemfile.lock b/updater/spec/fixtures/bundler_git/original/Gemfile.lock new file mode 100644 index 000000000..cfe797d21 --- /dev/null +++ b/updater/spec/fixtures/bundler_git/original/Gemfile.lock @@ -0,0 +1,21 @@ +GIT + remote: git@github.com:dependabot-fixtures/ruby-dummy-git-dependency.git + revision: 20151f9b67c8a04461fa0ee28385b6187b86587b + ref: v1.0.0 + specs: + dummy-git-dependency (1.0.0) + +GEM + remote: https://rubygems.org/ + specs: + +PLATFORMS + aarch64-linux + x86_64-darwin-19 + x86_64-linux + +DEPENDENCIES + dummy-git-dependency! + +BUNDLED WITH + 2.2.11 diff --git a/updater/spec/fixtures/bundler_grouped_by_types/original/Gemfile b/updater/spec/fixtures/bundler_grouped_by_types/original/Gemfile new file mode 100644 index 000000000..75e1bd971 --- /dev/null +++ b/updater/spec/fixtures/bundler_grouped_by_types/original/Gemfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "rack", "~> 2.1.3" +gem "toml-rb", "~> 2.2.0" + +group :development do + gem "rubocop", "~> 0.75.0" +end diff --git a/updater/spec/fixtures/bundler_grouped_by_types/original/Gemfile.lock b/updater/spec/fixtures/bundler_grouped_by_types/original/Gemfile.lock new file mode 100644 index 000000000..4f3322cf2 --- /dev/null +++ b/updater/spec/fixtures/bundler_grouped_by_types/original/Gemfile.lock @@ -0,0 +1,35 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.2) + citrus (3.0.2) + jaro_winkler (1.5.6) + parallel (1.23.0) + parser (3.2.2.3) + ast (~> 2.4.1) + racc + racc (1.7.1) + rack (2.1.3) + rainbow (3.1.1) + rubocop (0.75.0) + jaro_winkler (~> 1.5.1) + parallel (~> 1.10) + parser (>= 2.6) + rainbow (>= 2.2.2, < 4.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 1.4.0, < 1.7) + ruby-progressbar (1.13.0) + toml-rb (2.2.0) + citrus (~> 3.0, > 3.0) + unicode-display_width (1.6.1) + +PLATFORMS + ruby + +DEPENDENCIES + rack (~> 2.1.3) + rubocop (~> 0.75.0) + toml-rb (~> 2.2.0) + +BUNDLED WITH + 1.14.6 diff --git a/updater/spec/fixtures/bundler_vendored/original/Gemfile b/updater/spec/fixtures/bundler_vendored/original/Gemfile new file mode 100644 index 000000000..cd2c75c9e --- /dev/null +++ b/updater/spec/fixtures/bundler_vendored/original/Gemfile @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "dummy-pkg-a", "~> 2.0.0" +gem "dummy-pkg-b", "~> 1.1.0" +gem "dummy-git-dependency", git: "git@github.com:dependabot-fixtures/ruby-dummy-git-dependency.git", ref: "v1.0.0" diff --git a/updater/spec/fixtures/bundler_vendored/original/Gemfile.lock b/updater/spec/fixtures/bundler_vendored/original/Gemfile.lock new file mode 100644 index 000000000..190b6d954 --- /dev/null +++ b/updater/spec/fixtures/bundler_vendored/original/Gemfile.lock @@ -0,0 +1,24 @@ +GIT + remote: git@github.com:dependabot-fixtures/ruby-dummy-git-dependency.git + revision: 20151f9b67c8a04461fa0ee28385b6187b86587b + ref: v1.0.0 + specs: + dummy-git-dependency (1.0.0) + +GEM + remote: https://rubygems.org/ + specs: + dummy-pkg-a (2.0.0) + dummy-pkg-b (1.1.0) + dummy-pkg-a (~> 2.0) + +PLATFORMS + x86_64-darwin-22 + +DEPENDENCIES + dummy-git-dependency! + dummy-pkg-a (~> 2.0.0) + dummy-pkg-b (~> 1.1.0) + +BUNDLED WITH + 2.4.13 diff --git a/updater/spec/fixtures/docker/original/Dockerfile.bundler b/updater/spec/fixtures/docker/original/Dockerfile.bundler new file mode 100644 index 000000000..326a6102e --- /dev/null +++ b/updater/spec/fixtures/docker/original/Dockerfile.bundler @@ -0,0 +1 @@ +FROM ghcr.io/dependabot/dependabot-updater-bundler:v2.0.20230317134336@sha256:f1a76307b9a43d4d25289852e2d0ee73f1bf2bbef1a935d9255c2e56f9db64fc diff --git a/updater/spec/fixtures/docker/original/Dockerfile.cargo b/updater/spec/fixtures/docker/original/Dockerfile.cargo new file mode 100644 index 000000000..3c283b986 --- /dev/null +++ b/updater/spec/fixtures/docker/original/Dockerfile.cargo @@ -0,0 +1 @@ +FROM ghcr.io/dependabot/dependabot-updater-cargo:v2.0.20230317130517@sha256:8913caf469d377ae28525b184b40f61b554d839082f594b31a36beb7344d12bc diff --git a/updater/spec/fixtures/job_definitions/README.md b/updater/spec/fixtures/job_definitions/README.md new file mode 100644 index 000000000..afa7b467c --- /dev/null +++ b/updater/spec/fixtures/job_definitions/README.md @@ -0,0 +1,26 @@ +### Job Definition Fixtures + +These fixtures match the file format consumed by dependabot/cli. + +#### Creating a fixture + +We can generate these files from real projects using the CLI tool, e.g. + +```sh +dependabot update bundler dependabot/dependabot-core -o test.yml +``` + +The resultant `test.yml` file will contain an `input` attribute which can be +extracted to use as a fixture, e.g. + +```yml +input: + # The entire input object can be extracted to a fixture file. + job: + package-manager: bundler + ... +``` + +**Maintainers**: It is also possible to generate this file from the service, +refer to internal documentation. + diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all.yaml new file mode 100644 index 000000000..b698aad46 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all.yaml @@ -0,0 +1,38 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + existing-pull-requests: [] + updating-a-pull-request: false + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: everything-everywhere-all-at-once + rules: + patterns: + - "*" diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_by_dependency_type.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_by_dependency_type.yaml new file mode 100644 index 000000000..e38bdcb52 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_by_dependency_type.yaml @@ -0,0 +1,43 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + existing-pull-requests: [] + updating-a-pull-request: false + lockfile-only: false + update-subdependencies: false + ignore-conditions: + - dependency-name: rubocop + version-requirement: "> 1.56.0" + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + grouped-updates-experimental-rules: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: dev-dependencies + rules: + dependency-type: "development" + - name: production-dependencies + rules: + dependency-type: "production" diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_empty_group.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_empty_group.yaml new file mode 100644 index 000000000..ce148d3c6 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_empty_group.yaml @@ -0,0 +1,38 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + existing-pull-requests: [] + updating-a-pull-request: false + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: everything-everywhere-all-at-once + rules: + patterns: + - "*bagel" diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_overlapping_groups.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_overlapping_groups.yaml new file mode 100644 index 000000000..763524abf --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_overlapping_groups.yaml @@ -0,0 +1,42 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + existing-pull-requests: [] + updating-a-pull-request: false + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: my-group + rules: + patterns: + - "dummy-pkg-*" + - name: my-overlapping-group + rules: + patterns: + - "dummy-pkg-*" diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_semver_grouping.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_semver_grouping.yaml new file mode 100644 index 000000000..75c3d2c06 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_semver_grouping.yaml @@ -0,0 +1,40 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + existing-pull-requests: [] + updating-a-pull-request: false + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + grouped-updates-experimental-rules: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: small-bumps + rules: + update-types: + - "minor" + - "patch" diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_semver_grouping_with_global_ignores.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_semver_grouping_with_global_ignores.yaml new file mode 100644 index 000000000..d08e0f691 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_semver_grouping_with_global_ignores.yaml @@ -0,0 +1,41 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + existing-pull-requests: [] + updating-a-pull-request: false + lockfile-only: false + update-subdependencies: false + ignore-conditions: + - dependency-name: "*" + update-types: ["version-update:semver-major"] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + grouped-updates-experimental-rules: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: patches + rules: + update-types: + - "patch" diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_existing_pr.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_existing_pr.yaml new file mode 100644 index 000000000..0d92199da --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_existing_pr.yaml @@ -0,0 +1,43 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + existing-pull-requests: [] + existing-group-pull-requests: + - dependency-group-name: "group-b" + dependencies: + - dependency-name: "dummy-pkg-b" + dependency-version: "1.2.0" + updating-a-pull-request: false + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: group-b + rules: + patterns: + - "dummy-pkg-b" diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_ungrouped.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_ungrouped.yaml new file mode 100644 index 000000000..51c66fbf5 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_ungrouped.yaml @@ -0,0 +1,38 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + existing-pull-requests: [] + updating-a-pull-request: false + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: group-b + rules: + patterns: + - "dummy-pkg-b" diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_vendoring.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_vendoring.yaml new file mode 100644 index 000000000..9540fd733 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_vendoring.yaml @@ -0,0 +1,38 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + existing-pull-requests: [] + updating-a-pull-request: false + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: true + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: everything-everywhere-all-at-once + rules: + patterns: + - "*" diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_peer_manifests.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_peer_manifests.yaml new file mode 100644 index 000000000..1becba2bb --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_peer_manifests.yaml @@ -0,0 +1 @@ +404: Not Found \ No newline at end of file diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh.yaml new file mode 100644 index 000000000..b0f9fd2ab --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh.yaml @@ -0,0 +1,45 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + - dummy-pkg-b + existing-pull-requests: [] + existing-group-pull-requests: + - dependency-group-name: everything-everywhere-all-at-once + dependencies: + - dependency-name: dummy-pkg-b + dependency-version: 1.2.0 + updating-a-pull-request: true + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: everything-everywhere-all-at-once + rules: + patterns: + - "*" + dependency-group-to-refresh: everything-everywhere-all-at-once diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_dependencies_changed.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_dependencies_changed.yaml new file mode 100644 index 000000000..66546c9d6 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_dependencies_changed.yaml @@ -0,0 +1,48 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + - dummy-pkg-b + - dummy-pkg-c + existing-pull-requests: [] + existing-group-pull-requests: + - dependency-group-name: everything-everywhere-all-at-once + dependencies: + - dependency-name: dummy-pkg-b + dependency-version: 1.2.0 + - dependency-name: dummy-pkg-c + dependency-version: 0.99.0 + updating-a-pull-request: true + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: everything-everywhere-all-at-once + rules: + patterns: + - "*" + dependency-group-to-refresh: everything-everywhere-all-at-once diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_empty_group.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_empty_group.yaml new file mode 100644 index 000000000..77614afe1 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_empty_group.yaml @@ -0,0 +1,45 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + - dummy-pkg-b + existing-pull-requests: [] + existing-group-pull-requests: + - dependency-group-name: everything-everywhere-all-at-once + dependencies: + - dependency-name: dummy-pkg-b + dependency-version: 1.2.0 + updating-a-pull-request: true + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: everything-everywhere-all-at-once + rules: + patterns: + - "*bagel" + dependency-group-to-refresh: everything-everywhere-all-at-once diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_missing_group.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_missing_group.yaml new file mode 100644 index 000000000..64ece40d8 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_missing_group.yaml @@ -0,0 +1,41 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + - dummy-pkg-b + existing-pull-requests: [] + existing-group-pull-requests: + - dependency-group-name: everything-everywhere-all-at-once + dependencies: + - dependency-name: dummy-pkg-b + dependency-version: 1.2.0 + updating-a-pull-request: true + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: [] + dependency-group-to-refresh: everything-everywhere-all-at-once diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_similar_pr.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_similar_pr.yaml new file mode 100644 index 000000000..11db4aeb3 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_similar_pr.yaml @@ -0,0 +1,53 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + - dummy-pkg-b + existing-pull-requests: [] + existing-group-pull-requests: + - dependency-group-name: everything-everywhere-all-at-once + dependencies: + - dependency-name: dummy-pkg-b + dependency-version: 1.1.5 + - dependency-group-name: overlapping-group + dependencies: + - dependency-name: dummy-pkg-b + dependency-version: 1.2.0 + updating-a-pull-request: true + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: everything-everywhere-all-at-once + rules: + patterns: + - "*" + - name: overlapping-group + rules: + patterns: + - "dummy-pkg-*" + dependency-group-to-refresh: everything-everywhere-all-at-once diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_versions_changed.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_versions_changed.yaml new file mode 100644 index 000000000..79e42f2c2 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_versions_changed.yaml @@ -0,0 +1,45 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + - dummy-pkg-b + existing-pull-requests: [] + existing-group-pull-requests: + - dependency-group-name: everything-everywhere-all-at-once + dependencies: + - dependency-name: dummy-pkg-b + dependency-version: 1.1.5 + updating-a-pull-request: true + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false + dependency-groups: + - name: everything-everywhere-all-at-once + rules: + patterns: + - "*" + dependency-group-to-refresh: everything-everywhere-all-at-once diff --git a/updater/spec/fixtures/job_definitions/bundler/version_updates/update_all_simple.yaml b/updater/spec/fixtures/job_definitions/bundler/version_updates/update_all_simple.yaml new file mode 100644 index 000000000..2be7d5ec6 --- /dev/null +++ b/updater/spec/fixtures/job_definitions/bundler/version_updates/update_all_simple.yaml @@ -0,0 +1,33 @@ +job: + package-manager: bundler + source: + provider: github + repo: dependabot/smoke-tests + directory: "/bundler" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + dependencies: + existing-pull-requests: [] + updating-a-pull-request: false + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false diff --git a/updater/spec/fixtures/job_definitions/docker/version_updates/group_update_peer_manifests.yaml b/updater/spec/fixtures/job_definitions/docker/version_updates/group_update_peer_manifests.yaml new file mode 100644 index 000000000..a7d9ab31e --- /dev/null +++ b/updater/spec/fixtures/job_definitions/docker/version_updates/group_update_peer_manifests.yaml @@ -0,0 +1,39 @@ +job: + package-manager: docker + source: + provider: github + repo: github/dependabot-action + directory: "/docker" + branch: + api-endpoint: https://api.github.com/ + hostname: github.com + commit: 302aaa943c6507c10cbbfd1b7f0fd623c5743807 + dependencies: + dependency-groups: + - name: dependabot-core-images + rules: + patterns: + - "dependabot/*" + existing-pull-requests: [] + updating-a-pull-request: false + lockfile-only: false + update-subdependencies: false + ignore-conditions: [] + requirements-update-strategy: + allowed-updates: + - dependency-type: direct + update-type: all + credentials-metadata: + - type: git_source + host: github.com + security-advisories: [] + max-updater-run-time: 2700 + vendor-dependencies: false + experiments: + grouped-updates-prototype: true + reject-external-code: false + commit-message-options: + prefix: + prefix-development: + include-scope: + security-updates-only: false diff --git a/updater/spec/fixtures/jobs/job_with_credentials.json b/updater/spec/fixtures/jobs/job_with_credentials.json new file mode 100644 index 000000000..558883ed8 --- /dev/null +++ b/updater/spec/fixtures/jobs/job_with_credentials.json @@ -0,0 +1,58 @@ +{ + "job": { + "allowed-updates": [ + { + "dependency-type": "direct", + "update-type": "all" + }, + { + "dependency-type": "indirect", + "update-type": "security" + } + ], + "credentials-metadata": [ + { + "type": "git_source", + "host": "github.com" + }, + { + "type": "rubygems_index", + "host": "my.rubygems-host.org" + } + ], + "dependencies": null, + "directory": "/", + "existing-pull-requests": [], + "ignore-conditions": [], + "security-advisories": [], + "package_manager": "bundler", + "repo-name": "dependabot-fixtures/dependabot-test-ruby-package", + "source": { + "provider": "github", + "repo": "dependabot-fixtures/dependabot-test-ruby-package", + "directory": "/", + "branch": null, + "hostname": "github.com", + "api-endpoint": "https://api.github.com/" + }, + "lockfile-only": false, + "requirements-update-strategy": null, + "update-subdependencies": false, + "updating-a-pull-request": false, + "vendor-dependencies": false, + "security-updates-only": false + }, + "credentials": [ + { + "type": "git_source", + "host": "github.com", + "username": "x-access-token", + "password": "v1.exampletokenfromgithubinityesitisforsure" + }, + { + "type": "rubygems_index", + "host": "my.rubygems-host.org", + "token": "secret" + } + ] +} diff --git a/updater/spec/fixtures/rubygems-index b/updater/spec/fixtures/rubygems-index new file mode 100644 index 000000000..b743cc67c --- /dev/null +++ b/updater/spec/fixtures/rubygems-index @@ -0,0 +1,10 @@ +created_at: 2017-03-27T04:38:13+00:00 +--- +dummy-pkg-a 1.0.0 bf914ad70e2044413345b8efd4911d69 +dummy-pkg-a 1.0.1 92e70e285b9ea3e5fb66913f1a5a26b4 +dummy-pkg-a 1.1.0 b28c6dc43fc68172975b324020cf2266 +dummy-pkg-a 2.0.0 48fd354677a031497800da75a6fea68c +dummy-pkg-a 2.1.0.rc1 3c5834b0450820da13956ca11e668e7a +dummy-pkg-b 1.0.0 8285ab1075a8ad736c6c5f6640e2a7b4 +dummy-pkg-b 1.1.0 6eb2d48e2ee123f80bde3304865a492a +dummy-pkg-b 1.2.0 3a55f5f8a99bf9a76a61f34e5a1226fc diff --git a/updater/spec/fixtures/rubygems-info-a b/updater/spec/fixtures/rubygems-info-a new file mode 100644 index 000000000..ce8008061 --- /dev/null +++ b/updater/spec/fixtures/rubygems-info-a @@ -0,0 +1,6 @@ +--- +1.0.0 |checksum:bf80371809ed088b2a99ed8bd5640e02b95e5cbbfd27350801cbfdf137abe363 +1.0.1 |checksum:f8ec34efa64c2d74c29710ecbf33296f11c57f248d12af70dd9d107f97d23807 +1.1.0 |checksum:be4df310095b9fb2b3f1c70204c706f3dce52a6418d2719ce832c9f20526e382 +2.0.0 |checksum:45ffe617cf34fa9acf7a9f153c6c4f723e53b539f9eef737b03629e8eb8aa858 +2.1.0.rc1 |checksum:79d997001f9cc71bf3df1abd89396d2ecc990b0aada04c3d9f692b644396c843,rubygems:> 1.3.1 diff --git a/updater/spec/fixtures/rubygems-info-b b/updater/spec/fixtures/rubygems-info-b new file mode 100644 index 000000000..4a54f7824 --- /dev/null +++ b/updater/spec/fixtures/rubygems-info-b @@ -0,0 +1,4 @@ +--- +1.0.0 dummy-pkg-a:< 2.0.0|checksum:0147d64042d5ab109d185f54957fcfb88f1ff9158651944ff75c6c82d47ab444 +1.1.0 dummy-pkg-a:~> 2.0|checksum:c8725691239b43d5f4c343b64d30afae6dd25ff1a79cca4d80534804c638b113 +1.2.0 dummy-pkg-a:~> 2.0|checksum:51b99c7db0d39924d690e19282f63d1fba9cc002ef55a139d9b6a4b0469399a1 diff --git a/updater/spec/fixtures/rubygems-versions-a.json b/updater/spec/fixtures/rubygems-versions-a.json new file mode 100644 index 000000000..4740cd92c --- /dev/null +++ b/updater/spec/fixtures/rubygems-versions-a.json @@ -0,0 +1,59 @@ +[ + { + "authors": "Dependabot", + "built_at": "2017-06-08T00:00:00.000Z", + "created_at": "2017-06-08T11:17:36.474Z", + "description": "", + "downloads_count": 3115, + "metadata": {}, + "number": "1.2.0", + "summary": "A dummy package for testing Dependabot", + "platform": "ruby", + "rubygems_version": ">= 0", + "ruby_version": ">= 0", + "prerelease": false, + "licenses": [ + "MIT" + ], + "requirements": [], + "sha": "51b99c7db0d39924d690e19282f63d1fba9cc002ef55a139d9b6a4b0469399a1" + }, + { + "authors": "Dependabot", + "built_at": "2017-06-08T00:00:00.000Z", + "created_at": "2017-06-08T11:08:53.333Z", + "description": "", + "downloads_count": 1562, + "metadata": {}, + "number": "1.1.0", + "summary": "A dummy package for testing Dependabot", + "platform": "ruby", + "rubygems_version": ">= 0", + "ruby_version": ">= 0", + "prerelease": false, + "licenses": [ + "MIT" + ], + "requirements": [], + "sha": "c8725691239b43d5f4c343b64d30afae6dd25ff1a79cca4d80534804c638b113" + }, + { + "authors": "Dependabot", + "built_at": "2017-06-08T00:00:00.000Z", + "created_at": "2017-06-08T11:08:20.605Z", + "description": "", + "downloads_count": 1569, + "metadata": {}, + "number": "1.0.0", + "summary": "A dummy package for testing Dependabot", + "platform": "ruby", + "rubygems_version": ">= 0", + "ruby_version": ">= 0", + "prerelease": false, + "licenses": [ + "MIT" + ], + "requirements": [], + "sha": "0147d64042d5ab109d185f54957fcfb88f1ff9158651944ff75c6c82d47ab444" + } +] diff --git a/updater/spec/fixtures/rubygems-versions-b.json b/updater/spec/fixtures/rubygems-versions-b.json new file mode 100644 index 000000000..4740cd92c --- /dev/null +++ b/updater/spec/fixtures/rubygems-versions-b.json @@ -0,0 +1,59 @@ +[ + { + "authors": "Dependabot", + "built_at": "2017-06-08T00:00:00.000Z", + "created_at": "2017-06-08T11:17:36.474Z", + "description": "", + "downloads_count": 3115, + "metadata": {}, + "number": "1.2.0", + "summary": "A dummy package for testing Dependabot", + "platform": "ruby", + "rubygems_version": ">= 0", + "ruby_version": ">= 0", + "prerelease": false, + "licenses": [ + "MIT" + ], + "requirements": [], + "sha": "51b99c7db0d39924d690e19282f63d1fba9cc002ef55a139d9b6a4b0469399a1" + }, + { + "authors": "Dependabot", + "built_at": "2017-06-08T00:00:00.000Z", + "created_at": "2017-06-08T11:08:53.333Z", + "description": "", + "downloads_count": 1562, + "metadata": {}, + "number": "1.1.0", + "summary": "A dummy package for testing Dependabot", + "platform": "ruby", + "rubygems_version": ">= 0", + "ruby_version": ">= 0", + "prerelease": false, + "licenses": [ + "MIT" + ], + "requirements": [], + "sha": "c8725691239b43d5f4c343b64d30afae6dd25ff1a79cca4d80534804c638b113" + }, + { + "authors": "Dependabot", + "built_at": "2017-06-08T00:00:00.000Z", + "created_at": "2017-06-08T11:08:20.605Z", + "description": "", + "downloads_count": 1569, + "metadata": {}, + "number": "1.0.0", + "summary": "A dummy package for testing Dependabot", + "platform": "ruby", + "rubygems_version": ">= 0", + "ruby_version": ">= 0", + "prerelease": false, + "licenses": [ + "MIT" + ], + "requirements": [], + "sha": "0147d64042d5ab109d185f54957fcfb88f1ff9158651944ff75c6c82d47ab444" + } +] diff --git a/updater/spec/spec_helper.rb b/updater/spec/spec_helper.rb index 1a085708e..bc9455743 100644 --- a/updater/spec/spec_helper.rb +++ b/updater/spec/spec_helper.rb @@ -54,6 +54,12 @@ def fixture(path) File.read(File.join("spec", "fixtures", path)) end + + def job_definition_fixture(path) + YAML.load( + fixture(File.join("job_definitions", "#{path}.yaml")) + ) + end end VCR.configure do |config| diff --git a/updater/spec/support/dependency_file_helpers.rb b/updater/spec/support/dependency_file_helpers.rb new file mode 100644 index 000000000..9ace9d322 --- /dev/null +++ b/updater/spec/support/dependency_file_helpers.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module DependencyFileHelpers + def encode_dependency_files(files) + files.map do |file| + base64_file = file.dup + base64_file.content = Base64.encode64(file.content) unless file.binary? + base64_file.to_h + end + end +end diff --git a/updater/spec/support/dummy_pkg_helpers.rb b/updater/spec/support/dummy_pkg_helpers.rb new file mode 100644 index 000000000..c22572b6d --- /dev/null +++ b/updater/spec/support/dummy_pkg_helpers.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +# This module provides some shortcuts for working with our two mock RubyGems packages: +# - https://rubygems.org/gems/dummy-pkg-a +# - https://rubygems.org/gems/dummy-pkg-b +# +module DummyPkgHelpers + def stub_rubygems_calls + stub_request(:get, "https://index.rubygems.org/versions"). + to_return(status: 200, body: fixture("rubygems-index")) + + stub_request(:get, "https://index.rubygems.org/info/dummy-pkg-a"). + to_return(status: 200, body: fixture("rubygems-info-a")) + stub_request(:get, "https://rubygems.org/api/v1/versions/dummy-pkg-a.json"). + to_return(status: 200, body: fixture("rubygems-versions-a.json")) + + stub_request(:get, "https://index.rubygems.org/info/dummy-pkg-b"). + to_return(status: 200, body: fixture("rubygems-info-b")) + stub_request(:get, "https://rubygems.org/api/v1/versions/dummy-pkg-b.json"). + to_return(status: 200, body: fixture("rubygems-versions-b.json")) + end + + def original_bundler_files(fixture: "bundler", directory: "/") + bundler_files_for(fixture: fixture, state: "original", directory: directory) + end + + def updated_bundler_files(fixture: "bundler", directory: "/") + bundler_files_for(fixture: fixture, state: "updated", directory: directory) + end + + def bundler_files_for(fixture:, state:, directory: "/") + [ + Dependabot::DependencyFile.new( + name: "Gemfile", + content: fixture("#{fixture}/#{state}/Gemfile"), + directory: directory + ), + Dependabot::DependencyFile.new( + name: "Gemfile.lock", + content: fixture("#{fixture}/#{state}/Gemfile.lock"), + directory: directory + ) + ] + end + + def create_temporary_content_directory(fixture:, directory: "/", state: "original") + tmp_dir = Dir.mktmpdir + FileUtils.cp_r(File.join("spec", "fixtures", fixture, state, "/."), File.join(tmp_dir, directory)) + + # The content directory needs to a repo + Dir.chdir(tmp_dir) do + system("git init . && git add . && git commit --allow-empty -m 'Init'", out: File::NULL) + end + + tmp_dir + end + + def updated_bundler_files_hash(fixture: "bundler") + updated_bundler_files(fixture: fixture).map(&:to_h) + end +end