Import updater files from dependabot-core

copy-updater-files.ps1

@@ -2,21 +2,120 @@ Param(
     [string] $tag = "v0.230.0"
 )
 
-$hash = [ordered]@{
-    ".ruby-version"                               = "../.ruby-version"
+$files = @(
+    ".ruby-version"
 
-    "updater/lib/dependabot/environment.rb"       = "lib/dependabot/environment.rb"
-    "updater/spec/dependabot/environment_spec.rb" = "spec/dependabot/environment_spec.rb"
-    # "updater/spec/spec_helper.rb"                 = "spec/spec_helper.rb"
-}
+    "updater/bin/fetch_files.rb"
+    "updater/bin/update_files.rb"
+
+    "updater/config/.npmrc"
+    "updater/config/.yarnrc"
+
+    "updater/lib/dependabot/logger/formats.rb"
+    "updater/lib/dependabot/updater/operations/create_group_update_pull_request.rb"
+    "updater/lib/dependabot/updater/operations/create_security_update_pull_request.rb"
+    "updater/lib/dependabot/updater/operations/group_update_all_versions.rb"
+    "updater/lib/dependabot/updater/operations/refresh_group_update_pull_request.rb"
+    "updater/lib/dependabot/updater/operations/refresh_security_update_pull_request.rb"
+    "updater/lib/dependabot/updater/operations/refresh_version_update_pull_request.rb"
+    "updater/lib/dependabot/updater/operations/update_all_versions.rb"
+    "updater/lib/dependabot/updater/dependency_group_change_batch.rb"
+    "updater/lib/dependabot/updater/error_handler.rb"
+    "updater/lib/dependabot/updater/errors.rb"
+    "updater/lib/dependabot/updater/group_update_creation.rb"
+    "updater/lib/dependabot/updater/operations.rb"
+    "updater/lib/dependabot/updater/security_update_helpers.rb"
+    "updater/lib/dependabot/api_client.rb"
+    "updater/lib/dependabot/base_command.rb"
+    "updater/lib/dependabot/dependency_change.rb"
+    "updater/lib/dependabot/dependency_change_builder.rb"
+    "updater/lib/dependabot/dependency_group_engine.rb"
+    "updater/lib/dependabot/dependency_snapshot.rb"
+    "updater/lib/dependabot/environment.rb"
+    "updater/lib/dependabot/file_fetcher_command.rb"
+    "updater/lib/dependabot/job.rb"
+    "updater/lib/dependabot/sentry.rb"
+    "updater/lib/dependabot/service.rb"
+    "updater/lib/dependabot/setup.rb"
+    "updater/lib/dependabot/update_files_command.rb"
+    "updater/lib/dependabot/updater.rb"
+
+    # "updater/spec/dependabot/updater/operations/group_update_all_versions_spec.rb"
+    # "updater/spec/dependabot/updater/operations/refresh_group_update_pull_request_spec.rb"
+    "updater/spec/dependabot/updater/error_handler_spec.rb"
+    "updater/spec/dependabot/updater/operations_spec.rb"
+    "updater/spec/dependabot/api_client_spec.rb"
+    # "updater/spec/dependabot/dependency_change_builder_spec.rb"
+    "updater/spec/dependabot/dependency_change_spec.rb"
+    "updater/spec/dependabot/dependency_group_engine_spec.rb"
+    # "updater/spec/dependabot/dependency_snapshot_spec.rb"
+    "updater/spec/dependabot/environment_spec.rb"
+    # "updater/spec/dependabot/file_fetcher_command_spec.rb"
+    # "updater/spec/dependabot/integration_spec.rb"
+    "updater/spec/dependabot/job_spec.rb"
+    "updater/spec/dependabot/sentry_spec.rb"
+    "updater/spec/dependabot/service_spec.rb"
+    # "updater/spec/dependabot/update_files_command_spec.rb"
+    # "updater/spec/dependabot/updater_spec.rb"
+
+    "updater/spec/fixtures/rubygems-index"
+    "updater/spec/fixtures/rubygems-info-a"
+    "updater/spec/fixtures/rubygems-versions-a.json"
+    "updater/spec/fixtures/rubygems-info-b"
+    "updater/spec/fixtures/rubygems-versions-b.json"
+    "updater/spec/fixtures/bundler/original/Gemfile"
+    "updater/spec/fixtures/bundler/original/Gemfile.lock"
+    "updater/spec/fixtures/bundler/updated/Gemfile"
+    "updater/spec/fixtures/bundler/updated/Gemfile.lock"
+    "updater/spec/fixtures/bundler_gemspec/original/Gemfile"
+    "updater/spec/fixtures/bundler_gemspec/original/Gemfile.lock"
+    "updater/spec/fixtures/bundler_gemspec/original/library.gemspec"
+    "updater/spec/fixtures/bundler_git/original/Gemfile"
+    "updater/spec/fixtures/bundler_git/original/Gemfile.lock"
+    "updater/spec/fixtures/bundler_grouped_by_types/original/Gemfile"
+    "updater/spec/fixtures/bundler_grouped_by_types/original/Gemfile.lock"
+    "updater/spec/fixtures/bundler_vendored/original/Gemfile"
+    "updater/spec/fixtures/bundler_vendored/original/Gemfile.lock"
+    "updater/spec/fixtures/docker/original/Dockerfile.bundler"
+    "updater/spec/fixtures/docker/original/Dockerfile.cargo"
+    "updater/spec/fixtures/jobs/job_with_credentials.json"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_by_dependency_type.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_empty_group.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_overlapping_groups.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_existing_pr.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_ungrouped.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_with_vendoring.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_semver_grouping_with_global_ignores.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_all_semver_grouping.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_peer_manifests.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_dependencies_changed.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_empty_group.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_missing_group.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_versions_changed.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/group_update_refresh_similar_pr.yaml"
+    "updater/spec/fixtures/job_definitions/bundler/version_updates/update_all_simple.yaml"
+    "updater/spec/fixtures/job_definitions/docker/version_updates/group_update_peer_manifests.yaml"
+    "updater/spec/fixtures/job_definitions/README.md"
+
+    "updater/spec/support/dependency_file_helpers.rb"
+    "updater/spec/support/dummy_pkg_helpers.rb"
+
+    # "updater/spec/spec_helper.rb"
+)
 
 $baseUrl = "https://raw.githubusercontent.com/dependabot/dependabot-core"
-$destinationFolder = Join-Path -Path '.' -ChildPath 'updater'
-
-foreach ($h in $hash.GetEnumerator()) {
-    $sourceUrl = "$baseUrl/$tag/$($h.Name)"
-    $destinationPath = Join-Path -Path "$destinationFolder" -ChildPath "$($h.Value)"
-    Write-Host "`Downloading $($h.Name) ..."
-    [System.IO.Directory]::CreateDirectory("$(Split-Path -Path "$destinationPath")") | Out-Null
-    Invoke-WebRequest -Uri $sourceUrl -OutFile $destinationPath
+
+foreach ($name in $files) {
+    $sourceUrl = "$baseUrl/$tag/$($name)"
+    $destinationPath = Join-Path -Path '.' -ChildPath "$name"
+
+    # Write-Host "`Downloading $name ..."
+    # [System.IO.Directory]::CreateDirectory("$(Split-Path -Path "$destinationPath")") | Out-Null
+    # Invoke-WebRequest -Uri $sourceUrl -OutFile $destinationPath
+
+    echo "Downloading $($name) ..."
+    mkdir -p "$(dirname "$destinationPath")"
+    curl -sL "$sourceUrl" -o "$destinationPath"
 }

updater/Gemfile

@@ -11,6 +11,11 @@ source "https://rubygems.org"
 # gem "dependabot-omnibus", github: "dependabot/dependabot-core", branch: "main"
 gem "dependabot-omnibus", "~>0.232.0"
 
+gem "http", "~> 5.1"
+gem "octokit", "6.1.1"
+gem "sentry-raven", "~> 3.1"
+gem "terminal-table", "~> 3.0.2"
+
 group :test do
   gem "rspec"
   gem "rubocop"

updater/Gemfile.lock

@@ -107,19 +107,32 @@ GEM
     faraday-net_http (3.0.2)
     faraday-retry (2.2.0)
       faraday (~> 2.0)
+    ffi (1.15.5)
+    ffi-compiler (1.0.1)
+      ffi (>= 1.0.0)
+      rake
     gitlab (4.19.0)
       httparty (~> 0.20)
       terminal-table (>= 1.5.1)
     hashdiff (1.0.1)
+    http (5.1.1)
+      addressable (~> 2.8)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.2)
+      llhttp-ffi (~> 0.4.0)
     http-accept (1.7.0)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
+    http-form_data (2.3.0)
     httparty (0.21.0)
       mini_mime (>= 1.0.0)
       multi_xml (>= 0.5.2)
     jmespath (1.6.2)
     json (2.6.3)
     language_server-protocol (3.17.0.3)
+    llhttp-ffi (0.4.0)
+      ffi-compiler (~> 1.0)
+      rake (~> 13.0)
     mime-types (3.5.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2023.0808)
@@ -143,6 +156,7 @@ GEM
     public_suffix (5.0.3)
     racc (1.7.1)
     rainbow (3.1.1)
+    rake (13.0.6)
     regexp_parser (2.8.1)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
@@ -185,6 +199,8 @@ GEM
     sawyer (0.9.2)
       addressable (>= 2.3.5)
       faraday (>= 0.17.3, < 3)
+    sentry-raven (3.1.2)
+      faraday (>= 1.0)
     sorbet-runtime (0.5.11014)
     stringio (3.0.8)
     terminal-table (3.0.2)
@@ -207,9 +223,13 @@ PLATFORMS
 
 DEPENDENCIES
   dependabot-omnibus (~> 0.232.0)
+  http (~> 5.1)
+  octokit (= 6.1.1)
   rspec
   rubocop
   rubocop-performance
+  sentry-raven (~> 3.1)
+  terminal-table (~> 3.0.2)
   vcr
   webmock
 

updater/bin/fetch_files.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(__dir__ + "/../lib")
+
+$stdout.sync = true
+
+require "raven"
+require "dependabot/setup"
+require "dependabot/file_fetcher_command"
+require "debug" if ENV["DEBUG"]
+
+class UpdaterKilledError < StandardError; end
+
+trap("TERM") do
+  puts "Received SIGTERM"
+  error = UpdaterKilledError.new("Updater process killed with SIGTERM")
+  tags = { update_job_id: ENV.fetch("DEPENDABOT_JOB_ID", nil) }
+  Raven.capture_exception(error, tags: tags)
+  exit
+end
+
+begin
+  Dependabot::FileFetcherCommand.new.run
+rescue Dependabot::RunFailure
+  exit 1
+end

updater/bin/update_files.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(__dir__ + "/../lib")
+
+$stdout.sync = true
+
+require "raven"
+require "dependabot/setup"
+require "dependabot/update_files_command"
+require "debug" if ENV["DEBUG"]
+
+class UpdaterKilledError < StandardError; end
+
+trap("TERM") do
+  puts "Received SIGTERM"
+  error = UpdaterKilledError.new("Updater process killed with SIGTERM")
+  tags = { update_job_id: ENV.fetch("DEPENDABOT_JOB_ID", nil) }
+  Raven.capture_exception(error, tags: tags)
+  exit
+end
+
+begin
+  Dependabot::UpdateFilesCommand.new.run
+rescue Dependabot::RunFailure
+  exit 1
+end

updater/config/.npmrc

@@ -0,0 +1,20 @@
+# TODO: Remove these hacks once we've deprecated npm 6 support as it no longer
+# spwans a child process to npm install git dependencies.
+
+# Only set our custom CA cert for npm because the system ca's + our custom ca
+# causes npm to blow up when installing git dependencies (E2BIG exception). This
+# happens because the ca-file contents are passed as a cli argument to npm
+# install from npm/cli/lib/pack.js as --ca="contents of ca file" - "ca" is
+# populated automatically by npm when setting "--cafile" and passed through in
+# when spawning the cli to install git dependencies.
+cafile=/usr/local/share/ca-certificates/dbot-ca.crt
+# Because npm doesn't pass through all npm config when doing git installs in
+# npm/cli/lib/pack.js we also need to disable audit here to prevent npm from
+# auditing git dependencies, we do this to sped up installs
+audit=false
+# Similarly, dry-run and ignore-scripts are also not passed through when doing
+# git installs in npm/cli/lib/pack.js so we set dry-run and ignore-scripts to
+# prevent any lifecycle hooks for git installs. dry-run disables "prepare" and
+# "prepack" scripts, ignore-scripts disables all other scripts
+dry-run=true
+ignore-scripts=true

updater/config/.yarnrc

@@ -0,0 +1,6 @@
+# TODO: Remove these hacks once we've deprecated npm 6 support as it no longer
+# spwans a child process to npm install git dependencies.
+# yarn lockfile v1
+
+# Tell yarn to use the system-wide CA bundle overriding the .npmrc cafile
+cafile "/etc/ssl/certs/ca-certificates.crt"