diff --git a/app/jobs/shipit/cache_deploy_spec_job.rb b/app/jobs/shipit/cache_deploy_spec_job.rb index b9e5aad91..671ebf9c3 100644 --- a/app/jobs/shipit/cache_deploy_spec_job.rb +++ b/app/jobs/shipit/cache_deploy_spec_job.rb @@ -18,9 +18,8 @@ def perform(stack) commit = stack.commits.reachable.last commands = Commands.for(stack) - commands.with_temporary_working_directory(commit:, recursive: false) do |path| - stack.update!(cached_deploy_spec: DeploySpec::FileSystem.new(path, stack)) - end + spec = commands.cacheable_deploy_spec(commit:) + stack.update!(cached_deploy_spec: spec) # A duplicate enqueued while this job held the dedupe lock was dropped; # if the head moved under us, that dropped job's work is still diff --git a/app/models/shipit/deploy_spec/file_system.rb b/app/models/shipit/deploy_spec/file_system.rb index 36fac6b3f..fc55f1f49 100644 --- a/app/models/shipit/deploy_spec/file_system.rb +++ b/app/models/shipit/deploy_spec/file_system.rb @@ -24,6 +24,11 @@ def cacheable DeploySpec.new(cacheable_config) end + # NOTE: +file+ and +build_config+ (which probes and reads the + # inherit_from chain) are the only methods through which spec + # evaluation touches the disk. GitObjectFileSystem relies on that + # invariant by overriding both; if you add a new disk access, route it + # through one of them or update the subclass. def file(path, root: false) if root || directory.blank? @app_dir.join(path) diff --git a/app/models/shipit/deploy_spec/git_object_file_system.rb b/app/models/shipit/deploy_spec/git_object_file_system.rb new file mode 100644 index 000000000..de82b665f --- /dev/null +++ b/app/models/shipit/deploy_spec/git_object_file_system.rb @@ -0,0 +1,197 @@ +# frozen_string_literal: true + +module Shipit + class DeploySpec + # A DeploySpec::FileSystem that reads files straight out of the git object + # database instead of a working-tree checkout. + # + # It is constructed with an *empty* temporary directory. Every disk access + # in the spec evaluation code funnels through exactly two seams of the + # parent class: +file+ (framework discovery probes and config candidates) + # and +read_config+ (the +inherit_from+ chain). Both are overridden here + # to materialize the requested path from `git cat-file` into the temporary + # directory *before* returning, so callers' subsequent +exist?+/+read+ + # calls behave exactly as they would against a full checkout. + # + # Whenever byte-identical behavior with a checkout cannot be guaranteed + # (symlinks, submodules, .gitattributes in an ancestor directory, paths + # escaping the repository, runaway inherit_from chains), FallbackRequired + # is raised and the caller is expected to fall back to the checkout-based + # code path. + # + # Unlike the checkout path (which clones a snapshot first), reads target + # the stack's live git cache. A concurrent ClearGitCacheJob or git gc + # degrades to a command failure, which the caller treats as a fallback; + # since every read is pinned to a single commit sha there is no torn-read + # hazard. + class GitObjectFileSystem < FileSystem + class FallbackRequired < StandardError + attr_reader :reason, :detail + + def initialize(reason, detail = nil) + @reason = reason + @detail = detail + super("#{reason}: #{detail}") + end + end + + MAX_INHERIT_READS = 10 + GLOB_CHARS = /[*?\[]/ + TREE_MODE = '040000' + SYMLINK_MODE = '120000' + GITLINK_MODE = '160000' + + def initialize(app_dir, stack, commands:, sha:) + super(app_dir, stack) + @root = @app_dir.cleanpath + @commands = commands + @sha = sha + @listings = {} + @materialized = Set.new + @inherit_reads = 0 + @inherit_chain = [] + end + + def file(path, root: false) + pathname = super + if path.to_s.match?(GLOB_CHARS) + dir = repo_rel(pathname.dirname) + validate_dir!(dir) + pattern = File.basename(path.to_s) + entries(dir).each_key do |name| + next unless File.fnmatch(pattern, name) + + materialize(dir.empty? ? name : File.join(dir, name)) + end + else + materialize(repo_rel(pathname)) + end + pathname + end + + private + + # The inherit_from seam. The parent class's build_config checks + # inherits_from_path.exist? BEFORE calling read_config, so interception + # must happen here: resolve the reference the same way the parent is + # about to (path.dirname.join(value)), validate containment, bound the + # chain, and materialize -- then let the parent proceed against a real + # file. The old checkout-based path is uncapped (a cycle loops forever) + # and follows escaping paths onto the worker filesystem; both are hard + # fallbacks here. + def build_config(path, config_obj) + if config_obj.present? && config_obj.key?(SHIPIT_CONFIG_INHERIT_FROM_KEY) + @inherit_reads += 1 + raise FallbackRequired.new(:inherit_depth, @inherit_chain.join(' -> ')) if @inherit_reads > MAX_INHERIT_READS + + inherits_from = path.dirname.join(config_obj[SHIPIT_CONFIG_INHERIT_FROM_KEY]) + key = repo_rel(inherits_from) + @inherit_chain << key + materialize(key) + end + super + end + + # Containment validation. Inputs are always tmpdir-absolute Pathnames + # (both seams receive resolved paths). Returns the canonical + # repo-relative key ("" for the root itself, no leading "./", no + # trailing "/"). The tmpdir was created by us, so @root is canonical + # and symlink-free; cleanpath-based prefix containment is sufficient + # and requires no filesystem access. + def repo_rel(pathname) + clean = Pathname(pathname).cleanpath + string = clean.to_s + root = @root.to_s + raise FallbackRequired.new(:escape, string) unless string == root || string.start_with?("#{root}/") + + string == root ? "" : string[(root.length + 1)..] + end + + # Single choke point: every file access materializes here or is + # provably absent at the commit. Idempotent per instance. + def materialize(key) + return if @materialized.include?(key) + + case blob_mode(key) + when :absent, TREE_MODE + # Absent at this commit, or a directory itself: nothing to write. + @materialized << key + when SYMLINK_MODE + # `git cat-file` on a symlink returns the link target *text* as if + # it were file content. Never serve that. + raise FallbackRequired.new(:symlink, key) + when GITLINK_MODE + raise FallbackRequired.new(:submodule, key) + else + content = @commands.git_read_object(@sha, key) + target = @root.join(key) + target.dirname.mkpath + File.binwrite(target, content) + @materialized << key + end + end + + # Walks the path's directory components (validating each one) and + # returns the final component's git mode, or :absent. + def blob_mode(key) + parts = key.split('/') + prefix = "" + parts[0...-1].each do |component| + mode = entries(prefix)[component] + path_so_far = prefix.empty? ? component : "#{prefix}/#{component}" + case mode + when nil + return :absent + when TREE_MODE + prefix = path_so_far + when SYMLINK_MODE + raise FallbackRequired.new(:symlink, path_so_far) + when GITLINK_MODE + raise FallbackRequired.new(:submodule, path_so_far) + else # a regular file as an intermediate path component + raise FallbackRequired.new(:file_in_path, path_so_far) + end + end + + (parts.last && entries(prefix)[parts.last]) || :absent + end + + # The glob branch lists a directory without materializing a file, so the + # directory's own path components must be validated explicitly: + # `git ls-tree -- '/'` on a symlinked directory or on a + # regular file returns an empty listing with exit 0, which would + # silently diverge from Dir[] on a checkout (which follows symlinks). + def validate_dir!(dir) + return if dir.empty? + + case blob_mode(dir) + when TREE_MODE, :absent + nil + when SYMLINK_MODE + raise FallbackRequired.new(:symlink, dir) + when GITLINK_MODE + raise FallbackRequired.new(:submodule, dir) + else + raise FallbackRequired.new(:file_in_path, dir) + end + end + + # Memoized directory listings, keyed by canonical repo-relative dir + # ("" = root). Any listed ancestor containing a .gitattributes entry + # forces a fallback: a checkout applies eol/text/smudge attributes, + # `git cat-file` emits raw bytes, and attributes affecting a path can + # only live in its ancestor directories -- which are exactly the ones + # we list. A .gitattributes in an unrelated subtree never triggers this. + def entries(dir) + @listings[dir] ||= begin + listing = @commands.git_ls_dir(@sha, dir) + if listing.key?('.gitattributes') + raise FallbackRequired.new(:gitattributes, dir.empty? ? '' : dir) + end + + listing + end + end + end + end +end diff --git a/lib/shipit.rb b/lib/shipit.rb index 1b6f738f2..647d648b9 100644 --- a/lib/shipit.rb +++ b/lib/shipit.rb @@ -78,6 +78,26 @@ def task_execution_strategy @task_execution_strategy ||= Shipit::TaskExecutionStrategy::Default end + CHECKOUT_LESS_DEPLOY_SPEC_MODES = %i[disabled shadow enabled].freeze + + # Controls how cacheable deploy specs are evaluated. See + # StackCommands#cacheable_deploy_spec. :shadow runs both the checkout-based + # and the git-object-database paths and reports divergence; :enabled serves + # from the git object database with automatic fallback to a checkout. + def checkout_less_deploy_spec + @checkout_less_deploy_spec || :disabled + end + + def checkout_less_deploy_spec=(mode) + mode = mode.to_sym if mode.respond_to?(:to_sym) + unless CHECKOUT_LESS_DEPLOY_SPEC_MODES.include?(mode) + raise ArgumentError, + "checkout_less_deploy_spec must be one of #{CHECKOUT_LESS_DEPLOY_SPEC_MODES.inspect}, got #{mode.inspect}" + end + + @checkout_less_deploy_spec = mode + end + self.timeout_exit_codes = [].freeze self.respect_bare_shipit_file = true diff --git a/lib/shipit/stack_commands.rb b/lib/shipit/stack_commands.rb index 3097f5391..d03eb196e 100644 --- a/lib/shipit/stack_commands.rb +++ b/lib/shipit/stack_commands.rb @@ -3,6 +3,7 @@ # rubocop:disable Lint/MissingCopEnableDirective, Lint/MissingSuper require 'pathname' require 'fileutils' +require 'open3' module Shipit class StackCommands < Commands @@ -59,8 +60,86 @@ def fetch_deployed_revision end def build_cacheable_deploy_spec - with_temporary_working_directory(recursive: false) do |dir| - DeploySpec::FileSystem.new(dir, @stack).cacheable + cacheable_deploy_spec(commit: nil) + end + + # Evaluates the stack's cacheable deploy spec and returns it as a plain, + # disk-detached Shipit::DeploySpec. Depending on Shipit.checkout_less_deploy_spec: + # :disabled -> checkout-based path (today's behavior) + # :enabled -> git-object-database path; any doubt or error falls back + # to the checkout-based path + # :shadow -> checkout-based result is authoritative and returned; the + # git-object path additionally runs and any divergence is + # reported, but never raises + def cacheable_deploy_spec(commit: nil) + mode = Shipit.checkout_less_deploy_spec + return checkout_cacheable_deploy_spec(commit).first if mode == :disabled || commit.nil? + + case mode + when :enabled + begin + spec, = git_object_cacheable_deploy_spec(commit) + notify_checkout_less(:hit) + spec + rescue DeploySpec::GitObjectFileSystem::FallbackRequired => e + notify_checkout_less(:fallback, reason: e.reason, detail: e.detail) + checkout_cacheable_deploy_spec(commit).first + rescue Command::Error, SystemCallError => e + notify_checkout_less(:fallback, reason: :git_error, detail: e.message) + checkout_cacheable_deploy_spec(commit).first + rescue StandardError => e + notify_checkout_less(:fallback, reason: :unexpected_error, detail: "#{e.class}: #{e.message}") + checkout_cacheable_deploy_spec(commit).first + end + when :shadow + old_spec, old_root = checkout_cacheable_deploy_spec(commit) + begin + new_spec, new_root = git_object_cacheable_deploy_spec(commit) + # Specs legitimately embed their (ephemeral) evaluation directory in + # some values (e.g. release-gem /x.gemspec), so both sides are + # compared with their own root normalized out. + old_config = normalize_spec_config(old_spec.config, old_root) + new_config = normalize_spec_config(new_spec.config, new_root) + if old_config == new_config + notify_checkout_less(:hit) + else + differing = (old_config.keys | new_config.keys) + .reject { |key| old_config[key] == new_config[key] } + notify_checkout_less(:shadow_mismatch, detail: differing.first(5).join(',')) + end + rescue DeploySpec::GitObjectFileSystem::FallbackRequired => e + # Preserve the reason taxonomy: shadow mode is what produces the + # fallback-rate data the rollout gate is evaluated on. + notify_checkout_less(:fallback, reason: e.reason, detail: e.detail) + rescue StandardError => e + notify_checkout_less(:fallback, reason: :shadow_error, detail: "#{e.class}: #{e.message}") + end + old_spec + end + end + + # Raw blob content at a commit, bypassing Shipit::Command: Command runs + # through a PTY, which rewrites newlines and is unsafe for raw bytes and + # NUL-delimited output. Local object-database reads need no env/auth. + def git_read_object(sha, repo_rel_path) + git_read('cat-file', 'blob', "#{sha}:#{repo_rel_path}") + end + + # Directory listing at a commit: { name => mode }. "" lists the root + # tree. The trailing slash on the pathspec lists the directory's + # children rather than the directory entry itself. + def git_ls_dir(sha, repo_rel_dir) + args = ['ls-tree', '-z', sha] + # :(literal) disables pathspec magic so directory names containing + # glob characters or a leading ':' are taken verbatim. + args += ['--', ":(literal)#{repo_rel_dir}/"] unless repo_rel_dir.empty? + output = git_read(*args) + output.split("\0").each_with_object({}) do |record, listing| + next if record.empty? + + meta, name = record.split("\t", 2) # limit 2: filenames may contain tabs + mode = meta.split(' ', 3).first + listing[File.basename(name)] = mode end end @@ -129,6 +208,94 @@ def quiet_git_arg private + # Both path methods return [spec, evaluation_root] so shadow mode can + # normalize root-dependent values out of the comparison. + def checkout_cacheable_deploy_spec(commit) + with_temporary_working_directory(commit:, recursive: false) do |dir| + [DeploySpec::FileSystem.new(dir, @stack).cacheable, dir.to_s] + end + end + + def git_object_cacheable_deploy_spec(commit) + ensure_no_checkout_conversion_config! + + unless fetched?(commit).tap(&:run).success? + @stack.acquire_git_cache_lock do + fetch.run! unless fetched?(commit).tap(&:run).success? + end + end + + Dir.mktmpdir do |dir| + [DeploySpec::GitObjectFileSystem.new(dir, @stack, commands: self, sha: commit.sha).cacheable, dir.to_s] + end + end + + # A checkout applies core.autocrlf conversions and attributes from a + # core.attributesfile, neither of which leaves a trace in the tree, while + # `git cat-file` emits raw bytes. Guarding at runtime (rather than a + # point-in-time preflight) keeps the guarantee if a base image change + # introduces such config later. autocrlf=false and autocrlf=input do not + # convert on checkout; core.eol alone is inert without text attributes, + # which are covered by the in-tree .gitattributes guard and the + # attributesfile check here. + def ensure_no_checkout_conversion_config! + output = git_read('config', '--get-regexp', '^core\.(autocrlf|attributesfile)$', allow_failure: true) + output.split("\n").each do |line| + key, value = line.split(' ', 2) + next if key == 'core.autocrlf' && %w[false input].include?(value.to_s.downcase) + + raise DeploySpec::GitObjectFileSystem::FallbackRequired.new(:git_config, line) + end + end + + def normalize_spec_config(value, root) + case value + when String then value.gsub(root, '$SPEC_ROOT') + when Hash then value.transform_values { |nested| normalize_spec_config(nested, root) } + when Array then value.map { |nested| normalize_spec_config(nested, root) } + else value + end + end + + # Environment variables that select which repository/object store git + # operates on. Cleared explicitly: git_read bypasses Shipit::Command (and + # therefore its scrubbed BASE_ENV), and any of these set on the worker + # process would silently override chdir:. + GIT_REPO_SELECTION_ENV = { + 'GIT_DIR' => nil, + 'GIT_WORK_TREE' => nil, + 'GIT_INDEX_FILE' => nil, + 'GIT_OBJECT_DIRECTORY' => nil, + 'GIT_ALTERNATE_OBJECT_DIRECTORIES' => nil, + 'GIT_COMMON_DIR' => nil + }.freeze + + def git_read(*args, allow_failure: false) + output, error, status = Open3.capture3( + GIT_REPO_SELECTION_ENV, 'git', *args, + chdir: @stack.git_path.to_s, binmode: true + ) + unless status.success? + return "" if allow_failure + + raise Command::Failed.new("git #{args.first} failed: #{error.strip}", status.exitstatus) + end + + output + end + + def notify_checkout_less(event, reason: nil, detail: nil) + payload = { stack_id: @stack.id, mode: Shipit.checkout_less_deploy_spec, event:, reason:, detail: }.compact + ActiveSupport::Notifications.instrument('checkout_less_deploy_spec.shipit', payload) + if event == :hit + Rails.logger.debug { "[checkout_less_deploy_spec] hit stack=#{@stack.id}" } + else + Rails.logger.warn( + "[checkout_less_deploy_spec] #{event} stack=#{@stack.id} reason=#{reason} detail=#{detail}" + ) + end + end + def github Shipit.github(organization: @stack.repository.owner) end diff --git a/test/jobs/cache_deploy_spec_job_test.rb b/test/jobs/cache_deploy_spec_job_test.rb index 5e71ef05c..51c70a3c0 100644 --- a/test/jobs/cache_deploy_spec_job_test.rb +++ b/test/jobs/cache_deploy_spec_job_test.rb @@ -10,12 +10,11 @@ class CacheDeploySpecJobTest < ActiveSupport::TestCase @job = CacheDeploySpecJob.new end - test "#perform checkout the repository to the last recorded commit and cache the deploy spec" do + test "#perform evaluates the cacheable spec for the last recorded commit and caches it" do @stack.update!(cached_deploy_spec: DeploySpec.new('review' => { 'checklist' => %w[foo bar] })) - dir = Pathname(Dir.tmpdir) - StackCommands.any_instance.expects(:with_temporary_working_directory) - .with(commit: @last_commit, recursive: false).yields(dir) + StackCommands.any_instance.expects(:cacheable_deploy_spec) + .with(commit: @last_commit).returns(DeploySpec.new({})) assert_equal %w[foo bar], @stack.checklist @job.perform(@stack) @@ -46,8 +45,8 @@ class CacheDeploySpecJobTest < ActiveSupport::TestCase @stack.stubs(:commits).returns(stub(reachable:)) @stack.stubs(:update!) # side-effect callbacks are irrelevant to this test - StackCommands.any_instance.expects(:with_temporary_working_directory) - .with(commit: @last_commit, recursive: false).yields(Pathname(Dir.tmpdir)) + StackCommands.any_instance.expects(:cacheable_deploy_spec) + .with(commit: @last_commit).returns(DeploySpec.new({})) assert_enqueued_with(job: CacheDeploySpecJob, args: [@stack]) do @job.perform(@stack) @@ -55,8 +54,8 @@ class CacheDeploySpecJobTest < ActiveSupport::TestCase end test "#perform does not re-enqueue itself when the head is unchanged" do - StackCommands.any_instance.expects(:with_temporary_working_directory) - .with(commit: @last_commit, recursive: false).yields(Pathname(Dir.tmpdir)) + StackCommands.any_instance.expects(:cacheable_deploy_spec) + .with(commit: @last_commit).returns(DeploySpec.new({})) assert_no_enqueued_jobs(only: CacheDeploySpecJob) do @job.perform(@stack) diff --git a/test/models/shipit/deploy_spec/git_object_file_system_test.rb b/test/models/shipit/deploy_spec/git_object_file_system_test.rb new file mode 100644 index 000000000..6a50b42f8 --- /dev/null +++ b/test/models/shipit/deploy_spec/git_object_file_system_test.rb @@ -0,0 +1,322 @@ +# frozen_string_literal: true + +require 'test_helper' +require 'open3' + +module Shipit + class DeploySpec + class GitObjectFileSystemTest < ActiveSupport::TestCase + setup do + @stack = shipit_stacks(:shipit) + @tmpdirs = [] + end + + teardown do + @tmpdirs.each { |dir| FileUtils.rm_rf(dir) } + end + + # --- Golden equivalence: checkout-based and git-object-based specs match --- + + test "golden: env-specific shipit.production.yml" do + assert_golden('shipit.production.yml' => "deploy:\n override:\n - echo deploy\n") + end + + test "golden: bare shipit.yml produces the same warning stub on both paths" do + # respect_bare_shipit_file replaces bare configs with a warning stub; + # both paths must short-circuit identically. + assert_golden('shipit.yml' => "deploy:\n override:\n - echo bare\n") + end + + test "golden: shipit..yml takes priority over bare shipit.yml" do + assert_golden( + 'shipit.yml' => "deploy:\n override:\n - echo wrong\n", + 'shipit.production.yml' => "deploy:\n override:\n - echo right\n" + ) + end + + test "golden: .shipit/ variants" do + assert_golden('.shipit/production.yml' => "deploy:\n override:\n - echo dotdir\n") + end + + test "golden: no config at all, pure discovery" do + assert_golden( + 'Gemfile' => "source 'https://rubygems.org'\n", + 'Gemfile.lock' => "GEM\n" + ) + end + + test "golden: inherit_from in same directory" do + assert_golden( + 'shipit.production.yml' => "inherit_from: base.yml\ndeploy:\n override:\n - echo child\n", + 'base.yml' => "machine:\n environment:\n FOO: bar\n" + ) + end + + test "golden: inherit_from three levels deep across directories" do + assert_golden( + 'shipit.production.yml' => "inherit_from: configs/a.yml\n", + 'configs/a.yml' => "inherit_from: b.yml\ndeploy:\n override:\n - echo a\n", + 'configs/b.yml' => "machine:\n environment:\n DEPTH: '3'\n" + ) + end + + test "golden: machine.directory re-roots discovery" do + assert_golden( + 'shipit.production.yml' => "machine:\n directory: frontend\n", + 'frontend/package.json' => '{"private": true}', + 'frontend/yarn.lock' => "# yarn\n" + ) + end + + test "golden: gemspec glob with zero, one and many matches" do + assert_golden('README.md' => "no gemspecs\n") + assert_golden('foo.gemspec' => "Gem::Specification.new\n") + assert_golden( + 'a.gemspec' => "Gem::Specification.new\n", + 'b.gemspec' => "Gem::Specification.new\n" + ) + end + + test "golden: gemspec filename with a space" do + assert_golden('my gem.gemspec' => "Gem::Specification.new\n") + end + + test "golden: package.json private false publishes" do + assert_golden('package.json' => '{"name": "x", "private": false, "version": "1.0.0"}') + end + + test "golden: lerna.json content is read" do + assert_golden( + 'package.json' => '{"private": true}', + 'lerna.json' => '{"version": "1.2.3"}' + ) + end + + test "golden: .gitattributes in an unrelated subtree does not interfere" do + assert_golden( + 'shipit.production.yml' => "deploy:\n override:\n - echo ok\n", + 'vendor/.gitattributes' => "*.png binary\n", + 'vendor/thing.txt' => "x\n" + ) + end + + # --- Fallback guards --- + + test "fallback: inherit_from escaping the repository" do + repo, sha = make_repo('shipit.production.yml' => "inherit_from: ../../evil.yml\n") + error = assert_fallback(repo, sha) + assert_equal :escape, error.reason + end + + test "fallback: absolute inherit_from path" do + repo, sha = make_repo('shipit.production.yml' => "inherit_from: /etc/passwd\n") + error = assert_fallback(repo, sha) + assert_equal :escape, error.reason + end + + test "fallback: inherit_from cycle hits the depth cap" do + repo, sha = make_repo( + 'shipit.production.yml' => "inherit_from: a.yml\n", + 'a.yml' => "inherit_from: b.yml\n", + 'b.yml' => "inherit_from: a.yml\n" + ) + error = assert_fallback(repo, sha) + assert_equal :inherit_depth, error.reason + end + + test "fallback: symlinked config file" do + repo, sha = make_repo('real.yml' => "deploy:\n override:\n - echo x\n") do |dir| + File.symlink('real.yml', dir.join('shipit.production.yml')) + end + error = assert_fallback(repo, sha) + assert_equal :symlink, error.reason + end + + test "fallback: symlinked intermediate directory" do + repo, sha = make_repo( + 'shipit.production.yml' => "machine:\n directory: linkdir\n", + 'realdir/Gemfile' => "source 'https://rubygems.org'\n" + ) do |dir| + File.symlink('realdir', dir.join('linkdir')) + end + error = assert_fallback(repo, sha) + assert_equal :symlink, error.reason + end + + test "fallback: submodule gitlink on an accessed path" do + repo, base_sha = make_repo('shipit.production.yml' => "machine:\n directory: sub\n") + # Record a gitlink entry (mode 160000) without needing a real submodule. + git(repo, 'update-index', '--add', '--cacheinfo', "160000,#{base_sha},sub") + git(repo, 'commit', '-qm', 'add gitlink') + sha = git_out(repo, 'rev-parse', 'HEAD') + error = assert_fallback(repo, sha) + assert_equal :submodule, error.reason + end + + test "fallback: regular file as intermediate path component" do + repo, sha = make_repo( + 'shipit.production.yml' => "machine:\n directory: Gemfile/sub\n", + 'Gemfile' => "source 'https://rubygems.org'\n" + ) + error = assert_fallback(repo, sha) + assert_equal :file_in_path, error.reason + end + + test "fallback: .gitattributes at the repository root" do + repo, sha = make_repo( + 'shipit.production.yml' => "deploy:\n override:\n - echo x\n", + '.gitattributes' => "* text=auto\n" + ) + error = assert_fallback(repo, sha) + assert_equal :gitattributes, error.reason + end + + test "fallback: .gitattributes inside machine.directory" do + repo, sha = make_repo( + 'shipit.production.yml' => "machine:\n directory: app\n", + 'app/.gitattributes' => "*.rb eol=lf\n", + 'app/Gemfile' => "source 'https://rubygems.org'\n" + ) + error = assert_fallback(repo, sha) + assert_equal :gitattributes, error.reason + end + + test "fallback: glob listing under a symlinked directory" do + repo, sha = make_repo('realdir/x.gemspec' => "Gem::Specification.new\n") do |dir| + File.symlink('realdir', dir.join('linkdir')) + end + commands = commands_for(repo) + + Dir.mktmpdir do |dir| + fs = GitObjectFileSystem.new(dir, @stack, commands:, sha:) + error = assert_raises(GitObjectFileSystem::FallbackRequired) do + fs.file('linkdir/*.gemspec', root: true) + end + assert_equal :symlink, error.reason + end + end + + test "fallback: repository-level checkout conversion config" do + repo, _sha = make_repo('shipit.production.yml' => "deploy:\n override:\n - echo x\n") + git(repo, 'config', 'core.autocrlf', 'true') + commands = commands_for(repo) + + error = assert_raises(GitObjectFileSystem::FallbackRequired) do + commands.send(:ensure_no_checkout_conversion_config!) + end + assert_equal :git_config, error.reason + end + + test "checkout conversion config guard tolerates safe autocrlf values" do + repo, _sha = make_repo('shipit.production.yml' => "deploy:\n override:\n - echo x\n") + git(repo, 'config', 'core.autocrlf', 'input') + commands = commands_for(repo) + + assert_nothing_raised do + commands.send(:ensure_no_checkout_conversion_config!) + end + end + + # --- Idempotency --- + + test "repeat access to the same file reads the object database once" do + repo, sha = make_repo('package.json' => '{"private": true, "version": "1.0.0"}') + commands = commands_for(repo) + reads = Hash.new(0) + original = commands.method(:git_read_object) + commands.define_singleton_method(:git_read_object) do |commit_sha, path| + reads[path] += 1 + original.call(commit_sha, path) + end + + Dir.mktmpdir do |dir| + fs = GitObjectFileSystem.new(dir, @stack, commands:, sha:) + fs.file('package.json').read + fs.file('package.json').exist? + fs.file('package.json').read + end + + assert_equal 1, reads['package.json'] + end + + private + + def assert_golden(files) + repo, sha = make_repo(files) + expected, expected_root = checkout_config(repo, sha) + actual, actual_root = git_object_config(repo, sha) + assert_equal normalize(expected, expected_root), normalize(actual, actual_root) + end + + # Specs embed their evaluation directory in some values (e.g. + # release-gem /x.gemspec); normalize both roots out, mirroring + # what shadow mode does in production. + def normalize(value, root) + case value + when String then value.gsub(root, '$ROOT') + when Hash then value.transform_values { |nested| normalize(nested, root) } + when Array then value.map { |nested| normalize(nested, root) } + else value + end + end + + def assert_fallback(repo, sha) + assert_raises(GitObjectFileSystem::FallbackRequired) do + git_object_config(repo, sha) + end + end + + def make_repo(files) + dir = Pathname(Dir.mktmpdir) + @tmpdirs << dir + git(dir, 'init', '-q', '-b', 'main') + git(dir, 'config', 'user.email', 'test@example.com') + git(dir, 'config', 'user.name', 'Test') + git(dir, 'config', 'commit.gpgsign', 'false') + files.each do |path, content| + full = dir.join(path) + full.dirname.mkpath + File.write(full, content) + end + yield dir if block_given? + git(dir, 'add', '-A') + git(dir, 'commit', '-qm', 'test commit') + [dir, git_out(dir, 'rev-parse', 'HEAD')] + end + + def git(dir, *args) + _, error, status = Open3.capture3('git', *args, chdir: dir.to_s) + raise "git #{args.join(' ')} failed: #{error}" unless status.success? + end + + def git_out(dir, *args) + output, error, status = Open3.capture3('git', *args, chdir: dir.to_s) + raise "git #{args.join(' ')} failed: #{error}" unless status.success? + + output.strip + end + + def checkout_config(repo, sha) + Dir.mktmpdir do |dir| + workdir = File.join(dir, 'wc') + git(Pathname(dir), 'clone', '-q', repo.to_s, workdir) + git(Pathname(workdir), 'checkout', '-q', sha) + [FileSystem.new(workdir, @stack).cacheable.config, workdir] + end + end + + def git_object_config(repo, sha) + commands = commands_for(repo) + Dir.mktmpdir do |dir| + [GitObjectFileSystem.new(dir, @stack, commands:, sha:).cacheable.config, dir] + end + end + + def commands_for(repo) + stack = @stack + stack.stubs(:git_path).returns(repo) + StackCommands.new(stack) + end + end + end +end diff --git a/test/unit/cacheable_deploy_spec_test.rb b/test/unit/cacheable_deploy_spec_test.rb new file mode 100644 index 000000000..cf480b0f4 --- /dev/null +++ b/test/unit/cacheable_deploy_spec_test.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +require 'test_helper' + +module Shipit + class CacheableDeploySpecTest < ActiveSupport::TestCase + setup do + @stack = shipit_stacks(:shipit) + @commands = StackCommands.new(@stack) + @commit = @stack.commits.last + @old_spec = DeploySpec.new('deploy' => { 'override' => ['echo old'] }) + @new_spec = DeploySpec.new('deploy' => { 'override' => ['echo new'] }) + @events = [] + @subscriber = ActiveSupport::Notifications.subscribe('checkout_less_deploy_spec.shipit') do |*, payload| + @events << payload + end + end + + teardown do + ActiveSupport::Notifications.unsubscribe(@subscriber) + Shipit.checkout_less_deploy_spec = :disabled + end + + test "the flag rejects unknown modes" do + assert_raises(ArgumentError) { Shipit.checkout_less_deploy_spec = :bogus } + assert_equal :disabled, Shipit.checkout_less_deploy_spec + end + + test "disabled mode uses the checkout path with recursive: false" do + Shipit.checkout_less_deploy_spec = :disabled + @commands.expects(:with_temporary_working_directory) + .with(commit: @commit, recursive: false).returns([@old_spec, '/tmp/old']) + @commands.expects(:git_object_cacheable_deploy_spec).never + + assert_equal @old_spec, @commands.cacheable_deploy_spec(commit: @commit) + end + + test "a nil commit always uses the checkout path" do + Shipit.checkout_less_deploy_spec = :enabled + @commands.expects(:checkout_cacheable_deploy_spec).with(nil).returns([@old_spec, '/tmp/old']) + @commands.expects(:git_object_cacheable_deploy_spec).never + + assert_equal @old_spec, @commands.cacheable_deploy_spec(commit: nil) + end + + test "enabled mode serves the git object path and reports a hit" do + Shipit.checkout_less_deploy_spec = :enabled + @commands.expects(:git_object_cacheable_deploy_spec).with(@commit).returns([@new_spec, '/tmp/new']) + @commands.expects(:checkout_cacheable_deploy_spec).never + + assert_equal @new_spec, @commands.cacheable_deploy_spec(commit: @commit) + assert_equal([:hit], @events.map { |e| e[:event] }) + end + + test "enabled mode falls back on FallbackRequired" do + Shipit.checkout_less_deploy_spec = :enabled + error = DeploySpec::GitObjectFileSystem::FallbackRequired.new(:symlink, 'some/path') + @commands.expects(:git_object_cacheable_deploy_spec).raises(error) + @commands.expects(:checkout_cacheable_deploy_spec).with(@commit).returns([@old_spec, '/tmp/old']) + + assert_equal @old_spec, @commands.cacheable_deploy_spec(commit: @commit) + assert_equal([{ event: :fallback, reason: :symlink }], + @events.map { |e| e.slice(:event, :reason) }) + end + + test "enabled mode falls back on git command failure" do + Shipit.checkout_less_deploy_spec = :enabled + @commands.expects(:git_object_cacheable_deploy_spec).raises(Command::Failed.new('boom', 128)) + @commands.expects(:checkout_cacheable_deploy_spec).with(@commit).returns([@old_spec, '/tmp/old']) + + assert_equal @old_spec, @commands.cacheable_deploy_spec(commit: @commit) + assert_equal :git_error, @events.first[:reason] + end + + test "enabled mode falls back on unexpected errors" do + Shipit.checkout_less_deploy_spec = :enabled + @commands.expects(:git_object_cacheable_deploy_spec).raises(RuntimeError.new('surprise')) + @commands.expects(:checkout_cacheable_deploy_spec).with(@commit).returns([@old_spec, '/tmp/old']) + + assert_equal @old_spec, @commands.cacheable_deploy_spec(commit: @commit) + assert_equal :unexpected_error, @events.first[:reason] + end + + test "shadow mode returns the checkout result and reports a hit on match" do + Shipit.checkout_less_deploy_spec = :shadow + same = DeploySpec.new(@old_spec.config) + @commands.expects(:checkout_cacheable_deploy_spec).with(@commit).returns([@old_spec, '/tmp/old']) + @commands.expects(:git_object_cacheable_deploy_spec).with(@commit).returns([same, '/tmp/new']) + + assert_equal @old_spec, @commands.cacheable_deploy_spec(commit: @commit) + assert_equal([:hit], @events.map { |e| e[:event] }) + end + + test "shadow mode returns the checkout result and reports a mismatch on divergence" do + Shipit.checkout_less_deploy_spec = :shadow + @commands.expects(:checkout_cacheable_deploy_spec).with(@commit).returns([@old_spec, '/tmp/old']) + @commands.expects(:git_object_cacheable_deploy_spec).with(@commit).returns([@new_spec, '/tmp/new']) + + assert_equal @old_spec, @commands.cacheable_deploy_spec(commit: @commit) + assert_equal([:shadow_mismatch], @events.map { |e| e[:event] }) + assert_includes @events.first[:detail], 'deploy' + end + + test "shadow mode preserves the fallback reason taxonomy" do + Shipit.checkout_less_deploy_spec = :shadow + error = DeploySpec::GitObjectFileSystem::FallbackRequired.new(:gitattributes, 'app') + @commands.expects(:checkout_cacheable_deploy_spec).with(@commit).returns([@old_spec, '/tmp/old']) + @commands.expects(:git_object_cacheable_deploy_spec).raises(error) + + assert_equal @old_spec, @commands.cacheable_deploy_spec(commit: @commit) + assert_equal :gitattributes, @events.first[:reason] + assert_equal 'app', @events.first[:detail] + end + + test "events carry the active mode" do + Shipit.checkout_less_deploy_spec = :enabled + @commands.expects(:git_object_cacheable_deploy_spec).with(@commit).returns([@new_spec, '/tmp/new']) + + @commands.cacheable_deploy_spec(commit: @commit) + assert_equal :enabled, @events.first[:mode] + end + + test "shadow mode never propagates new-path exceptions" do + Shipit.checkout_less_deploy_spec = :shadow + @commands.expects(:checkout_cacheable_deploy_spec).with(@commit).returns([@old_spec, '/tmp/old']) + @commands.expects(:git_object_cacheable_deploy_spec).raises(RuntimeError.new('boom')) + + assert_equal @old_spec, @commands.cacheable_deploy_spec(commit: @commit) + assert_equal :shadow_error, @events.first[:reason] + end + + test "shadow mode propagates old-path exceptions unchanged" do + Shipit.checkout_less_deploy_spec = :shadow + @commands.expects(:checkout_cacheable_deploy_spec).raises(RuntimeError.new('old path broke')) + + assert_raises(RuntimeError) { @commands.cacheable_deploy_spec(commit: @commit) } + end + + test "build_cacheable_deploy_spec delegates to the wrapper with a nil commit" do + @commands.expects(:cacheable_deploy_spec).with(commit: nil).returns(@old_spec) + assert_equal @old_spec, @commands.build_cacheable_deploy_spec + end + end +end