cli: refactor which and exec to optionally use an env cache

Moreover, we load as little files as possible to reduce the amount
of time loading to a minimum.

.gitignore

@@ -8,3 +8,4 @@ pkg/
 /coverage/
 /.yardoc/
 test/gem_home
+tmp/

autoproj.gemspec

@@ -33,5 +33,6 @@ Gem::Specification.new do |s|
     s.add_development_dependency "flexmock", '~> 2.0', ">= 2.0.0"
     s.add_development_dependency "minitest", "~> 5.0", ">= 5.0"
     s.add_development_dependency "simplecov"
+    s.add_development_dependency "aruba"
 end
 

lib/autoproj.rb

@@ -35,9 +35,12 @@
 require 'autoproj/source_package_query'
 require 'autoproj/os_package_query'
 
+require 'autoproj/ops/install'
 require 'autoproj/ops/tools'
 require 'autoproj/ops/loader'
 require 'autoproj/ops/configuration'
+require 'autoproj/ops/cached_env'
+require 'autoproj/ops/which'
 
 require 'autoproj/workspace'
 

lib/autoproj/aruba_minitest.rb

@@ -0,0 +1,66 @@
+require 'aruba/api'
+
+module Autoproj
+    # Minitest-usable Aruba wrapper
+    #
+    # Aruba 0.14 is incompatible with Minitest because of their definition
+    # of the #run method This change hacks around the problem, by moving
+    # the Aruba API to a side stub object.
+    #
+    # The run methods are renamed as they have been renamed in Aruba 1.0
+    # alpha, run -> run_command and run_simple -> run_command_and_stop
+    module ArubaMinitest
+        class API
+            include ::Aruba::Api
+        end
+
+        def setup
+            super
+            @aruba_api = API.new
+            @aruba_api.setup_aruba
+        end
+
+        def teardown
+            stop_all_commands
+            super
+        end
+
+        def run_command_and_stop(*args, fail_on_error: true)
+            cmd = run_command(*args)
+            cmd.stop
+            if fail_on_error
+                assert_command_finished_successfully(cmd)
+            end
+            cmd
+        end
+
+        def run_command(*args)
+            @aruba_api.run(*args)
+        end
+
+        def chmod(*args) # also defined by Rake
+            @aruba_api.chmod(*args)
+        end
+
+        def method_missing(m, *args, &block)
+            if @aruba_api.respond_to?(m)
+                return @aruba_api.send(m, *args, &block)
+            else
+                super
+            end
+        end
+
+        def assert_command_stops(cmd, fail_on_error: true)
+            cmd.stop
+            if fail_on_error
+                assert_command_finished_successfully(cmd)
+            end
+        end
+
+        def assert_command_finished_successfully(cmd)
+            refute cmd.timed_out?, "#{cmd} timed out on stop"
+            assert_equal 0, cmd.exit_status, "#{cmd} finished with a non-zero exit status (#{cmd.exit_status})\n-- STDOUT\n#{cmd.stdout}\n-- STDERR\n#{cmd.stderr}"
+        end
+    end
+end
+

lib/autoproj/cli/exec.rb

@@ -1,21 +1,55 @@
-require 'autoproj/cli/inspection_tool'
+require 'autoproj/find_workspace'
+require 'autoproj/ops/cached_env'
+require 'autoproj/ops/which'
+
 module Autoproj
     module CLI
-        class Exec < InspectionTool
-            def run(cmd, *args)
-                initialize_and_load
-                finalize_setup(Array.new)
+        class Exec
+            def initialize
+                @root_dir = Autoproj.find_workspace_dir
+                if !@root_dir
+                    require 'autoproj/workspace'
+                    # Will do all sorts of error reporting,
+                    # or may be able to resolve
+                    @root_dir = Workspace.default.root_dir
+                end
+            end
+
+            def load_cached_env
+                env = Ops.load_cached_env(@root_dir)
+                return if !env
+
+                Autobuild::Environment.
+                    environment_from_export(env, ENV)
+            end
+
+            def run(cmd, *args, use_cached_env: false)
+                if use_cached_env
+                    env = load_cached_env
+                end
+
+                if !env
+                    require 'autoproj'
+                    require 'autoproj/cli/inspection_tool'
+                    ws = Workspace.from_dir(@root_dir)
+                    loader = InspectionTool.new(ws)
+                    loader.initialize_and_load
+                    loader.finalize_setup(Array.new)
+                    env = ws.full_env.resolved_env
+                end
 
+                path = env['PATH'].split(File::PATH_SEPARATOR)
                 program = 
-                    begin ws.which(cmd)
+                    begin Ops.which(cmd, path_entries: path)
                     rescue ::Exception => e
+                        require 'autoproj'
                         raise CLIInvalidArguments, e.message, e.backtrace
                     end
-                env = ws.full_env.resolved_env
 
                 begin
                     ::Process.exec(env, program, *args)
                 rescue ::Exception => e
+                    require 'autoproj'
                     raise CLIInvalidArguments, e.message, e.backtrace
                 end
             end

lib/autoproj/cli/main.rb

@@ -2,6 +2,7 @@
 require 'tty/color'
 require 'autoproj/cli/main_test'
 require 'autoproj/cli/main_plugin'
+require 'autoproj/reporter'
 
 module Autoproj
     module CLI
@@ -453,15 +454,23 @@ def manifest(*name)
             end
 
             desc 'exec', "runs a command, applying the workspace's environment first"
+            option :use_cache, type: :boolean, desc: 'use the cached environment instead of loading "\
+                " the whole configuration'
             def exec(*args)
                 require 'autoproj/cli/exec'
-                CLI::Exec.new.run(*args)
+                Autoproj.report(on_package_failures: default_report_on_package_failures, debug: options[:debug], silent: true) do
+                    CLI::Exec.new.run(*args, use_cached_env: options[:use_cache])
+                end
             end
 
             desc 'which', "resolves the full path to a command within the Autoproj workspace"
+            option :use_cache, type: :boolean, desc: 'use the cached environment instead of loading "\
+                " the whole configuration'
             def which(cmd)
                 require 'autoproj/cli/which'
-                CLI::Which.new.run(cmd)
+                Autoproj.report(on_package_failures: default_report_on_package_failures, debug: options[:debug], silent: true) do
+                    CLI::Which.new.run(cmd, use_cached_env: options[:use_cache])
+                end
             end
         end
     end

lib/autoproj/cli/which.rb

@@ -1,14 +1,51 @@
-require 'autoproj/cli/inspection_tool'
+require 'autoproj'
+require 'autoproj/ops/cached_env'
+require 'autoproj/ops/which'
+
 module Autoproj
     module CLI
-        class Which < InspectionTool
-            def run(cmd)
-                initialize_and_load
-                finalize_setup(Array.new)
+        class Which
+            def initialize
+                @root_dir = Autoproj.find_workspace_dir
+                if !@root_dir
+                    require 'autoproj/workspace'
+                    # Will do all sorts of error reporting,
+                    # or may be able to resolve
+                    @root_dir = Workspace.default.root_dir
+                end
+            end
+
+            def load_cached_env
+                env = Ops.load_cached_env(@root_dir)
+                return if !env
+
+                Autobuild::Environment.
+                    environment_from_export(env, ENV)
+            end
+
+            def run(cmd, use_cached_env: false)
+                if use_cached_env
+                    env = load_cached_env
+                end
+
+                if !env
+                    require 'autoproj'
+                    require 'autoproj/cli/inspection_tool'
+                    ws = Workspace.from_dir(@root_dir)
+                    loader = InspectionTool.new(ws)
+                    loader.initialize_and_load
+                    loader.finalize_setup(Array.new)
+                    env = ws.full_env.resolved_env
+                end
 
-                puts ws.which(cmd)
-            rescue Workspace::ExecutableNotFound => e
+                path = env['PATH'].split(File::PATH_SEPARATOR)
+                puts Ops.which(cmd, path_entries: path)
+            rescue ExecutableNotFound => e
+                require 'autoproj' # make sure everything is available for error reporting
                 raise CLIInvalidArguments, e.message, e.backtrace
+            rescue Exception
+                require 'autoproj' # make sure everything is available for error reporting
+                raise
             end
         end
     end

lib/autoproj/exceptions.rb

@@ -1,3 +1,5 @@
+require 'autobuild/exceptions'
+
 module Autoproj
     class ConfigError < RuntimeError
         attr_accessor :file
@@ -84,6 +86,9 @@ class OutdatedWorkspace < InvalidWorkspace; end
     # Exception raised when initializing on a workspace that is not the current
     # one
     class MismatchingWorkspace < InvalidWorkspace; end
+
+    # Raised by 'which' when an executable cannot be found
+    class ExecutableNotFound < ArgumentError; end
 end
 
 

lib/autoproj/ops/cached_env.rb

@@ -0,0 +1,36 @@
+require 'autobuild/environment'
+
+module Autoproj
+    module Ops
+        def self.cached_env_path(root_dir)
+            File.join(root_dir, '.autoproj', 'env.yml')
+        end
+
+        def self.load_cached_env(root_dir)
+            path = cached_env_path(root_dir)
+            if File.file?(path)
+                env = YAML.load(File.read(path))
+                Autobuild::Environment::ExportedEnvironment.new(
+                    env['set'], env['unset'], env['update'])
+            end
+        end
+
+        def self.save_cached_env(root_dir, env)
+            env = env.exported_environment
+            path = cached_env_path(root_dir)
+            existing =
+                begin
+                    YAML.load(File.read(path))
+                rescue Exception
+                end
+
+            env = Hash['set' => env.set, 'unset' => env.unset, 'update' => env.update]
+            if env != existing
+                Ops.atomic_write(path) do |io|
+                    YAML.dump(env, io)
+                end
+                true
+            end
+        end
+    end
+end

lib/autoproj/ops/which.rb

@@ -0,0 +1,45 @@
+require 'pathname'
+require 'autoproj/exceptions'
+require 'autobuild/environment'
+
+module Autoproj
+    module Ops
+        # Find the given executable file in PATH
+        #
+        # If `cmd` is an absolute path, it will either return it or raise if
+        # `cmd` is not executable. Otherwise, looks for an executable named
+        # `cmd` in PATH and returns it, or raises if it cannot be found. The
+        # exception contains a more detailed reason for failure
+        # 
+        #
+        # @param [String] cmd
+        # @return [String] the resolved program
+        # @raise [ExecutableNotFound] if an executable file named `cmd` cannot
+        #   be found
+        def self.which(cmd, path_entries: nil)
+            path = Pathname.new(cmd)
+            if path.absolute?
+                if path.file? && path.executable?
+                    return cmd
+                elsif path.exist?
+                    raise ExecutableNotFound.new(cmd), "given command `#{cmd}` exists but is not an executable file"
+                else
+                    raise ExecutableNotFound.new(cmd), "given command `#{cmd}` does not exist, an executable file was expected"
+                end
+            else
+                if path_entries.respond_to?(:call)
+                    path_entries = path_entries.call
+                end
+                absolute = Autobuild::Environment.find_executable_in_path(cmd, path_entries)
+
+                if absolute
+                    return absolute
+                elsif file = Autobuild::Environment.find_in_path(cmd, path_entries)
+                    raise ExecutableNotFound.new(cmd), "`#{cmd}` resolves to #{file} which is not executable"
+                else
+                    raise ExecutableNotFound.new(cmd), "cannot resolve `#{cmd}` to an executable in the workspace"
+                end
+            end
+        end
+    end
+end

lib/autoproj/test.rb

@@ -271,8 +271,7 @@ def ws_create_os_package_resolver
                 os_package_manager: 'os')
         end
 
-        def ws_create
-            dir = make_tmpdir
+        def ws_create(dir = make_tmpdir)
             require 'autoproj/ops/main_config_switcher'
             FileUtils.cp_r Ops::MainConfigSwitcher::MAIN_CONFIGURATION_TEMPLATE, File.join(dir, 'autoproj')
             FileUtils.mkdir_p File.join(dir, '.autoproj')