Clearer terminology 'evaluate' -> 'run' (#116)

* Clearer terminology 'evaluate' -> 'run'

* Bump version

Project.toml

@@ -1,6 +1,6 @@
 name = "ReTestItems"
 uuid = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
-version = "1.19.0"
+version = "1.20.0"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

src/ReTestItems.jl

@@ -142,16 +142,16 @@ will be run.
 - `nworker_threads::Union{String,Int}`: The number of threads to use for each worker process. Defaults to 2.
   Can also be set using the `RETESTITEMS_NWORKER_THREADS` environment variable. Interactive threads are
   supported through a string (e.g. "auto,2").
-- `worker_init_expr::Expr`: an expression that will be evaluated on each worker process before any tests are run.
+- `worker_init_expr::Expr`: an expression that will be run on each worker process before any tests are run.
   Can be used to load packages or set up the environment. Must be a `:block` expression.
-- `test_end_expr::Expr`: an expression that will be evaluated after each testitem is run.
+- `test_end_expr::Expr`: an expression that will be run after each testitem is run.
   Can be used to verify that global state is unchanged after running a test. Must be a `:block` expression.
 - `memory_threshold::Real`: Sets the fraction of memory that can be in use before a worker processes are
   restarted to free memory. Defaults to $(DEFAULT_MEMORY_THRESHOLD[]). Only supported with `nworkers > 0`.
   For example, if set to 0.8, then when >80% of the available memory is in use, a worker process will be killed and
-  replaced with a new worker before the next testitem is evaluated. The testitem will then be run on the new worker
+  replaced with a new worker before the next testitem is run. The testitem will then be run on the new worker
   process, regardless of if memory pressure dropped below the threshold. If the memory pressure remains above the
-  threshold, then a worker process will again be replaced before the next testitem is evaluated.
+  threshold, then a worker process will again be replaced before the next testitem is run.
   Can also be set using the `RETESTITEMS_MEMORY_THRESHOLD` environment variable.
   **Note**: the `memory_threshold` keyword is experimental and may be removed in future versions.
 - `report::Bool=false`: If `true`, write a JUnit-format XML file summarising the test results.
@@ -357,7 +357,7 @@ function _runtests_in_current_env(
                 end
             end
             # Now all workers are started, we can begin processing test items.
-            @info "Starting evaluating test items"
+            @info "Starting running test items"
             starting = get_starting_testitems(testitems, nworkers)
             @sync for (i, w) in enumerate(workers)
                 ti = starting[i]
@@ -380,7 +380,7 @@ function _runtests_in_current_env(
     return nothing
 end
 
-# Start a new `Worker` with `nworker_threads` threads and evaluate `worker_init_expr` on it.
+# Start a new `Worker` with `nworker_threads` threads and run `worker_init_expr` on it.
 # The provided `worker_num` is only for logging purposes, and not persisted as part of the worker.
 function start_worker(proj_name, nworker_threads, worker_init_expr, ntestitems; worker_num=nothing)
     w = Worker(; threads="$nworker_threads")
@@ -439,13 +439,13 @@ function record_timeout!(testitem, run_number::Int, timeout_limit::Real)
             string(mins, "m", lpad(secs, 2, "0"), "s")
         end
     end
-    msg = "Timed out after $time_str evaluating test item $(repr(testitem.name)) (run=$run_number)"
+    msg = "Timed out after $time_str running test item $(repr(testitem.name)) (run=$run_number)"
     record_test_error!(testitem, msg, timeout_limit)
 end
 
 function record_worker_terminated!(testitem, worker::Worker, run_number::Int)
     termsignal = worker.process.termsignal
-    msg = "Worker process aborted (signal=$termsignal) evaluating test item $(repr(testitem.name)) (run=$run_number)"
+    msg = "Worker process aborted (signal=$termsignal) running test item $(repr(testitem.name)) (run=$run_number)"
     record_test_error!(testitem, msg)
 end
 
@@ -537,11 +537,11 @@ function manage_worker(
                 @debugv 1 "Test item $(repr(testitem.name)) timed out. Terminating worker $worker"
                 terminate!(worker)
                 wait(worker)
-                @error "$worker timed out evaluating test item $(repr(testitem.name)) after $timeout seconds. \
+                @error "$worker timed out running test item $(repr(testitem.name)) after $timeout seconds. \
                     Recording test error."
                 record_timeout!(testitem, run_number, timeout)
             elseif e isa WorkerTerminatedException
-                @error "$worker died evaluating test item $(repr(testitem.name)). \
+                @error "$worker died running test item $(repr(testitem.name)). \
                     Recording test error."
                 record_worker_terminated!(testitem, worker, run_number)
             else
@@ -924,13 +924,13 @@ function runtestitem(
         # disabled for now since there were issues when tests tried serialize/deserialize
         # with things defined in an anonymous module
         # environment = Module()
-        @debugv 1 "Evaluating test item $(repr(name))$(_on_worker())."
+        @debugv 1 "Running test item $(repr(name))$(_on_worker())."
         _, stats = @timed_with_compilation _redirect_logs(logs == :eager ? DEFAULT_STDOUT[] : logpath(ti)) do
             with_source_path(() -> Core.eval(Main, mod_expr), ti.file)
             with_source_path(() -> Core.eval(Main, test_end_mod_expr), ti.file)
             nothing # return nothing as the first return value of @timed_with_compilation
         end
-        @debugv 1 "Done evaluating test item $(repr(name))$(_on_worker())."
+        @debugv 1 "Done running test item $(repr(name))$(_on_worker())."
     catch err
         err isa InterruptException && rethrow()
         # Handle exceptions thrown outside a `@test` in the body of the @testitem:

src/macros.jl

@@ -7,7 +7,7 @@ gettls(k, d) = get(task_local_storage(), k, d)
 """
     TestSetup(name, code)
 
-A module that a `TestItem` can require to be evaluated before that `TestItem` is run.
+A module that a `TestItem` can require to be run before that `TestItem` is run.
 Used for declaring code that multiple `TestItem`s rely on.
 Should only be created via the `@testsetup` macro.
 """
@@ -32,7 +32,7 @@ A module only used for tests, and which `@testitem`s can depend on.
 
 Useful for setup logic that is used across multiple test items.
 The setup will run once, before any `@testitem` that requires it is executed.
-If running with multiple processes, each test-setup with be evaluated once on each process.
+If running with multiple processes, each test-setup with be run once on each process.
 
 Each test-setup module will live for the lifetime of the tests.
 Mutable state should be avoided, since the order in which test items run is non-deterministic,
@@ -125,9 +125,9 @@ struct TestItem
     code::Any
     testsetups::Vector{TestSetup} # populated by runtests coordinator
     workerid::Base.RefValue{Int} # populated when the test item is scheduled
-    testsets::Vector{DefaultTestSet} # populated when the test item is finished evaluating
-    eval_number::Base.RefValue{Int} # to keep track of how many items have been evaluated so far
-    stats::Vector{PerfStats} # populated when the test item is finished evaluating
+    testsets::Vector{DefaultTestSet} # populated when the test item is finished running
+    eval_number::Base.RefValue{Int} # to keep track of how many items have been run so far
+    stats::Vector{PerfStats} # populated when the test item is finished running
     scheduled_for_evaluation::ScheduledForEvaluation # to keep track of whether the test item has been scheduled for evaluation
 end
 function TestItem(number, name, id, tags, default_imports, setups, retries, file, line, project_root, code)
@@ -182,7 +182,7 @@ A `@testitem` is wrapped into a module when run, so must import any additional p
         end
     end
 
-The test item's code is evaluated as top-level code in a new module, so it can include imports, define new structs or helper functions, and declare tests and testsets.
+The test item's code is run as top-level code in a new module, so it can include imports, define new structs or helper functions, and declare tests and testsets.
 
     @testitem "DoCoolStuff" begin
         function do_really_cool_stuff()

test/integrationtests.jl

@@ -573,7 +573,7 @@ end
     """,
     replace(c.output, r" on worker \d+" => ""))
 
-    # Since the test setup never succeeds it will be evaluated mutliple times. Here we test
+    # Since the test setup never succeeds it will be run mutliple times. Here we test
     # that we don't accumulate logs from all previous failed attempts (which would get
     # really spammy if the test setup is used by 100 test items).
     @test !occursin("""
@@ -610,7 +610,7 @@ end
     # Test the error is as expected
     err = only(non_passes(results))
     @test err.test_type == :nontest_error
-    @test err.value == string(ErrorException("Worker process aborted (signal=6) evaluating test item \"Abort\" (run=1)"))
+    @test err.value == string(ErrorException("Worker process aborted (signal=6) running test item \"Abort\" (run=1)"))
 end
 
 @testset "test retrying failing testitem" begin
@@ -665,7 +665,7 @@ end
     # Test the error is as expected
     err = only(non_passes(results))
     @test err.test_type == :nontest_error
-    @test err.value == string(ErrorException("Timed out after 4s evaluating test item \"Test item takes 60 seconds\" (run=1)"))
+    @test err.value == string(ErrorException("Timed out after 4s running test item \"Test item takes 60 seconds\" (run=1)"))
 end
 
 @testset "testitem timeout set via env variable" begin
@@ -682,7 +682,7 @@ end
     # Test the error is as expected
     err = only(non_passes(results))
     @test err.test_type == :nontest_error
-    @test err.value == string(ErrorException("Timed out after 4s evaluating test item \"Test item takes 60 seconds\" (run=1)"))
+    @test err.value == string(ErrorException("Timed out after 4s running test item \"Test item takes 60 seconds\" (run=1)"))
 end
 
 @testset "Error outside `@testitem`" begin

test/references/retry_tests_report_1worker.xml

@@ -244,7 +244,7 @@ No Captured Logs for test item "Has retries=1 and always fails" at test/testfile
 		<error message="Error during test at test/testfiles/_retry_tests.jl:54">Error in testset "Timeout always" on worker 72554:
 Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:54
   Got exception outside of a @test
-  Timed out after 5s evaluating test item "Timeout always" (run=1)
+  Timed out after 5s running test item "Timeout always" (run=1)
 Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:54 on worker 72554
 
 signal (15): Terminated: 15
@@ -269,7 +269,7 @@ Allocations: 5231943 (Pool: 5229611; Big: 2332); GC: 30
 		<error message="Error during test at test/testfiles/_retry_tests.jl:54">Error in testset "Timeout always" on worker 72554:
 Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:54
   Got exception outside of a @test
-  Timed out after 5s evaluating test item "Timeout always" (run=2)
+  Timed out after 5s running test item "Timeout always" (run=2)
 Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:54 on worker 72554
 
 signal (15): Terminated: 15
@@ -294,7 +294,7 @@ Allocations: 3068888 (Pool: 3067775; Big: 1113); GC: 3
 		<error message="Error during test at test/testfiles/_retry_tests.jl:54">Error in testset "Timeout always" on worker 72554:
 Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:54
   Got exception outside of a @test
-  Timed out after 5s evaluating test item "Timeout always" (run=3)
+  Timed out after 5s running test item "Timeout always" (run=3)
 Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:54 on worker 72554
 
 signal (15): Terminated: 15
@@ -319,7 +319,7 @@ Allocations: 3068845 (Pool: 3067734; Big: 1111); GC: 3
 		<error message="Error during test at test/testfiles/_retry_tests.jl:64">Error in testset "Timeout first, pass after" on worker 72859:
 Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:64
   Got exception outside of a @test
-  Timed out after 5s evaluating test item "Timeout first, pass after" (run=1)
+  Timed out after 5s running test item "Timeout first, pass after" (run=1)
 Captured Logs for test item "Timeout first, pass after" at test/testfiles/_retry_tests.jl:64 on worker 72859
 
 signal (15): Terminated: 15