Skip to content

Commit e5a5d71

Browse files
cmcainecmcaine
committed
Capture output per test
(For most tests)
1 parent ee1bcd3 commit e5a5d71

File tree

12 files changed

+96
-51
lines changed

12 files changed

+96
-51
lines changed

‎Manifest.toml‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# This file is machine-generated - editing it directly is not advised
22

3+
julia_version = "1.7.2"
34
manifest_format = "2.0"
45

56
[[deps.Base64]]
@@ -9,6 +10,12 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
910
deps = ["Printf"]
1011
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
1112

13+
[[deps.IOCapture]]
14+
deps = ["Logging", "Random"]
15+
git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a"
16+
uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
17+
version = "0.2.2"
18+
1219
[[deps.InteractiveUtils]]
1320
deps = ["Markdown"]
1421
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
@@ -49,11 +56,6 @@ uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
4956
[[deps.Serialization]]
5057
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
5158

52-
[[deps.Suppressor]]
53-
git-tree-sha1 = "c6ed566db2fe3931292865b966d6d140b7ef32a9"
54-
uuid = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
55-
version = "0.2.1"
56-
5759
[[deps.Test]]
5860
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
5961
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

‎Project.toml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,17 @@ authors = ["Sascha Mann <[email protected]>", "Exercism", "Contributors"]
44
version = "0.1.0"
55

66
[deps]
7+
IOCapture = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
78
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
8-
Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
99
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
1010

1111
[compat]
1212
JSON = "0.21"
13-
Suppressor = "0.2"
1413
julia = "1.7"
1514

1615
[extras]
17-
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
1816
InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
17+
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
1918

2019
[targets]
2120
test = ["Test", "InteractiveUtils"]

‎src/runner.jl‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,44 @@
1-
using Suppressor
1+
using IOCapture
22
using Test
33

4+
const test_outputs = Dict()
5+
6+
macro test(expr)
7+
f(result, output) = test_outputs[result] = (output=output, expr=expr);
8+
:(
9+
c = $IOCapture.capture() do
10+
$Test.@test $(esc(expr));
11+
end;
12+
$f(c.value, c.output);
13+
c.value
14+
)
15+
end
16+
17+
macro test_throws(extype, expr)
18+
f(result, output) = test_outputs[result] = (output=output, expr=expr);
19+
:(
20+
c = $IOCapture.capture() do
21+
$Test.@test_throws $extype $(esc(expr));
22+
end;
23+
$f(c.value, c.output);
24+
c.value
25+
)
26+
end
27+
28+
# test_logs is difficult.
29+
430
"""
531
runtests(testfile)
632
733
Wrap the testfile in a ReportingTestSet and capture all output.
834
Returns the output and ReportingTestSet.
935
"""
1036
function runtests(testfile)
11-
# The Suppressor macro wraps everything in a try...finally block, therefore rts needs to be introduced before
12-
local rts
13-
output = @capture_out rts = @testset ReportingTestSet "" begin
14-
include(testfile)
37+
c = IOCapture.capture() do
38+
@testset ReportingTestSet "" begin
39+
include(testfile)
40+
end
1541
end
1642

17-
output, rts
43+
c.output, c.value
1844
end

‎src/tojson.jl‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ const TEST_RESULT_COLLAPSE_THRESHOLD = 2
99
const MAX_REPORTED_FAILURES_PER_TESTSET = 5
1010

1111
"""
12-
tojson(output::String, ts::ReportingTestSet)
12+
tojson(general_output::String, ts::ReportingTestSet)
1313
1414
Takes user output and a ReportingTestSet and converts it to a JSON string as
1515
expected by the interface.
@@ -55,7 +55,7 @@ For more information, check the reference:
5555
https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
5656
"""
5757
# TODO: Capture output per-test
58-
function tojson(output::String, ts::ReportingTestSet)
58+
function tojson(general_output::String, ts::ReportingTestSet)
5959
if length(ts.results) == 1 && ts.results[1] isa Test.Error
6060
# There has been a syntax error or similar and no tests have run.
6161
# Otherwise ts.results[1] will be a ReportingTestSet.
@@ -82,22 +82,31 @@ function tojson(output::String, ts::ReportingTestSet)
8282
# Flag set in push_result!(), used for top-level status property
8383
any_failed = false
8484
# All stdout from the top level test set, used for all tests.
85-
output = truncate_output(output)
85+
general_output = truncate_output(general_output)
86+
87+
unescape(e) = e
88+
unescape(e::String) = startswith(e, "\$(") ? unescape(Meta.parse(e[3:end-1])) : e
89+
unescape(e::Expr) = e.head == :escape ? unescape(e.args[1]) : e
8690

8791
function test_code(result::Test.Result)
92+
if haskey(test_outputs, result)
93+
expr = test_outputs[result].expr
94+
else
95+
expr = unescape(result.orig_expr)
96+
end
8897
if result isa Test.Pass && result.test_type == :test_throws
89-
"@test_throws $(result.data) $(result.orig_expr)"
98+
"@test_throws $(result.data) $(expr)"
9099
elseif result isa Test.Fail && startswith(string(result.test_type), "test_throws")
91-
"@test_throws $(result.data) $(result.orig_expr)"
100+
"@test_throws $(result.data) $(expr)"
92101
elseif result isa Test.LogTestFailure
93-
"@test_logs $(join(result.patterns, ' ')) $(result.orig_expr)"
102+
"@test_logs $(join(result.patterns, ' ')) $(expr)"
94103
elseif result isa Test.Error && result.test_type == :test_unbroken
95-
"@test_broken $(result.orig_expr)"
104+
"@test_broken $(expr)"
96105
elseif result isa Test.Broken
97106
macro_name = result.test_type === :skipped ? "@test_skip " : "@test_broken "
98-
"$macro_name $(result.orig_expr)"
107+
"$macro_name $(expr)"
99108
else
100-
"@test $(result.orig_expr)"
109+
"@test $(expr)"
101110
end
102111
end
103112

@@ -133,6 +142,12 @@ function tojson(output::String, ts::ReportingTestSet)
133142

134143
any_failed = any_failed || status in ("fail", "error")
135144

145+
if haskey(test_outputs, result)
146+
output = truncate_output(test_outputs[result].output)
147+
else
148+
output = general_output
149+
end
150+
136151
return push!(tests, Dict(filter( ((k, v),) -> !isnothing(v), (
137152
"name" => name,
138153
"status" => status,

‎test/fixtures/all_passing_with_debugging/results.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,15 @@
44
"tests": [
55
{
66
"name": "first test",
7-
"test_code": "@test x == 1",
7+
"test_code": "@test f1(1) == 1",
88
"status": "pass",
9-
"output": "x = 1\nI'M HERE!\n"
9+
"output": "x = 1\n"
1010
},
1111
{
1212
"name": "second test",
13-
"test_code": "@test 2 == 2",
13+
"test_code": "@test f2(2) == 2",
1414
"status": "pass",
15-
"output": "x = 1\nI'M HERE!\n"
15+
"output": "I'M HERE!\n"
1616
}
1717
]
1818
}

‎test/fixtures/all_passing_with_debugging/runtests.jl‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,18 @@
11
using Test
22

3+
function f1(x)
4+
@show x # debugging
5+
end
6+
7+
function f2(x)
8+
println("I'M HERE!")
9+
x
10+
end
11+
312
@testset "first test" begin
4-
x = 1
5-
@show x # Debugging
6-
@test x == 1
13+
@test f1(1) == 1
714
end
815

916
@testset "second test" begin
10-
println("I'M HERE!")
11-
@test 2 == 2
17+
@test f2(2) == 2
1218
end

‎test/fixtures/complex-numbers/results.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33
"version": 2,
44
"tests": [
55
{
6-
"name": "ComplexNumber <: Number",
6+
"name": "$(Expr(:escape, :(ComplexNumber <: Number)))",
77
"test_code": "@test ComplexNumber <: Number",
88
"status": "pass"
99
},
1010
{
11-
"name": "ComplexNumber(0, 1) ^ 2 == ComplexNumber(-1, 0)",
11+
"name": "$(Expr(:escape, :(ComplexNumber(0, 1) ^ 2 == ComplexNumber(-1, 0))))",
1212
"test_code": "@test ComplexNumber(0, 1) ^ 2 == ComplexNumber(-1, 0)",
1313
"status": "pass"
1414
},
@@ -97,19 +97,19 @@
9797
"name": "Syntax sugar jm.5",
9898
"test_code": "@test false",
9999
"status": "fail",
100-
"message": "Test Failed at ./runtests.jl:77\n Expression: false"
100+
"message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, false))"
101101
},
102102
{
103103
"name": "Syntax sugar jm.6",
104104
"test_code": "@test false",
105105
"status": "fail",
106-
"message": "Test Failed at ./runtests.jl:77\n Expression: false"
106+
"message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, false))"
107107
},
108108
{
109109
"name": "Syntax sugar jm.7",
110110
"test_code": "@test false",
111111
"status": "fail",
112-
"message": "Test Failed at ./runtests.jl:77\n Expression: false"
112+
"message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, false))"
113113
}
114114
]
115115
}

‎test/fixtures/everything_at_once/results.json‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,19 @@
55
{
66
"name": "first test",
77
"test_code": "@test x == 1",
8-
"status": "pass",
9-
"output": "x = 1\n"
8+
"status": "pass"
109
},
1110
{
1211
"name": "second test",
1312
"test_code": "@test 1 == 2",
1413
"status": "fail",
15-
"message": "Test Failed at ./runtests.jl:10\n Expression: 1 == 2\n Evaluated: 1 == 2",
16-
"output": "x = 1\n"
14+
"message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, :(1 == 2)))"
1715
},
1816
{
1917
"name": "third test",
2018
"test_code": "@test error(\"\")",
2119
"status": "error",
22-
"message": "",
23-
"output": "x = 1\n"
20+
"message": ""
2421
}
2522
]
2623
}

‎test/fixtures/nested/results.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,19 @@
1111
"name": "outer » inner 1.2",
1212
"test_code": "@test 1 == 2",
1313
"status": "fail",
14-
"message": "Test Failed at ./runtests.jl:8\n Expression: 1 == 2\n Evaluated: 1 == 2"
14+
"message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, :(1 == 2)))"
1515
},
1616
{
1717
"name": "outer » inner 2",
1818
"test_code": "@test 3 == 4",
1919
"status": "fail",
20-
"message": "Test Failed at ./runtests.jl:12\n Expression: 3 == 4\n Evaluated: 3 == 4"
20+
"message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, :(3 == 4)))"
2121
},
2222
{
2323
"name": "outer 2 » inner 3",
2424
"test_code": "@test 5 == 6",
2525
"status": "fail",
26-
"message": "Test Failed at ./runtests.jl:20\n Expression: 5 == 6\n Evaluated: 5 == 6"
26+
"message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, :(5 == 6)))"
2727
},
2828
{
2929
"name": "outer 2",
@@ -45,7 +45,7 @@
4545
"name": "outer 3 » inner 4",
4646
"test_code": "@test f(5) == 6",
4747
"status": "fail",
48-
"message": "Test Failed at ./runtests.jl:32\n Expression: f(5) == 6\n Evaluated: 5 == 6"
48+
"message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, :(f(5) == 6)))"
4949
}
5050
]
5151
}

‎test/fixtures/one_fail/results.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"name": "second test",
1212
"test_code": "@test x == 1",
1313
"status": "fail",
14-
"message": "Test Failed at ./runtests.jl:9\n Expression: x == 1\n Evaluated: 2 == 1"
14+
"message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, :(x == 1)))"
1515
},
1616
{
1717
"name": "third test",

0 commit comments

Comments
 (0)