diff --git a/bin/run.sh b/bin/run.sh index ee56e14..23c895f 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -93,7 +93,7 @@ src_tests=$(awk "${awk_json}"' ' "${tmp_dir}/${slug}/${slug}-tests.factor") # 2. Parse Factor stdout into NDJSON segments and failures: -# segments: {"type":"segment","idx":N,"failed":bool,"output":"..."} +# segments: {"type":"segment","idx":N,"failed":bool,"name":"...","output":"..."} # failures: {"type":"failure","line_no":N,"message":"..."} parsed=$(printf '%s\n' "${raw_output}" | awk "${awk_json}"' function close_segment( out, i) { @@ -102,8 +102,8 @@ parsed=$(printf '%s\n' "${raw_output}" | awk "${awk_json}"' for (i = 1; i <= seg_n; i++) out = out (i > 1 ? "\n" : "") seg[i] sub(/^\n+/, "", out) sub(/\n+$/, "", out) - printf "{\"type\":\"segment\",\"idx\":%d,\"failed\":%s,\"output\":%s}\n", - idx, (seg_failed ? "true" : "false"), json_str(out) + printf "{\"type\":\"segment\",\"idx\":%d,\"failed\":%s,\"name\":%s,\"output\":%s}\n", + idx, (seg_failed ? "true" : "false"), json_str(cur_name), json_str(out) } function close_failure( body, i) { body = "" @@ -126,6 +126,9 @@ parsed=$(printf '%s\n' "${raw_output}" | awk "${awk_json}"' BEGIN { state = "inline" idx = 0 + pending_name = "" + cur_name = "" + desc_re = "^###DESC### " header_re = "^(Unit Test|Unit Test~|Unit Test V~|Long Unit Test|Must Fail|Must Fail With|Must Not Fail|Must Infer|Must Infer As): " } state == "inline" && $0 ~ header_re { @@ -134,6 +137,18 @@ parsed=$(printf '%s\n' "${raw_output}" | awk "${awk_json}"' seg_failed = 0 seg_n = 0 delete seg + cur_name = pending_name + pending_name = "" + next + } + # A generated "###DESC### " line labels the test that + # follows it (it is printed before that test'"'"'s "Unit Test:" header). + # Stash it as the upcoming segment'"'"'s name; never add it to any + # segment'"'"'s output. Concept exercises emit no such lines. + state == "inline" && $0 ~ desc_re { + line = $0 + sub(desc_re, "", line) + pending_name = line next } state == "inline" && $0 == "###FAIL_BEGIN###" { @@ -188,7 +203,7 @@ failures=$(printf '%s\n' "${parsed}" | awk '/"type":"failure"/' || true) # 3. If no segments emitted, surface a top-level error from the raw output. if [[ -z "${segments}" ]]; then - cleaned=$(printf '%s\n' "${raw_output}" | awk '/^\([UO]\) /{exit} {print}' \ + cleaned=$(printf '%s\n' "${raw_output}" | awk '/^\([UO]\) /{exit} !/^###DESC### /{print}' \ | awk ' NF { print @@ -229,8 +244,10 @@ jq -n \ else "fail" end else "pass" end ) as $status + | ($seg.name // "") as $label | { - name: ("Test " + ((.key + 1) | tostring)), + name: (if ($label | length) > 0 then $label + else ("Test " + ((.key + 1) | tostring)) end), status: $status, test_code: ($src.test_code // ""), } diff --git a/tests/all-fail/all-fail/all-fail-tests.factor b/tests/all-fail/all-fail/all-fail-tests.factor index 8db9da5..5176820 100644 --- a/tests/all-fail/all-fail/all-fail-tests.factor +++ b/tests/all-fail/all-fail/all-fail-tests.factor @@ -1,8 +1,10 @@ USING: all-fail exercism-tools tools.test ; IN: all-fail.tests +"greet returns hello" description { "hello" } [ greet ] unit-test STOP-HERE +"greet returns world" description { "world" } [ greet ] unit-test diff --git a/tests/all-fail/exercism-tools/exercism-tools.factor b/tests/all-fail/exercism-tools/exercism-tools.factor index 32a6f61..428c2d3 100644 --- a/tests/all-fail/exercism-tools/exercism-tools.factor +++ b/tests/all-fail/exercism-tools/exercism-tools.factor @@ -9,6 +9,12 @@ SYNTAX: STOP-HERE SYNTAX: TASK: lexer get next-line ; +! Label the test that follows with its description. The marker lets the +! wrapper strip this line from captured output and attach it to the next +! test as a name, rather than leaving it in the previous test's output. +: description ( str -- ) + "###DESC### " write print ; + ! Print one failure block in a stable, parser-friendly form. Bracketed by ! markers so a wrapper can split the stream reliably and avoid Factor's ! noisy callstack output (which is interleaved with subsequent failures). diff --git a/tests/all-fail/expected_results.json b/tests/all-fail/expected_results.json index e223f2c..07f910b 100644 --- a/tests/all-fail/expected_results.json +++ b/tests/all-fail/expected_results.json @@ -3,13 +3,13 @@ "status": "fail", "tests": [ { - "name": "Test 1", + "name": "greet returns hello", "status": "fail", "test_code": "{ \"hello\" } [ greet ] unit-test", "message": "=== Expected:\n\"hello\"\n=== Got:\n\"wrong\"" }, { - "name": "Test 2", + "name": "greet returns world", "status": "fail", "test_code": "{ \"world\" } [ greet ] unit-test", "message": "=== Expected:\n\"world\"\n=== Got:\n\"wrong\"" diff --git a/tests/concept-concurrency/exercism-tools/exercism-tools.factor b/tests/concept-concurrency/exercism-tools/exercism-tools.factor index 32a6f61..428c2d3 100644 --- a/tests/concept-concurrency/exercism-tools/exercism-tools.factor +++ b/tests/concept-concurrency/exercism-tools/exercism-tools.factor @@ -9,6 +9,12 @@ SYNTAX: STOP-HERE SYNTAX: TASK: lexer get next-line ; +! Label the test that follows with its description. The marker lets the +! wrapper strip this line from captured output and attach it to the next +! test as a name, rather than leaving it in the previous test's output. +: description ( str -- ) + "###DESC### " write print ; + ! Print one failure block in a stable, parser-friendly form. Bracketed by ! markers so a wrapper can split the stream reliably and avoid Factor's ! noisy callstack output (which is interleaved with subsequent failures). diff --git a/tests/concept-not-parsing/exercism-tools/exercism-tools.factor b/tests/concept-not-parsing/exercism-tools/exercism-tools.factor index 32a6f61..428c2d3 100644 --- a/tests/concept-not-parsing/exercism-tools/exercism-tools.factor +++ b/tests/concept-not-parsing/exercism-tools/exercism-tools.factor @@ -9,6 +9,12 @@ SYNTAX: STOP-HERE SYNTAX: TASK: lexer get next-line ; +! Label the test that follows with its description. The marker lets the +! wrapper strip this line from captured output and attach it to the next +! test as a name, rather than leaving it in the previous test's output. +: description ( str -- ) + "###DESC### " write print ; + ! Print one failure block in a stable, parser-friendly form. Bracketed by ! markers so a wrapper can split the stream reliably and avoid Factor's ! noisy callstack output (which is interleaved with subsequent failures). diff --git a/tests/concept-partial-fail/exercism-tools/exercism-tools.factor b/tests/concept-partial-fail/exercism-tools/exercism-tools.factor index 32a6f61..428c2d3 100644 --- a/tests/concept-partial-fail/exercism-tools/exercism-tools.factor +++ b/tests/concept-partial-fail/exercism-tools/exercism-tools.factor @@ -9,6 +9,12 @@ SYNTAX: STOP-HERE SYNTAX: TASK: lexer get next-line ; +! Label the test that follows with its description. The marker lets the +! wrapper strip this line from captured output and attach it to the next +! test as a name, rather than leaving it in the previous test's output. +: description ( str -- ) + "###DESC### " write print ; + ! Print one failure block in a stable, parser-friendly form. Bracketed by ! markers so a wrapper can split the stream reliably and avoid Factor's ! noisy callstack output (which is interleaved with subsequent failures). diff --git a/tests/concept-stub/exercism-tools/exercism-tools.factor b/tests/concept-stub/exercism-tools/exercism-tools.factor index 32a6f61..428c2d3 100644 --- a/tests/concept-stub/exercism-tools/exercism-tools.factor +++ b/tests/concept-stub/exercism-tools/exercism-tools.factor @@ -9,6 +9,12 @@ SYNTAX: STOP-HERE SYNTAX: TASK: lexer get next-line ; +! Label the test that follows with its description. The marker lets the +! wrapper strip this line from captured output and attach it to the next +! test as a name, rather than leaving it in the previous test's output. +: description ( str -- ) + "###DESC### " write print ; + ! Print one failure block in a stable, parser-friendly form. Bracketed by ! markers so a wrapper can split the stream reliably and avoid Factor's ! noisy callstack output (which is interleaved with subsequent failures). diff --git a/tests/concept-success/exercism-tools/exercism-tools.factor b/tests/concept-success/exercism-tools/exercism-tools.factor index 32a6f61..428c2d3 100644 --- a/tests/concept-success/exercism-tools/exercism-tools.factor +++ b/tests/concept-success/exercism-tools/exercism-tools.factor @@ -9,6 +9,12 @@ SYNTAX: STOP-HERE SYNTAX: TASK: lexer get next-line ; +! Label the test that follows with its description. The marker lets the +! wrapper strip this line from captured output and attach it to the next +! test as a name, rather than leaving it in the previous test's output. +: description ( str -- ) + "###DESC### " write print ; + ! Print one failure block in a stable, parser-friendly form. Bracketed by ! markers so a wrapper can split the stream reliably and avoid Factor's ! noisy callstack output (which is interleaved with subsequent failures). diff --git a/tests/empty-file/empty-file/empty-file-tests.factor b/tests/empty-file/empty-file/empty-file-tests.factor index e278520..13389a6 100644 --- a/tests/empty-file/empty-file/empty-file-tests.factor +++ b/tests/empty-file/empty-file/empty-file-tests.factor @@ -1,2 +1,5 @@ USING: empty-file exercism-tools tools.test ; +IN: empty-file.tests + +"greet returns hello" description { "hello" } [ greet ] unit-test diff --git a/tests/empty-file/exercism-tools/exercism-tools.factor b/tests/empty-file/exercism-tools/exercism-tools.factor index 32a6f61..428c2d3 100644 --- a/tests/empty-file/exercism-tools/exercism-tools.factor +++ b/tests/empty-file/exercism-tools/exercism-tools.factor @@ -9,6 +9,12 @@ SYNTAX: STOP-HERE SYNTAX: TASK: lexer get next-line ; +! Label the test that follows with its description. The marker lets the +! wrapper strip this line from captured output and attach it to the next +! test as a name, rather than leaving it in the previous test's output. +: description ( str -- ) + "###DESC### " write print ; + ! Print one failure block in a stable, parser-friendly form. Bracketed by ! markers so a wrapper can split the stream reliably and avoid Factor's ! noisy callstack output (which is interleaved with subsequent failures). diff --git a/tests/empty-file/expected_results.json b/tests/empty-file/expected_results.json index 855a32f..5b7846b 100644 --- a/tests/empty-file/expected_results.json +++ b/tests/empty-file/expected_results.json @@ -1,5 +1,5 @@ { "version": 3, "status": "error", - "message": "/opt/test-runner/tests/empty-file/empty-file/empty-file-tests.factor\n\n2: { \"hello\" } [ greet ] unit-test\n ^\nNo word named 'greet' found in current vocabulary search path" + "message": "/opt/test-runner/tests/empty-file/empty-file/empty-file-tests.factor\n\n5: { \"hello\" } [ greet ] unit-test\n ^\nNo word named 'greet' found in current vocabulary search path" } diff --git a/tests/partial-fail/exercism-tools/exercism-tools.factor b/tests/partial-fail/exercism-tools/exercism-tools.factor index 32a6f61..428c2d3 100644 --- a/tests/partial-fail/exercism-tools/exercism-tools.factor +++ b/tests/partial-fail/exercism-tools/exercism-tools.factor @@ -9,6 +9,12 @@ SYNTAX: STOP-HERE SYNTAX: TASK: lexer get next-line ; +! Label the test that follows with its description. The marker lets the +! wrapper strip this line from captured output and attach it to the next +! test as a name, rather than leaving it in the previous test's output. +: description ( str -- ) + "###DESC### " write print ; + ! Print one failure block in a stable, parser-friendly form. Bracketed by ! markers so a wrapper can split the stream reliably and avoid Factor's ! noisy callstack output (which is interleaved with subsequent failures). diff --git a/tests/partial-fail/expected_results.json b/tests/partial-fail/expected_results.json index 4718da0..9f73a6a 100644 --- a/tests/partial-fail/expected_results.json +++ b/tests/partial-fail/expected_results.json @@ -3,13 +3,13 @@ "status": "fail", "tests": [ { - "name": "Test 1", + "name": "greet returns hello", "status": "pass", "test_code": "{ \"hello\" } [ \"first call\" print greet ] unit-test", "output": "first call" }, { - "name": "Test 2", + "name": "greet returns world", "status": "fail", "test_code": "{ \"world\" } [ 2 . greet ] unit-test", "message": "=== Expected:\n\"world\"\n=== Got:\n\"hello\"", diff --git a/tests/partial-fail/partial-fail/partial-fail-tests.factor b/tests/partial-fail/partial-fail/partial-fail-tests.factor index da97542..5f7c85b 100644 --- a/tests/partial-fail/partial-fail/partial-fail-tests.factor +++ b/tests/partial-fail/partial-fail/partial-fail-tests.factor @@ -1,8 +1,10 @@ USING: exercism-tools io partial-fail prettyprint tools.test ; IN: partial-fail.tests +"greet returns hello" description { "hello" } [ "first call" print greet ] unit-test STOP-HERE +"greet returns world" description { "world" } [ 2 . greet ] unit-test diff --git a/tests/practice-stub/exercism-tools/exercism-tools.factor b/tests/practice-stub/exercism-tools/exercism-tools.factor index 32a6f61..428c2d3 100644 --- a/tests/practice-stub/exercism-tools/exercism-tools.factor +++ b/tests/practice-stub/exercism-tools/exercism-tools.factor @@ -9,6 +9,12 @@ SYNTAX: STOP-HERE SYNTAX: TASK: lexer get next-line ; +! Label the test that follows with its description. The marker lets the +! wrapper strip this line from captured output and attach it to the next +! test as a name, rather than leaving it in the previous test's output. +: description ( str -- ) + "###DESC### " write print ; + ! Print one failure block in a stable, parser-friendly form. Bracketed by ! markers so a wrapper can split the stream reliably and avoid Factor's ! noisy callstack output (which is interleaved with subsequent failures). diff --git a/tests/practice-stub/expected_results.json b/tests/practice-stub/expected_results.json index 50af9b8..56de0c1 100644 --- a/tests/practice-stub/expected_results.json +++ b/tests/practice-stub/expected_results.json @@ -3,13 +3,13 @@ "status": "fail", "tests": [ { - "name": "Test 1", + "name": "greet returns hello", "status": "error", "test_code": "{ \"hello\" } [ greet ] unit-test", "message": "unimplemented" }, { - "name": "Test 2", + "name": "greet returns world", "status": "error", "test_code": "{ \"world\" } [ greet ] unit-test", "message": "unimplemented" diff --git a/tests/practice-stub/practice-stub/practice-stub-tests.factor b/tests/practice-stub/practice-stub/practice-stub-tests.factor index 9d2efa2..943c972 100644 --- a/tests/practice-stub/practice-stub/practice-stub-tests.factor +++ b/tests/practice-stub/practice-stub/practice-stub-tests.factor @@ -1,8 +1,10 @@ USING: exercism-tools practice-stub tools.test ; IN: practice-stub.tests +"greet returns hello" description { "hello" } [ greet ] unit-test STOP-HERE +"greet returns world" description { "world" } [ greet ] unit-test diff --git a/tests/success/exercism-tools/exercism-tools.factor b/tests/success/exercism-tools/exercism-tools.factor index 32a6f61..428c2d3 100644 --- a/tests/success/exercism-tools/exercism-tools.factor +++ b/tests/success/exercism-tools/exercism-tools.factor @@ -9,6 +9,12 @@ SYNTAX: STOP-HERE SYNTAX: TASK: lexer get next-line ; +! Label the test that follows with its description. The marker lets the +! wrapper strip this line from captured output and attach it to the next +! test as a name, rather than leaving it in the previous test's output. +: description ( str -- ) + "###DESC### " write print ; + ! Print one failure block in a stable, parser-friendly form. Bracketed by ! markers so a wrapper can split the stream reliably and avoid Factor's ! noisy callstack output (which is interleaved with subsequent failures). diff --git a/tests/success/expected_results.json b/tests/success/expected_results.json index 0b6ad83..92d0ab7 100644 --- a/tests/success/expected_results.json +++ b/tests/success/expected_results.json @@ -3,7 +3,7 @@ "status": "pass", "tests": [ { - "name": "Test 1", + "name": "greet returns hello", "status": "pass", "test_code": "{ \"hello\" } [ \"calling greet\" print greet ] unit-test" } diff --git a/tests/success/success/success-tests.factor b/tests/success/success/success-tests.factor index b60150a..bcbdbd3 100644 --- a/tests/success/success/success-tests.factor +++ b/tests/success/success/success-tests.factor @@ -1,4 +1,5 @@ USING: exercism-tools io prettyprint success tools.test ; IN: success.tests +"greet returns hello" description { "hello" } [ "calling greet" print greet ] unit-test diff --git a/tests/syntax-error/exercism-tools/exercism-tools.factor b/tests/syntax-error/exercism-tools/exercism-tools.factor index 32a6f61..428c2d3 100644 --- a/tests/syntax-error/exercism-tools/exercism-tools.factor +++ b/tests/syntax-error/exercism-tools/exercism-tools.factor @@ -9,6 +9,12 @@ SYNTAX: STOP-HERE SYNTAX: TASK: lexer get next-line ; +! Label the test that follows with its description. The marker lets the +! wrapper strip this line from captured output and attach it to the next +! test as a name, rather than leaving it in the previous test's output. +: description ( str -- ) + "###DESC### " write print ; + ! Print one failure block in a stable, parser-friendly form. Bracketed by ! markers so a wrapper can split the stream reliably and avoid Factor's ! noisy callstack output (which is interleaved with subsequent failures). diff --git a/tests/syntax-error/syntax-error/syntax-error-tests.factor b/tests/syntax-error/syntax-error/syntax-error-tests.factor index 6b6a996..20cc51a 100644 --- a/tests/syntax-error/syntax-error/syntax-error-tests.factor +++ b/tests/syntax-error/syntax-error/syntax-error-tests.factor @@ -1,4 +1,5 @@ USING: exercism-tools syntax-error tools.test ; IN: syntax-error.tests +"greet returns hello" description { "hello" } [ greet ] unit-test