Parse test description

bin/run.sh

@@ -93,7 +93,7 @@ src_tests=$(awk "${awk_json}"'
 ' "${tmp_dir}/${slug}/${slug}-tests.factor")
 
 # 2. Parse Factor stdout into NDJSON segments and failures:
-#    segments: {"type":"segment","idx":N,"failed":bool,"output":"..."}
+#    segments: {"type":"segment","idx":N,"failed":bool,"name":"...","output":"..."}
 #    failures: {"type":"failure","line_no":N,"message":"..."}
 parsed=$(printf '%s\n' "${raw_output}" | awk "${awk_json}"'
     function close_segment(   out, i) {
@@ -102,8 +102,8 @@ parsed=$(printf '%s\n' "${raw_output}" | awk "${awk_json}"'
         for (i = 1; i <= seg_n; i++) out = out (i > 1 ? "\n" : "") seg[i]
         sub(/^\n+/, "", out)
         sub(/\n+$/, "", out)
-        printf "{\"type\":\"segment\",\"idx\":%d,\"failed\":%s,\"output\":%s}\n",
-            idx, (seg_failed ? "true" : "false"), json_str(out)
+        printf "{\"type\":\"segment\",\"idx\":%d,\"failed\":%s,\"name\":%s,\"output\":%s}\n",
+            idx, (seg_failed ? "true" : "false"), json_str(cur_name), json_str(out)
     }
     function close_failure(   body, i) {
         body = ""
@@ -126,6 +126,9 @@ parsed=$(printf '%s\n' "${raw_output}" | awk "${awk_json}"'
     BEGIN {
         state = "inline"
         idx = 0
+        pending_name = ""
+        cur_name = ""
+        desc_re = "^###DESC### "
         header_re = "^(Unit Test|Unit Test~|Unit Test V~|Long Unit Test|Must Fail|Must Fail With|Must Not Fail|Must Infer|Must Infer As): "
     }
     state == "inline" && $0 ~ header_re {
@@ -134,6 +137,18 @@ parsed=$(printf '%s\n' "${raw_output}" | awk "${awk_json}"'
         seg_failed = 0
         seg_n = 0
         delete seg
+        cur_name = pending_name
+        pending_name = ""
+        next
+    }
+    # A generated "###DESC### <description>" line labels the test that
+    # follows it (it is printed before that test'"'"'s "Unit Test:" header).
+    # Stash it as the upcoming segment'"'"'s name; never add it to any
+    # segment'"'"'s output. Concept exercises emit no such lines.
+    state == "inline" && $0 ~ desc_re {
+        line = $0
+        sub(desc_re, "", line)
+        pending_name = line
         next
     }
     state == "inline" && $0 == "###FAIL_BEGIN###" {
@@ -188,7 +203,7 @@ failures=$(printf '%s\n' "${parsed}" | awk '/"type":"failure"/' || true)
 
 # 3. If no segments emitted, surface a top-level error from the raw output.
 if [[ -z "${segments}" ]]; then
-    cleaned=$(printf '%s\n' "${raw_output}" | awk '/^\([UO]\) /{exit} {print}' \
+    cleaned=$(printf '%s\n' "${raw_output}" | awk '/^\([UO]\) /{exit} !/^###DESC### /{print}' \
         | awk '
             NF {
                 print
@@ -229,8 +244,10 @@ jq -n \
                 else "fail" end
             else "pass" end
           ) as $status
+        | ($seg.name // "") as $label
         | {
-            name: ("Test " + ((.key + 1) | tostring)),
+            name: (if ($label | length) > 0 then $label
+                   else ("Test " + ((.key + 1) | tostring)) end),
             status: $status,
             test_code: ($src.test_code // ""),
           }

tests/all-fail/all-fail/all-fail-tests.factor

@@ -1,8 +1,10 @@
 USING: all-fail exercism-tools tools.test ;
 IN: all-fail.tests
 
+"greet returns hello" description
 { "hello" } [ greet ] unit-test
 
 STOP-HERE
 
+"greet returns world" description
 { "world" } [ greet ] unit-test

tests/all-fail/exercism-tools/exercism-tools.factor

@@ -9,6 +9,12 @@ SYNTAX: STOP-HERE
 SYNTAX: TASK:
     lexer get next-line ;
 
+! Label the test that follows with its description. The marker lets the
+! wrapper strip this line from captured output and attach it to the next
+! test as a name, rather than leaving it in the previous test's output.
+: description ( str -- )
+    "###DESC### " write print ;
+
 ! Print one failure block in a stable, parser-friendly form. Bracketed by
 ! markers so a wrapper can split the stream reliably and avoid Factor's
 ! noisy callstack output (which is interleaved with subsequent failures).

tests/all-fail/expected_results.json

@@ -3,13 +3,13 @@
   "status": "fail",
   "tests": [
     {
-      "name": "Test 1",
+      "name": "greet returns hello",
       "status": "fail",
       "test_code": "{ \"hello\" } [ greet ] unit-test",
       "message": "=== Expected:\n\"hello\"\n=== Got:\n\"wrong\""
     },
     {
-      "name": "Test 2",
+      "name": "greet returns world",
       "status": "fail",
       "test_code": "{ \"world\" } [ greet ] unit-test",
       "message": "=== Expected:\n\"world\"\n=== Got:\n\"wrong\""

tests/concept-concurrency/exercism-tools/exercism-tools.factor

@@ -9,6 +9,12 @@ SYNTAX: STOP-HERE
 SYNTAX: TASK:
     lexer get next-line ;
 
+! Label the test that follows with its description. The marker lets the
+! wrapper strip this line from captured output and attach it to the next
+! test as a name, rather than leaving it in the previous test's output.
+: description ( str -- )
+    "###DESC### " write print ;
+
 ! Print one failure block in a stable, parser-friendly form. Bracketed by
 ! markers so a wrapper can split the stream reliably and avoid Factor's
 ! noisy callstack output (which is interleaved with subsequent failures).

tests/concept-not-parsing/exercism-tools/exercism-tools.factor

@@ -9,6 +9,12 @@ SYNTAX: STOP-HERE
 SYNTAX: TASK:
     lexer get next-line ;
 
+! Label the test that follows with its description. The marker lets the
+! wrapper strip this line from captured output and attach it to the next
+! test as a name, rather than leaving it in the previous test's output.
+: description ( str -- )
+    "###DESC### " write print ;
+
 ! Print one failure block in a stable, parser-friendly form. Bracketed by
 ! markers so a wrapper can split the stream reliably and avoid Factor's
 ! noisy callstack output (which is interleaved with subsequent failures).

tests/concept-partial-fail/exercism-tools/exercism-tools.factor

@@ -9,6 +9,12 @@ SYNTAX: STOP-HERE
 SYNTAX: TASK:
     lexer get next-line ;
 
+! Label the test that follows with its description. The marker lets the
+! wrapper strip this line from captured output and attach it to the next
+! test as a name, rather than leaving it in the previous test's output.
+: description ( str -- )
+    "###DESC### " write print ;
+
 ! Print one failure block in a stable, parser-friendly form. Bracketed by
 ! markers so a wrapper can split the stream reliably and avoid Factor's
 ! noisy callstack output (which is interleaved with subsequent failures).

tests/concept-stub/exercism-tools/exercism-tools.factor

@@ -9,6 +9,12 @@ SYNTAX: STOP-HERE
 SYNTAX: TASK:
     lexer get next-line ;
 
+! Label the test that follows with its description. The marker lets the
+! wrapper strip this line from captured output and attach it to the next
+! test as a name, rather than leaving it in the previous test's output.
+: description ( str -- )
+    "###DESC### " write print ;
+
 ! Print one failure block in a stable, parser-friendly form. Bracketed by
 ! markers so a wrapper can split the stream reliably and avoid Factor's
 ! noisy callstack output (which is interleaved with subsequent failures).

tests/concept-success/exercism-tools/exercism-tools.factor

@@ -9,6 +9,12 @@ SYNTAX: STOP-HERE
 SYNTAX: TASK:
     lexer get next-line ;
 
+! Label the test that follows with its description. The marker lets the
+! wrapper strip this line from captured output and attach it to the next
+! test as a name, rather than leaving it in the previous test's output.
+: description ( str -- )
+    "###DESC### " write print ;
+
 ! Print one failure block in a stable, parser-friendly form. Bracketed by
 ! markers so a wrapper can split the stream reliably and avoid Factor's
 ! noisy callstack output (which is interleaved with subsequent failures).

tests/empty-file/empty-file/empty-file-tests.factor

@@ -1,2 +1,5 @@
 USING: empty-file exercism-tools tools.test ;
+IN: empty-file.tests
+
+"greet returns hello" description
 { "hello" } [ greet ] unit-test

tests/empty-file/exercism-tools/exercism-tools.factor

@@ -9,6 +9,12 @@ SYNTAX: STOP-HERE
 SYNTAX: TASK:
     lexer get next-line ;
 
+! Label the test that follows with its description. The marker lets the
+! wrapper strip this line from captured output and attach it to the next
+! test as a name, rather than leaving it in the previous test's output.
+: description ( str -- )
+    "###DESC### " write print ;
+
 ! Print one failure block in a stable, parser-friendly form. Bracketed by
 ! markers so a wrapper can split the stream reliably and avoid Factor's
 ! noisy callstack output (which is interleaved with subsequent failures).

tests/empty-file/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 3,
   "status": "error",
-  "message": "/opt/test-runner/tests/empty-file/empty-file/empty-file-tests.factor\n\n2: { \"hello\" } [ greet ] unit-test\n                      ^\nNo word named 'greet' found in current vocabulary search path"
+  "message": "/opt/test-runner/tests/empty-file/empty-file/empty-file-tests.factor\n\n5: { \"hello\" } [ greet ] unit-test\n                      ^\nNo word named 'greet' found in current vocabulary search path"
 }

tests/partial-fail/exercism-tools/exercism-tools.factor

@@ -9,6 +9,12 @@ SYNTAX: STOP-HERE
 SYNTAX: TASK:
     lexer get next-line ;
 
+! Label the test that follows with its description. The marker lets the
+! wrapper strip this line from captured output and attach it to the next
+! test as a name, rather than leaving it in the previous test's output.
+: description ( str -- )
+    "###DESC### " write print ;
+
 ! Print one failure block in a stable, parser-friendly form. Bracketed by
 ! markers so a wrapper can split the stream reliably and avoid Factor's
 ! noisy callstack output (which is interleaved with subsequent failures).

tests/partial-fail/expected_results.json

@@ -3,13 +3,13 @@
   "status": "fail",
   "tests": [
     {
-      "name": "Test 1",
+      "name": "greet returns hello",
       "status": "pass",
       "test_code": "{ \"hello\" } [ \"first call\" print greet ] unit-test",
       "output": "first call"
     },
     {
-      "name": "Test 2",
+      "name": "greet returns world",
       "status": "fail",
       "test_code": "{ \"world\" } [ 2 . greet ] unit-test",
       "message": "=== Expected:\n\"world\"\n=== Got:\n\"hello\"",

tests/partial-fail/partial-fail/partial-fail-tests.factor

@@ -1,8 +1,10 @@
 USING: exercism-tools io partial-fail prettyprint tools.test ;
 IN: partial-fail.tests
 
+"greet returns hello" description
 { "hello" } [ "first call" print greet ] unit-test
 
 STOP-HERE
 
+"greet returns world" description
 { "world" } [ 2 . greet ] unit-test

tests/practice-stub/exercism-tools/exercism-tools.factor

@@ -9,6 +9,12 @@ SYNTAX: STOP-HERE
 SYNTAX: TASK:
     lexer get next-line ;
 
+! Label the test that follows with its description. The marker lets the
+! wrapper strip this line from captured output and attach it to the next
+! test as a name, rather than leaving it in the previous test's output.
+: description ( str -- )
+    "###DESC### " write print ;
+
 ! Print one failure block in a stable, parser-friendly form. Bracketed by
 ! markers so a wrapper can split the stream reliably and avoid Factor's
 ! noisy callstack output (which is interleaved with subsequent failures).

tests/practice-stub/expected_results.json

@@ -3,13 +3,13 @@
   "status": "fail",
   "tests": [
     {
-      "name": "Test 1",
+      "name": "greet returns hello",
       "status": "error",
       "test_code": "{ \"hello\" } [ greet ] unit-test",
       "message": "unimplemented"
     },
     {
-      "name": "Test 2",
+      "name": "greet returns world",
       "status": "error",
       "test_code": "{ \"world\" } [ greet ] unit-test",
       "message": "unimplemented"

tests/practice-stub/practice-stub/practice-stub-tests.factor

@@ -1,8 +1,10 @@
 USING: exercism-tools practice-stub tools.test ;
 IN: practice-stub.tests
 
+"greet returns hello" description
 { "hello" } [ greet ] unit-test
 
 STOP-HERE
 
+"greet returns world" description
 { "world" } [ greet ] unit-test

tests/success/exercism-tools/exercism-tools.factor

@@ -9,6 +9,12 @@ SYNTAX: STOP-HERE
 SYNTAX: TASK:
     lexer get next-line ;
 
+! Label the test that follows with its description. The marker lets the
+! wrapper strip this line from captured output and attach it to the next
+! test as a name, rather than leaving it in the previous test's output.
+: description ( str -- )
+    "###DESC### " write print ;
+
 ! Print one failure block in a stable, parser-friendly form. Bracketed by
 ! markers so a wrapper can split the stream reliably and avoid Factor's
 ! noisy callstack output (which is interleaved with subsequent failures).

tests/success/expected_results.json

@@ -3,7 +3,7 @@
   "status": "pass",
   "tests": [
     {
-      "name": "Test 1",
+      "name": "greet returns hello",
       "status": "pass",
       "test_code": "{ \"hello\" } [ \"calling greet\" print greet ] unit-test"
     }

tests/success/success/success-tests.factor

@@ -1,4 +1,5 @@
 USING: exercism-tools io prettyprint success tools.test ;
 IN: success.tests
 
+"greet returns hello" description
 { "hello" } [ "calling greet" print greet ] unit-test

tests/syntax-error/exercism-tools/exercism-tools.factor

@@ -9,6 +9,12 @@ SYNTAX: STOP-HERE
 SYNTAX: TASK:
     lexer get next-line ;
 
+! Label the test that follows with its description. The marker lets the
+! wrapper strip this line from captured output and attach it to the next
+! test as a name, rather than leaving it in the previous test's output.
+: description ( str -- )
+    "###DESC### " write print ;
+
 ! Print one failure block in a stable, parser-friendly form. Bracketed by
 ! markers so a wrapper can split the stream reliably and avoid Factor's
 ! noisy callstack output (which is interleaved with subsequent failures).

tests/syntax-error/syntax-error/syntax-error-tests.factor

@@ -1,4 +1,5 @@
 USING: exercism-tools syntax-error tools.test ;
 IN: syntax-error.tests
 
+"greet returns hello" description
 { "hello" } [ greet ] unit-test