tap2json: script to convert TAP to JSON

This will be used by our CI to track flacky tests.

Signed-off-by: Matthieu Baerts (NGI0) <[email protected]>

Dockerfile

@@ -99,6 +99,6 @@ ENV CCACHE_COMPRESS true
 ENV KBUILD_BUILD_TIMESTAMP "0"
 ENV GCC_COLORS error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01
 
-COPY entrypoint.sh /
+COPY entrypoint.sh tap2json.py /
 
 ENTRYPOINT ["/entrypoint.sh"]

entrypoint.sh

@@ -1328,6 +1328,18 @@ _get_failed_tests_status() { local t fails=()
 	echo "${#fails[@]} failed test(s): ${fails[*]}"
 }
 
+_gen_results_files() {
+	LANG=C tap2junit "${RESULTS_DIR}"/*.tap
+
+	# remove prefix id (unique test) and comments (status, time)
+	sed -i 's/\(<testcase name="\)[0-9]\+[ -]*/\1/g;/<testcase name=/s/\s\+#.*">/">/g' "${RESULTS_DIR}"/*.tap.xml
+
+	LANG=C /tap2json.py \
+		--output "${RESULTS_DIR}/results.json" \
+		--info "run_id:${GITHUB_RUN_ID:-"none"}" \
+		"${RESULTS_DIR}/*.tap"
+}
+
 # $1: mode, rest: args for kconfig
 analyze() {
 	# reduce log that could be wrongly interpreted
@@ -1339,10 +1351,7 @@ analyze() {
 	printinfo "Analyze results"
 
 	if is_ci; then
-		LANG=C tap2junit "${RESULTS_DIR}"/*.tap
-
-		# remove prefix id (unique test) and comments (status, time)
-		sed -i 's/\(<testcase name="\)[0-9]\+[ -]*/\1/g;/<testcase name=/s/\s\+#.*">/">/g' "${RESULTS_DIR}"/*.tap.xml
+		_gen_results_files || true
 	fi
 
 	echo -ne "\n${COLOR_GREEN}"

tap2json.py

@@ -0,0 +1,140 @@
+#! /usr/bin/python3
+# SPDX-License-Identifier: GPL-2.0
+#
+# Very simple TAP to JSON parser
+#
+# JQ can be used to filter tests later, e.g.abs
+#    $ jq '.results.[] | select(.[].result == "fail")' results.json
+
+import argparse
+import json
+import os
+import re
+import sys
+
+
+def get_args_parser():
+	parser = argparse.ArgumentParser(
+		description="(Simple) TAP to JSON converter"
+	)
+
+	parser.add_argument(
+		"--output",
+		"-o",
+		action="store",
+		help="Output JSON file"
+	)
+
+	parser.add_argument(
+		"--info",
+		"-I",
+		action="append",
+		metavar="key:value",
+		help="Add extra info in the JSON, can be used multiple times"
+	)
+
+	parser.add_argument(
+		"--only-fails",
+		"-f",
+		action="store_true",
+		help="Only keep failed tests"
+	)
+
+	parser.add_argument(
+		"tapfiles",
+		metavar="tapfiles",
+		type=str,
+		nargs="*",
+		help="Input TAP file(s)"
+	)
+
+	return parser
+
+# Same as in NIPA
+TAP_RE = re.compile(r"(not )?ok (\d+)( -)? ([^#]*[^ ])( # )?([^ ].*)?$")
+
+def parse_tap(tap, name, only_fails):
+	results = {}
+	has_results = False
+
+	for line in tap:
+		try:
+			r = TAP_RE.match(line.rstrip()).groups()
+		except AttributeError:
+			continue
+
+		has_results = True
+
+		success = r[0] is None
+
+		if only_fails and success:
+			continue
+
+		result = {
+			'result': "pass" if success else "fail",
+			'name': r[3]
+		}
+
+		if r[5]:
+			result['comment'] = r[5]
+			if r[5].lower().startswith('skip') and success:
+				result['result'] = "skip"
+
+		results[r[1]] = result
+
+	# just in case, to catch errors
+	if not has_results:
+		results[0] = {'result': "fail", 'name': name}
+
+	return results
+
+
+def parse_all_tap(tap_files, only_fails):
+	results = {}
+
+	for tap in tap_files:
+		name = os.path.splitext(os.path.basename(tap))[0]
+		with open(tap, "r", encoding="utf-8") as fd:
+			result = parse_tap(fd.readlines(), name, only_fails)
+			if result:
+				results[name] = result
+
+	return results
+
+def add_info(results, infos):
+	results = {
+		"results": results
+	}
+
+	for info in infos:
+		info = info.split(':', 1)
+		if len(info) != 2:
+			print("Skip info: " + info[0], file=sys.stderr)
+			continue
+
+		results[info[0]] = info[1]
+
+	return results
+
+def write_json(out_file, results):
+	out = json.dumps(results)
+	if out_file:
+		with open(out_file, "w") as fd:
+			fd.write(out)
+	else:
+		print(out)
+
+if __name__ == "__main__":
+	arg_parser = get_args_parser()
+	args = arg_parser.parse_args()
+
+	if not args.tapfiles:
+		arg_parser.print_usage()
+		sys.exit(1)
+
+	results = parse_all_tap(args.tapfiles, args.only_fails)
+
+	if args.info:
+		results = add_info(results, args.info)
+
+	write_json(args.output, results)