Refactor TestRunner state

tools/flake8.ini

@@ -21,6 +21,7 @@ ignore = E128,E129,E226,E231,E251,E265,E302,E303,E305,E402,E731,W504,W601,N801,N
 exclude =
     .tox,
     third_party,
+    third_party_modified/mozlog,
     wptserve/docs/conf.py,
     wptserve/tests/functional/docroot/invalid.py
     wptserve/wptserve/cgi/

tools/requirements_tests.txt

@@ -5,4 +5,5 @@ jsonschema==4.23.0; python_version < '3.9'
 pyyaml==6.0.1
 types-pyyaml==6.0.12.20241230
 taskcluster==96.3.1
+mozfile==3.0.0
 mozterm==1.0.0

tools/third_party_modified/mozlog/mozlog/__init__.py

@@ -21,7 +21,7 @@
 
 # Backwards compatibility shim for consumers that use mozlog.structured
 structured = sys.modules[__name__]
-sys.modules["{}.structured".format(__name__)] = structured
+sys.modules[f"{__name__}.structured"] = structured
 
 __all__ = [
     "commandline",

tools/third_party_modified/mozlog/mozlog/capture.py

@@ -18,7 +18,7 @@ def run(self):
         while True:
             try:
                 msg = self.queue.get()
-            except (EOFError, IOError):
+            except (OSError, EOFError):
                 break
             if msg is None:
                 break
@@ -57,7 +57,7 @@ def flush(self):
         pass
 
 
-class CaptureIO(object):
+class CaptureIO:
     def __init__(self, logger, do_capture, mp_context=None):
         if mp_context is None:
             import multiprocessing as mp_context

tools/third_party_modified/mozlog/mozlog/commandline.py

@@ -14,22 +14,22 @@
 log_formatters = {
     "raw": (
         formatters.JSONFormatter,
-        "Raw structured log messages " "(provided by mozlog)",
+        "Raw structured log messages (provided by mozlog)",
     ),
     "unittest": (
         formatters.UnittestFormatter,
-        "Unittest style output " "(provided by mozlog)",
+        "Unittest style output (provided by mozlog)",
     ),
     "xunit": (
         formatters.XUnitFormatter,
-        "xUnit compatible XML " "(provided by mozlog)",
+        "xUnit compatible XML (provided by mozlog)",
     ),
-    "html": (formatters.HTMLFormatter, "HTML report " "(provided by mozlog)"),
-    "mach": (formatters.MachFormatter, "Human-readable output " "(provided by mozlog)"),
-    "tbpl": (formatters.TbplFormatter, "TBPL style log format " "(provided by mozlog)"),
+    "html": (formatters.HTMLFormatter, "HTML report (provided by mozlog)"),
+    "mach": (formatters.MachFormatter, "Human-readable output (provided by mozlog)"),
+    "tbpl": (formatters.TbplFormatter, "TBPL style log format (provided by mozlog)"),
     "grouped": (
         formatters.GroupingFormatter,
-        "Grouped summary of test results " "(provided by mozlog)",
+        "Grouped summary of test results (provided by mozlog)",
     ),
     "errorsummary": (formatters.ErrorSummaryFormatter, argparse.SUPPRESS),
 }

tools/third_party_modified/mozlog/mozlog/formatters/errorsummary.py

@@ -4,6 +4,8 @@
 
 
 import json
+import math
+import os
 from collections import defaultdict
 
 from .base import BaseFormatter
@@ -12,14 +14,23 @@
 class ErrorSummaryFormatter(BaseFormatter):
     def __init__(self):
         self.test_to_group = {}
+        self.manifest_groups = set()
         self.groups = defaultdict(
             lambda: {
                 "status": None,
-                "start": None,
-                "end": None,
+                "group_start": None,
+                "group_end": None,
+                "all_skipped": True,
+                "test_starts": {},
+                "test_times": [],
             }
         )
+        self.test_time_divisor = 1
         self.line_count = 0
+        self.dump_passing_tests = False
+
+        if os.environ.get("MOZLOG_DUMP_ALL_TESTS"):
+            self.dump_passing_tests = True
 
     def __call__(self, data):
         rv = BaseFormatter.__call__(self, data)
@@ -40,45 +51,65 @@ def _output_test(self, test, subtest, item):
             "expected": item["expected"],
             "message": item.get("message"),
             "stack": item.get("stack"),
+            "modifiers": item.get("extra", {}).get("modifiers", ""),
             "known_intermittent": item.get("known_intermittent", []),
         }
         return self._output("test_result", data)
 
-    def _update_group_result(self, group, item):
-        ginfo = self.groups[group]
-
-        if item["status"] == "SKIP":
-            if ginfo["status"] is None:
-                ginfo["status"] = "SKIP"
-        elif (
-            ("expected" not in item and item["status"] in ["OK", "PASS"]) or
-            ("expected" in item and item["status"] == item["expected"]) or
-            item["status"] in item.get("known_intermittent", [])
-        ):
-            if ginfo["status"] in (None, "SKIP"):
-                ginfo["status"] = "OK"
+    def _get_group_result(self, group, item):
+        group_info = self.groups[group]
+        result = group_info["status"]
+
+        if result == "ERROR":
+            return result
+
+        # If status == expected, we delete item[expected]
+        test_status = item["status"]
+        test_expected = item.get("expected", test_status)
+        known_intermittent = item.get("known_intermittent", [])
+
+        if test_status == "SKIP":
+            if result is None:
+                result = "SKIP"
         else:
-            ginfo["status"] = "ERROR"
+            self.groups[group]["all_skipped"] = False
+            if test_status == test_expected or test_status in known_intermittent:
+                result = "OK"
+            else:
+                result = "ERROR"
+
+        return result
+
+    def _clean_test_name(self, test):
+        retVal = test
+        # remove extra stuff like "(finished)"
+        if "(finished)" in test:
+            retVal = test.split(" ")[0]
+        return retVal
 
     def suite_start(self, item):
         self.test_to_group = {v: k for k in item["tests"] for v in item["tests"][k]}
+        self.manifest_groups = set(item["tests"].keys())
         # Initialize groups with no tests (missing manifests) with SKIP status
         for group, tests in item["tests"].items():
             if not tests:  # Empty test list
                 self.groups[group] = {
                     "status": "SKIP",
-                    "start": None,
-                    "end": None,
+                    "group_start": None,
+                    "group_end": None,
+                    "all_skipped": True,
+                    "test_starts": {},
+                    "test_times": [],
                 }
         return self._output("test_groups", {"groups": list(item["tests"].keys())})
 
     def suite_end(self, data):
         output = []
         for group, info in self.groups.items():
-            if info["start"] is None or info["end"] is None:
-                duration = None
+            if info["group_start"] is not None and info["group_end"] is not None:
+                duration = info["group_end"] - info["group_start"]
             else:
-                duration = info["end"] - info["start"]
+                duration = sum(info["test_times"])
 
             output.append(
                 self._output(
@@ -93,31 +124,65 @@ def suite_end(self, data):
 
         return "".join(output)
 
+    def group_start(self, item):
+        group = item["name"]
+        if group in self.manifest_groups:
+            self.groups[group]["group_start"] = item["time"]
+        else:
+            extra = item.get("extra") or {}
+            threads = extra.get("threads")
+            if threads and threads > 1:
+                self.test_time_divisor = threads
+
+    def group_end(self, item):
+        group = item["name"]
+        if group in self.manifest_groups and not self.groups[group]["all_skipped"]:
+            self.groups[group]["group_end"] = item["time"]
+        elif group not in self.manifest_groups:
+            self.test_time_divisor = 1
+
     def test_start(self, item):
-        group = self.test_to_group.get(item["test"], None)
-        if group and self.groups[group]["start"] is None:
-            self.groups[group]["start"] = item["time"]
+        test = self._clean_test_name(item["test"])
+        group = item.get("group", self.test_to_group.get(test, None))
+        if group:
+            self.groups[group]["test_starts"][test] = item["time"]
 
     def test_status(self, item):
-        group = self.test_to_group.get(item["test"], None)
+        group = item.get(
+            "group", self.test_to_group.get(self._clean_test_name(item["test"]), None)
+        )
         if group:
-            self._update_group_result(group, item)
+            self.groups[group]["status"] = self._get_group_result(group, item)
 
-        if "expected" not in item:
+        if not self.dump_passing_tests and "expected" not in item:
             return
 
-        return self._output_test(item["test"], item["subtest"], item)
+        if item.get("expected", "") == "":
+            item["expected"] = item["status"]
+
+        return self._output_test(
+            self._clean_test_name(item["test"]), item["subtest"], item
+        )
 
     def test_end(self, item):
-        group = self.test_to_group.get(item["test"], None)
+        test = self._clean_test_name(item["test"])
+        group = item.get("group", self.test_to_group.get(test, None))
         if group:
-            self._update_group_result(group, item)
-            self.groups[group]["end"] = item["time"]
+            self.groups[group]["status"] = self._get_group_result(group, item)
+            start_time = self.groups[group]["test_starts"].pop(test, None)
+            if item["status"] != "SKIP" and start_time is not None:
+                elapsed = item["time"] - start_time
+                self.groups[group]["test_times"].append(
+                    math.ceil(elapsed / self.test_time_divisor)
+                )
 
-        if "expected" not in item:
+        if not self.dump_passing_tests and "expected" not in item:
             return
 
-        return self._output_test(item["test"], None, item)
+        if item.get("expected", "") == "":
+            item["expected"] = item["status"]
+
+        return self._output_test(test, None, item)
 
     def log(self, item):
         if item["level"] not in ("ERROR", "CRITICAL"):
@@ -126,6 +191,20 @@ def log(self, item):
         data = {"level": item["level"], "message": item["message"]}
         return self._output("log", data)
 
+    def shutdown_failure(self, item):
+        data = {"status": "FAIL", "test": item["group"], "message": item["message"]}
+        data["group"] = [g for g in self.groups if item["group"].endswith(g)]
+        if data["group"]:
+            data["group"] = data["group"][0]
+            self.groups[data["group"]]["status"] = "FAIL"
+        else:
+            self.log({
+                "level": "ERROR",
+                "message": "Group '%s' was not found in known groups: %s.  Please look at item: %s"
+                % (item["group"], self.groups, item),
+            })
+        return self._output("log", data)
+
     def crash(self, item):
         data = {
             "test": item.get("test"),
@@ -135,22 +214,24 @@ def crash(self, item):
         }
 
         if item.get("test"):
-            data["group"] = self.test_to_group.get(item["test"], "")
+            data["group"] = self.test_to_group.get(
+                self._clean_test_name(item["test"]), ""
+            )
             if data["group"] == "":
                 # item['test'] could be the group name, not a test name
-                if item["test"] in self.groups:
-                    data["group"] = item["test"]
+                if self._clean_test_name(item["test"]) in self.groups:
+                    data["group"] = self._clean_test_name(item["test"])
 
             # unlike test group summary, if we crash expect error unless expected
             if (
                 (
-                    "expected" in item and
-                    "status" in item and
-                    item["status"] in item["expected"]
-                ) or
-                ("expected" in item and "CRASH" == item["expected"]) or
-                "status" in item and
-                item["status"] in item.get("known_intermittent", [])
+                    "expected" in item
+                    and "status" in item
+                    and item["status"] in item["expected"]
+                )
+                or ("expected" in item and "CRASH" == item["expected"])
+                or "status" in item
+                and item["status"] in item.get("known_intermittent", [])
             ):
                 self.groups[data["group"]]["status"] = "PASS"
             else: