Add conformance results invariant validator (#2205)

Author: JelleZijlstra

.github/workflows/conformance.yml

@@ -32,6 +32,11 @@ jobs:
           uv sync --python 3.12 --frozen
           uv run --python 3.12 --frozen python src/main.py
 
+      - name: Validate conformance invariants
+        working-directory: conformance
+        run: |
+          uv run --python 3.12 --frozen python src/validate_results.py
+
       - name: Assert conformance results are up to date
         run: |
           if [ -n "$(git status --porcelain -- conformance/results)" ]; then

conformance/results/mypy/constructors_call_metaclass.toml

@@ -1,4 +1,4 @@
-conformant = "Unupported"
+conformant = "Unsupported"
 notes = """
 Does not honor metaclass __call__ method when evaluating constructor call.
 Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

conformance/results/mypy/generics_defaults.toml

@@ -1,4 +1,8 @@
 conformant = "Partial"
+notes = """
+Does not detect a TypeVar with a default used after a TypeVarTuple.
+Does not fully support defaults on TypeVarTuple and ParamSpec.
+"""
 output = """
 generics_defaults.py:24: error: "T" cannot appear after "DefaultStrT" in type parameter list because it has no default type  [misc]
 generics_defaults.py:66: error: "AllTheDefaults" expects between 2 and 5 type arguments, but 1 given  [type-arg]

conformance/results/mypy/generics_defaults_referential.toml

@@ -1,4 +1,7 @@
 conformant = "Partial"
+notes = """
+Does not correctly handle defaults referencing other TypeVars.
+"""
 output = """
 generics_defaults_referential.py:23: error: Expression is of type "type[slice[StartT, StopT, StepT]]", not "type[slice[int, int, int | None]]"  [assert-type]
 generics_defaults_referential.py:38: error: Argument 1 to "Foo" has incompatible type "str"; expected "int"  [arg-type]

conformance/results/mypy/generics_defaults_specialization.toml

@@ -1,4 +1,7 @@
 conformant = "Partial"
+notes = """
+Does not correctly resolve defaults when classes are used directly.
+"""
 output = """
 generics_defaults_specialization.py:30: error: Bad number of arguments for type alias, expected between 0 and 1, given 2  [type-arg]
 generics_defaults_specialization.py:45: error: Expression is of type "type[Bar[DefaultStrT]]", not "type[Bar[str]]"  [assert-type]

conformance/results/results.html

@@ -277,21 +277,21 @@ <h3>Python Type System Conformance Test Results</h3>
 <th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Incorrectly allows constrained type variables to be solved to a union of their constraints.</p></span></div></th>
 </tr>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;generics_defaults</th>
-<th class="column col2 partially-conformant">Partial</th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not detect a TypeVar with a default used after a TypeVarTuple.</p><p>Does not fully support defaults on TypeVarTuple and ParamSpec.</p></span></div></th>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not forbid a `TypeVar` immediately following a `TypeVarTuple` in a parameter list from having a default.</p><p>Does not support `TypeVarTuple`.</p><p>Does not fully support defaults for `ParamSpec`s.</p></span></div></th>
 </tr>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;generics_defaults_referential</th>
-<th class="column col2 partially-conformant">Partial</th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not correctly handle defaults referencing other TypeVars.</p></span></div></th>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 conformant">Pass</th>
 </tr>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;generics_defaults_specialization</th>
-<th class="column col2 partially-conformant">Partial</th>
+<th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not correctly resolve defaults when classes are used directly.</p></span></div></th>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 conformant">Pass</th>
@@ -718,7 +718,7 @@ <h3>Python Type System Conformance Test Results</h3>
 <th class="column col2 partially-conformant"><div class="hover-text">Partial<span class="tooltip-text" id="bottom"><p>Does not reject invalid argument types to an inherited constructor in a specialized subclass of a generic superclass.</p><p>Does not reject class-scoped type variables used in the `self` annotation.</p><p>Does not support inferring type variables for generic classes where the `__init__` method uses method-scoped type variables.</p></span></div></th>
 </tr>
 <tr><th class="column col1">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;constructors_call_metaclass</th>
-<th class="column col2 not-conformant"><div class="hover-text">Unupported<span class="tooltip-text" id="bottom"><p>Does not honor metaclass __call__ method when evaluating constructor call.</p><p>Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.</p></span></div></th>
+<th class="column col2 not-conformant"><div class="hover-text">Unsupported<span class="tooltip-text" id="bottom"><p>Does not honor metaclass __call__ method when evaluating constructor call.</p><p>Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.</p></span></div></th>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 conformant">Pass</th>
 <th class="column col2 conformant">Pass</th>

conformance/src/validate_results.py

@@ -0,0 +1,105 @@
+"""
+Validate invariants for conformance result files.
+"""
+
+from pathlib import Path
+import sys
+import tomllib
+from typing import Any
+
+ALLOWED_RESULT_KEYS = frozenset(
+    {
+        "conformance_automated",
+        "conformant",
+        "errors_diff",
+        "ignore_errors",
+        "notes",
+        "output",
+    }
+)
+
+
+def main() -> int:
+    results_dir = Path(__file__).resolve().parent.parent / "results"
+    issues: list[str] = []
+    checked = 0
+
+    for type_checker_dir in sorted(results_dir.iterdir()):
+        if not type_checker_dir.is_dir():
+            continue
+        for file in sorted(type_checker_dir.iterdir()):
+            if file.name == "version.toml":
+                continue
+            checked += 1
+            try:
+                with file.open("rb") as f:
+                    info = tomllib.load(f)
+            except Exception as e:
+                issues.append(f"{file.relative_to(results_dir)}: failed to parse TOML ({e})")
+                continue
+
+            issues.extend(_validate_result(file, results_dir, info))
+
+    if issues:
+        print(f"Found {len(issues)} invariant violation(s) across {checked} file(s):")
+        for issue in issues:
+            print(f"- {issue}")
+        return 1
+
+    print(f"Validated {checked} conformance result file(s); no invariant violations found.")
+    return 0
+
+
+def _validate_result(file: Path, results_dir: Path, info: dict[str, Any]) -> list[str]:
+    issues: list[str] = []
+    rel_path = file.relative_to(results_dir)
+
+    unknown_keys = sorted(set(info) - ALLOWED_RESULT_KEYS)
+    if unknown_keys:
+        issues.append(
+            f"{rel_path}: unrecognized key(s): {', '.join(repr(key) for key in unknown_keys)}"
+        )
+
+    automated = info.get("conformance_automated")
+    if automated not in {"Pass", "Fail"}:
+        issues.append(
+            f"{rel_path}: conformance_automated must be 'Pass' or 'Fail' (got {automated!r})"
+        )
+        return issues
+    automated_is_pass = automated == "Pass"
+
+    conformant = info.get("conformant")
+    if conformant is None:
+        if automated_is_pass:
+            conformant_is_pass = True
+        else:
+            issues.append(
+                f"{rel_path}: conformant is required when conformance_automated is 'Fail'"
+            )
+            return issues
+    elif isinstance(conformant, str):
+        if conformant not in ("Pass", "Partial", "Unsupported"):
+            issues.append(f"{rel_path}: invalid conformance status {conformant!r}")
+        conformant_is_pass = conformant == "Pass"
+    else:
+        issues.append(f"{rel_path}: conformant must be a string when present")
+        return issues
+
+    if conformant_is_pass != automated_is_pass:
+        issues.append(
+            f"{rel_path}: conformant={conformant!r} does not match "
+            f"conformance_automated={automated!r}"
+        )
+
+    if conformant == "Partial":
+        notes = info.get("notes", "")
+        if not isinstance(notes, str) or not notes.strip():
+            issues.append(
+                f"{rel_path}: notes must be present when checker is not fully conformant"
+            )
+
+    return issues
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())