migrate typechecker to mypy (#691)

Author: mlin

.mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+mypy_path = $MYPY_CONFIG_FILE_DIR/stubs

Makefile

@@ -41,15 +41,12 @@ ci_housekeeping: sopretty check_check check doc
 ci_unit_tests: unit_tests
 
 check:
-	pyre \
-		--search-path stubs \
-		--typeshed `python3 -c 'import sys, site, os; print(next(p for p in (os.path.join(dir,"lib/pyre_check/typeshed") for dir in (sys.prefix,site.getuserbase())) if os.path.isdir(p)))'` \
-		--show-parse-errors check
+	mypy WDL
 	pylint -j `python3 -c 'import multiprocessing as mp; print(mp.cpu_count())'` --errors-only WDL
 	flake8 WDL
 
 check_check:
-	# regression test against pyre doing nothing (issue #100)
+	# regression test against pyre/mypy doing nothing (issue #100)
 	echo "check_check: str = 42" > WDL/DELETEME_check_check.py
 	$(MAKE) check > /dev/null 2>&1 && exit 1 || exit 0
 	rm WDL/DELETEME_check_check.py

WDL/CLI.py

@@ -146,7 +146,7 @@ def __call__(self, parser, namespace, values, option_string=None):
         # importlib_metadata doesn't seem to provide EntryPoint.dist to get from an entry point to
         # the metadata of the package providing it; continuing to use pkg_resources for this. Risk
         # that they give inconsistent results?
-        import pkg_resources
+        import pkg_resources  # type: ignore
 
         for group in runtime.config.default_plugins().keys():
             group = f"miniwdl.plugin.{group}"
@@ -1117,7 +1117,8 @@ def runner_input_json_file(available_inputs, namespace, input_file, downloadable
     if input_file:
         input_file = input_file.strip()
     if input_file:
-        import yaml  # delayed heavy import
+        # delayed heavy import:
+        import yaml  # type: ignore
 
         input_json = None
         if input_file[0] == "{":
@@ -1157,7 +1158,7 @@ def bold(line):
     ans = [
         "",
         bold(f"{target.name} ({target.pos.uri})"),
-        bold(f"{'-'*(len(target.name)+len(target.pos.uri)+3)}"),
+        bold(f"{'-' * (len(target.name) + len(target.pos.uri) + 3)}"),
     ]
     required_inputs = target.required_inputs
     ans.append(bold("\nrequired inputs:"))
@@ -1756,7 +1757,7 @@ def configure(cfg=None, show=False, force=False, **kwargs):
         die("`miniwdl configure` is for interactive use")
 
     from datetime import datetime
-    import bullet
+    import bullet  # type: ignore
     from xdg import XDG_CONFIG_HOME
 
     miniwdl_version = pkg_version()
@@ -2159,6 +2160,7 @@ def _type_to_input_template(ty: Type.Base):
     """
     if isinstance(ty, Type.StructInstance):
         ans = {}
+        assert ty.members
         for member_name, member_type in ty.members.items():
             if not member_type.optional:  # TODO: opt in to these
                 ans[member_name] = _type_to_input_template(member_type)

WDL/Env.py

@@ -1,7 +1,7 @@
-# pyre-strict
 """
 Environments, for identifier resolution during WDL typechecking and evaluation.
 """
+
 from typing import Optional, TypeVar, Generic, Any, Callable, Set, Iterator
 
 T = TypeVar("T")
@@ -18,9 +18,9 @@ class Binding(Generic[T]):
 
     _name: str
     _value: T
-    _info: Any  # pyre-ignore
+    _info: Any
 
-    def __init__(self, name: str, value: T, info: Any = None) -> None:  # pyre-ignore
+    def __init__(self, name: str, value: T, info: Any = None) -> None:
         self._name = name
         self._value = value
         self._info = info
@@ -39,7 +39,7 @@ def value(self) -> T:
         return self._value
 
     @property
-    def info(self) -> Any:  # pyre-ignore
+    def info(self) -> Any:
         ":type: Any"
         return self._info
 
@@ -76,7 +76,7 @@ def __bool__(self) -> bool:
 
     def __iter__(self) -> Iterator[Binding[T]]:
         mask = set()
-        pos = self
+        pos: Optional[Bindings[T]] = self
         while pos is not None:
             if isinstance(pos._binding, Binding) and pos._binding.name not in mask:
                 mask.add(pos._binding.name)
@@ -86,7 +86,7 @@ def __iter__(self) -> Iterator[Binding[T]]:
     def __len__(self) -> int:
         return sum(1 for _ in self)
 
-    def bind(self, name: str, value: T, info: Any = None) -> "Bindings[T]":  # pyre-ignore
+    def bind(self, name: str, value: T, info: Any = None) -> "Bindings[T]":
         """
         Return an environment with a new binding prepended. Any existing binding for the same name
         is shadowed by the new one. (This should not usually arise in view of the immutability of
@@ -146,7 +146,7 @@ def map(self, f: Callable[[Binding[T]], Optional[Binding[S]]]) -> "Bindings[S]":
         Copy the environment with each binding transformed by the given function. If the function
         returns ``None`` then the binding is excluded.
         """
-        ans = Bindings()
+        ans: Bindings[S] = Bindings()
         for b in self:
             fb = f(b)
             if isinstance(fb, Binding):
@@ -214,14 +214,14 @@ def wrap_namespace(self, namespace: str) -> "Bindings[T]":
         assert namespace
         if not namespace.endswith("."):
             namespace += "."
-        ans = Bindings()
+        ans: Bindings[T] = Bindings()
         for b in self:
             ans = Bindings(Binding(namespace + b.name, b.value, b.info), ans)
         return _rev(ans)
 
 
 def _rev(env: Bindings[T]) -> Bindings[T]:
-    ans = Bindings()
+    ans: Bindings[T] = Bindings()
     for b in env:
         ans = Bindings(b, ans)
     return ans

WDL/Error.py

@@ -1,10 +1,8 @@
-# pyre-strict
 from typing import (
     List,
     Optional,
     Union,
     Iterable,
-    TypeVar,
     Generator,
     Callable,
     Any,
@@ -69,9 +67,6 @@ def __init__(self, pos: SourcePosition, import_uri: str, message: Optional[str]
         self.pos = pos
 
 
-TVSourceNode = TypeVar("TVSourceNode", bound="SourceNode")
-
-
 @total_ordering
 class SourceNode:
     """Base class for an AST node, recording the source position"""
@@ -83,32 +78,36 @@ class SourceNode:
     Source position for this AST node
     """
 
+    parent: Optional["SourceNode"] = None
+    """
+    :type: Optional[SourceNode]
+
+    Parent node in the AST, if any
+    """
+
     def __init__(self, pos: SourcePosition) -> None:
         self.pos = pos
 
-    def __lt__(self, rhs: TVSourceNode) -> bool:
-        if isinstance(rhs, SourceNode):
-            return (
-                self.pos.abspath,
-                self.pos.line,
-                self.pos.column,
-                self.pos.end_line,
-                self.pos.end_column,
-            ) < (
-                rhs.pos.abspath,
-                rhs.pos.line,
-                rhs.pos.column,
-                rhs.pos.end_line,
-                rhs.pos.end_column,
-            )
-        return False
+    def __lt__(self, rhs: "SourceNode") -> bool:
+        return isinstance(rhs, SourceNode) and (
+            self.pos.abspath,
+            self.pos.line,
+            self.pos.column,
+            self.pos.end_line,
+            self.pos.end_column,
+        ) < (
+            rhs.pos.abspath,
+            rhs.pos.line,
+            rhs.pos.column,
+            rhs.pos.end_line,
+            rhs.pos.end_column,
+        )
 
-    def __eq__(self, rhs: TVSourceNode) -> bool:
-        assert isinstance(rhs, SourceNode)
-        return self.pos == rhs.pos
+    def __eq__(self, rhs: Any) -> bool:
+        return isinstance(rhs, SourceNode) and self.pos == rhs.pos
 
     @property
-    def children(self: TVSourceNode) -> Iterable[TVSourceNode]:
+    def children(self) -> Iterable["SourceNode"]:
         """
         :type: Iterable[SourceNode]
 
@@ -134,6 +133,8 @@ class ValidationError(Exception):
 
     The complete source text of the WDL document (if available)"""
 
+    declared_wdl_version: Optional[str] = None
+
     def __init__(self, node: Union[SourceNode, SourcePosition], message: str) -> None:
         if isinstance(node, SourceNode):
             self.node = node
@@ -162,25 +163,25 @@ def __init__(self, node: Union[SourceNode, SourcePosition], name: str) -> None:
 
 
 class NoSuchFunction(ValidationError):
-    def __init__(self, node: SourceNode, name: str) -> None:
+    def __init__(self, node: Union[SourceNode, SourcePosition], name: str) -> None:
         super().__init__(node, "No such function: " + name)
 
 
 class WrongArity(ValidationError):
-    def __init__(self, node: SourceNode, expected: int) -> None:
+    def __init__(self, node: Union[SourceNode, SourcePosition], expected: int) -> None:
         # avoiding circular dep:
         # assert isinstance(node, WDL.Expr.Apply)
         msg = "{} expects {} argument(s)".format(getattr(node, "function_name"), expected)
         super().__init__(node, msg)
 
 
 class NotAnArray(ValidationError):
-    def __init__(self, node: SourceNode) -> None:
+    def __init__(self, node: Union[SourceNode, SourcePosition]) -> None:
         super().__init__(node, "Not an array")
 
 
 class NoSuchMember(ValidationError):
-    def __init__(self, node: SourceNode, member: str) -> None:
+    def __init__(self, node: Union[SourceNode, SourcePosition], member: str) -> None:
         super().__init__(node, "No such member '{}'".format(member))
 
 
@@ -266,6 +267,10 @@ class MultipleValidationErrors(Exception):
     exceptions: List[ValidationError]
     """:type: List[ValidationError]"""
 
+    source_text: Optional[str] = None
+
+    declared_wdl_version: Optional[str] = None
+
     def __init__(
         self, *exceptions: List[Union[ValidationError, "MultipleValidationErrors"]]
     ) -> None:
@@ -290,7 +295,7 @@ class _MultiContext:
     def __init__(self) -> None:
         self._exceptions = []
 
-    def try1(self, fn: Callable[[], Any]) -> Optional[Any]:  # pyre-ignore
+    def try1(self, fn: Callable[[], Any]) -> Optional[Any]:
         try:
             return fn()
         except (ValidationError, MultipleValidationErrors) as exn:
@@ -304,8 +309,7 @@ def maybe_raise(self) -> None:
         if len(self._exceptions) == 1:
             raise self._exceptions[0]
         if self._exceptions:
-            # pyre-ignore
-            raise MultipleValidationErrors(*self._exceptions) from self._exceptions[0]
+            raise MultipleValidationErrors(*self._exceptions) from self._exceptions[0]  # type: ignore
 
 
 @contextmanager
@@ -345,7 +349,6 @@ class RuntimeError(Exception):
     Backend-specific information about an error (for example, pointer to a centralized log system)
     """
 
-    # pyre-ignore
     def __init__(self, *args, more_info: Optional[Dict[str, Any]] = None, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self.more_info = more_info if more_info else {}