Find precise fallback for ambiguous Any overloads (#22060)

It is a common pattern (especially in numeric libraries) to have a
fallback overload that has a relatively precise return type (i.e. not
just `Any` or `list[Any]`). We should try to find and use that overload
if there is an ambiguity caused by an argument that contains `Any`.

This will likely cause some new errors, but this should limit the
fallout from #22011

Author: ilevkivskyi

mypy/checker.py

@@ -222,6 +222,7 @@ def __init__(self) -> None:
 from mypy.state import state
 from mypy.subtypes import (
     find_member,
+    has_any_type,
     infer_class_variances,
     is_callable_compatible,
     is_equivalent,
@@ -6011,7 +6012,7 @@ def check_untyped_after_decorator(self, typ: Type, func: FuncDef) -> None:
         if not self.options.disallow_any_decorated or self.is_stub or self.current_node_deferred:
             return
 
-        if mypy.checkexpr.has_any_type(typ):
+        if has_any_type(typ):
             self.msg.untyped_decorated_function(typ, func)
 
     def check_async_with_item(

mypy/checkexpr.py

@@ -122,8 +122,10 @@
 from mypy.semanal_enum import ENUM_BASES
 from mypy.state import state
 from mypy.subtypes import (
+    common_type,
     covers_at_runtime,
     find_member,
+    has_any_type,
     is_same_type,
     is_subtype,
     merge_typevars_in_callables_by_name,
@@ -204,6 +206,7 @@
     has_recursive_types,
     has_type_vars,
     is_named_instance,
+    remove_dups,
     split_with_prefix_and_suffix,
 )
 from mypy.types_utils import (
@@ -228,6 +231,9 @@
 # see https://github.com/python/mypy/pull/5255#discussion_r196896335 for discussion.
 MAX_UNIONS: Final = 5
 
+# Maximum number or unique matched overload return types caused by Any
+# ambiguity where we try to find a precise fallback.
+MAX_PRECISE_OVERLOAD_FALLBACK: Final = 8
 
 # Types considered safe for comparisons with --strict-equality due to known behaviour of __eq__.
 # NOTE: All these types are subtypes of AbstractSet.
@@ -3097,11 +3103,17 @@ def infer_overload_return_type(
         if not matches:
             return None
         elif any_causes_overload_ambiguity(matches, return_types, arg_types, arg_kinds, arg_names):
+            return_types = remove_dups(return_types)
             # An argument of type or containing the type 'Any' caused ambiguity.
             # We try returning a precise type if we can. If not, we give up and just return 'Any'.
             if all_same_types(return_types):
                 self.chk.store_types(type_maps[0])
                 return return_types[0], inferred_types[0]
+            elif len(return_types) < MAX_PRECISE_OVERLOAD_FALLBACK and (
+                common := common_type(return_types)
+            ):
+                self.chk.store_types(type_maps[0])
+                return common, erase_type(inferred_types[0])
             elif all_same_types([erase_type(typ) for typ in return_types]):
                 self.chk.store_types(type_maps[0])
                 return erase_type(return_types[0]), erase_type(inferred_types[0])
@@ -6660,37 +6672,6 @@ def try_parse_as_type_expression(self, maybe_type_expr: Expression) -> Type | No
             return None
 
 
-def has_any_type(t: Type, ignore_in_type_obj: bool = False) -> bool:
-    """Whether t contains an Any type"""
-    return t.accept(HasAnyType(ignore_in_type_obj))
-
-
-class HasAnyType(types.BoolTypeQuery):
-    def __init__(self, ignore_in_type_obj: bool) -> None:
-        super().__init__(types.ANY_STRATEGY)
-        self.ignore_in_type_obj = ignore_in_type_obj
-
-    def visit_any(self, t: AnyType) -> bool:
-        return t.type_of_any != TypeOfAny.special_form  # special forms are not real Any types
-
-    def visit_callable_type(self, t: CallableType) -> bool:
-        if self.ignore_in_type_obj and t.is_type_obj():
-            return False
-        return super().visit_callable_type(t)
-
-    def visit_type_var(self, t: TypeVarType) -> bool:
-        default = [t.default] if t.has_default() else []
-        return self.query_types([t.upper_bound, *default] + t.values)
-
-    def visit_param_spec(self, t: ParamSpecType) -> bool:
-        default = [t.default] if t.has_default() else []
-        return self.query_types([t.upper_bound, *default, t.prefix])
-
-    def visit_type_var_tuple(self, t: TypeVarTupleType) -> bool:
-        default = [t.default] if t.has_default() else []
-        return self.query_types([t.upper_bound, *default])
-
-
 def has_coroutine_decorator(t: Type) -> bool:
     """Whether t came from a function decorated with `@coroutine`."""
     t = get_proper_type(t)

mypy/subtypes.py

@@ -36,6 +36,7 @@
 )
 from mypy.options import Options
 from mypy.state import state
+from mypy.type_visitor import ANY_STRATEGY, BoolTypeQuery
 from mypy.types import (
     MAX_PROTOCOL_DEPTH,
     MYPYC_NATIVE_INT_NAMES,
@@ -2537,6 +2538,55 @@ def erase_return_self_types(typ: Type, self_type: Instance) -> Type:
     return typ
 
 
+def common_type(types: list[Type]) -> Type | None:
+    """Return a type in the list that is both subtype and supertype of all other types.
+
+    If there are more than one such type, choose the one that has an Any component,
+    otherwise return None.
+    """
+    candidates = []
+    for candidate in types:
+        if all(is_equivalent(candidate, other) for other in types):
+            candidates.append(candidate)
+    if len(candidates) == 1:
+        return candidates[0]
+    candidates = [c for c in candidates if has_any_type(c)]
+    if len(candidates) == 1:
+        return candidates[0]
+    return None
+
+
+def has_any_type(t: Type, ignore_in_type_obj: bool = False) -> bool:
+    """Whether t contains an Any type"""
+    return t.accept(HasAnyType(ignore_in_type_obj))
+
+
+class HasAnyType(BoolTypeQuery):
+    def __init__(self, ignore_in_type_obj: bool) -> None:
+        super().__init__(ANY_STRATEGY)
+        self.ignore_in_type_obj = ignore_in_type_obj
+
+    def visit_any(self, t: AnyType) -> bool:
+        return t.type_of_any != TypeOfAny.special_form  # special forms are not real Any types
+
+    def visit_callable_type(self, t: CallableType) -> bool:
+        if self.ignore_in_type_obj and t.is_type_obj():
+            return False
+        return super().visit_callable_type(t)
+
+    def visit_type_var(self, t: TypeVarType) -> bool:
+        default = [t.default] if t.has_default() else []
+        return self.query_types([t.upper_bound, *default] + t.values)
+
+    def visit_param_spec(self, t: ParamSpecType) -> bool:
+        default = [t.default] if t.has_default() else []
+        return self.query_types([t.upper_bound, *default, t.prefix])
+
+    def visit_type_var_tuple(self, t: TypeVarTupleType) -> bool:
+        default = [t.default] if t.has_default() else []
+        return self.query_types([t.upper_bound, *default])
+
+
 def is_erased_instance(t: Instance) -> bool:
     """Is this an instance where all args are Any types?"""
     if not t.args:

mypy/suggestions.py

@@ -34,7 +34,6 @@
 
 from mypy.argmap import map_actuals_to_formals
 from mypy.build import Graph, State
-from mypy.checkexpr import has_any_type
 from mypy.find_sources import InvalidSourceList, SourceFinder
 from mypy.join import join_type_list
 from mypy.meet import meet_type_list
@@ -59,6 +58,7 @@
 from mypy.plugin import FunctionContext, MethodContext, Plugin
 from mypy.server.update import FineGrainedBuildManager
 from mypy.state import state
+from mypy.subtypes import has_any_type
 from mypy.traverser import TraverserVisitor
 from mypy.typeops import bind_self, make_simplified_union
 from mypy.types import (

test-data/unit/check-overloading.test

@@ -6932,3 +6932,23 @@ class B(A):
     def f(self, y: str) -> None: ...  # This is currently allowed (note different name)
     def f(self, *args, **kwargs) -> None: ...
 [builtins fixtures/tuple.pyi]
+
+[case testRespectPreciseOverloadFallbackIfPossible]
+from typing import overload, Any
+
+@overload
+def f(x: list[int]) -> list[tuple[int, ...]]: ...
+@overload
+def f(x: list[Any]) -> list[tuple[Any, ...]]: ...
+def f(x): pass
+
+a: Any
+la: list[Any]
+li: list[int]
+ls: list[str]
+
+reveal_type(f(a))  # N: Revealed type is "builtins.list[builtins.tuple[Any, ...]]"
+reveal_type(f(la))  # N: Revealed type is "builtins.list[builtins.tuple[Any, ...]]"
+reveal_type(f(li))  # N: Revealed type is "builtins.list[builtins.tuple[builtins.int, ...]]"
+reveal_type(f(ls))  # N: Revealed type is "builtins.list[builtins.tuple[Any, ...]]"
+[builtins fixtures/tuple.pyi]