Believe more __new__ return types

mypy/checkexpr.py

@@ -3277,11 +3277,11 @@ def analyze_ordinary_member_access(self, e: MemberExpr, is_lvalue: bool) -> Type
             member_type = analyze_member_access(
                 e.name,
                 original_type,
-                e,
-                is_lvalue,
-                False,
-                False,
-                self.msg,
+                context=e,
+                is_lvalue=is_lvalue,
+                is_super=False,
+                is_operator=False,
+                msg=self.msg,
                 original_type=original_type,
                 chk=self.chk,
                 in_literal_context=self.is_literal_context(),

mypy/checkmember.py

@@ -368,8 +368,27 @@ def validate_super_call(node: FuncBase, mx: MemberContext) -> None:
 def analyze_type_callable_member_access(name: str, typ: FunctionLike, mx: MemberContext) -> Type:
     # Class attribute.
     # TODO super?
-    ret_type = typ.items[0].ret_type
+    item = typ.items[0]
+    ret_type = item.ret_type
     assert isinstance(ret_type, ProperType)
+
+    # The following is a hack. mypy often represents types as CallableType, where the signature of
+    # CallableType is determined by __new__ or __init__ of the type (this logic is in
+    # type_object_type). Then if we ever need the TypeInfo or an instance of the type, we fish
+    # around for the return type, e.g. in CallableType.type_object. Unfortunately, this is incorrect
+    # if __new__ returns an unrelated type, but we can kind of salvage things here using
+    # CallableType.bound_args
+    self_type: Type = typ
+    if len(item.bound_args) == 1 and item.bound_args[0]:
+        bound_arg = get_proper_type(item.bound_args[0])
+        if isinstance(bound_arg, Instance) and isinstance(ret_type, Instance):
+            # Unfortunately, generic arguments have already been determined for us. We need these,
+            # see e.g. testGenericClassMethodUnboundOnClass. So just copy them over to our type.
+            # This does the wrong thing with custom __new__, see testNewReturnType15, but is
+            # no worse than previous behaviour.
+            ret_type = bound_arg.copy_modified(args=ret_type.args)
+            self_type = TypeType(ret_type)
+
     if isinstance(ret_type, TupleType):
         ret_type = tuple_fallback(ret_type)
     if isinstance(ret_type, TypedDictType):
@@ -391,7 +410,11 @@ def analyze_type_callable_member_access(name: str, typ: FunctionLike, mx: Member
             # See https://github.com/python/mypy/pull/1787 for more info.
             # TODO: do not rely on same type variables being present in all constructor overloads.
             result = analyze_class_attribute_access(
-                ret_type, name, mx, original_vars=typ.items[0].variables, mcs_fallback=typ.fallback
+                ret_type,
+                name,
+                mx.copy_modified(self_type=self_type),
+                original_vars=typ.items[0].variables,
+                mcs_fallback=typ.fallback,
             )
             if result:
                 return result

mypy/typeops.py

@@ -196,26 +196,41 @@ def class_callable(
     orig_self_type = get_proper_type(orig_self_type)
     default_ret_type = fill_typevars(info)
     explicit_type = init_ret_type if is_new else orig_self_type
-    if (
-        isinstance(explicit_type, (Instance, TupleType, UninhabitedType))
-        # We have to skip protocols, because it can be a subtype of a return type
-        # by accident. Like `Hashable` is a subtype of `object`. See #11799
-        and isinstance(default_ret_type, Instance)
-        and not default_ret_type.type.is_protocol
-        # Only use the declared return type from __new__ or declared self in __init__
-        # if it is actually returning a subtype of what we would return otherwise.
-        and is_subtype(explicit_type, default_ret_type, ignore_type_params=True)
-    ):
-        ret_type: Type = explicit_type
-    else:
-        ret_type = default_ret_type
+
+    ret_type: Type = default_ret_type
+    from_type_type = init_type.from_type_type
+    if isinstance(explicit_type, (Instance, TupleType, UninhabitedType)):
+        if is_new:
+            # For a long time, mypy didn't truly believe annotations on __new__
+            # This has led to some proliferation of code that depends on this, namely
+            # annotations on __new__ that hardcode the class in the return type.
+            # This can then cause problems for subclasses. Preserve the old behaviour in
+            # this case (although we should probably change it at some point)
+            # See testValueTypeWithNewInParentClass
+            # Also see testSelfTypeInGenericClassUsedFromAnotherGenericClass1
+            if not is_subtype(
+                default_ret_type, explicit_type, ignore_type_params=True
+            ) or is_subtype(explicit_type, default_ret_type, ignore_type_params=True):
+                ret_type = explicit_type
+                from_type_type = True
+        elif (
+            # We have to skip protocols, because it can be a subtype of a return type
+            # by accident. Like `Hashable` is a subtype of `object`. See #11799
+            isinstance(default_ret_type, Instance)
+            and not default_ret_type.type.is_protocol
+            # Only use the declared return type from declared self in __init__
+            # if it is actually returning a subtype of what we would return otherwise.
+            and is_subtype(explicit_type, default_ret_type, ignore_type_params=True)
+        ):
+            ret_type = explicit_type
 
     callable_type = init_type.copy_modified(
         ret_type=ret_type,
         fallback=type_type,
         name=None,
         variables=variables,
         special_sig=special_sig,
+        from_type_type=from_type_type,
     )
     c = callable_type.with_name(info.name)
     return c

mypy/types.py

@@ -1474,7 +1474,7 @@ def deserialize(cls, data: JsonDict | str) -> Instance:
     def copy_modified(
         self,
         *,
-        args: Bogus[list[Type]] = _dummy,
+        args: Bogus[Sequence[Type]] = _dummy,
         last_known_value: Bogus[LiteralType | None] = _dummy,
     ) -> Instance:
         new = Instance(

test-data/unit/check-classes.test

@@ -6928,7 +6928,7 @@ class A:
     def __new__(cls) -> int:  # E: Incompatible return type for "__new__" (returns "int", but must return a subtype of "A")
         pass
 
-reveal_type(A())  # N: Revealed type is "__main__.A"
+reveal_type(A())  # N: Revealed type is "builtins.int"
 
 [case testNewReturnType4]
 from typing import TypeVar, Type
@@ -7038,6 +7038,54 @@ class MyMetaClass(type):
 class MyClass(metaclass=MyMetaClass):
     pass
 
+[case testNewReturnType13]
+from typing import Protocol
+
+class Foo(Protocol):
+    def foo(self) -> str: ...
+
+class A:
+    def __new__(cls) -> Foo: ...  # E: Incompatible return type for "__new__" (returns "Foo", but must return a subtype of "A")
+
+reveal_type(A())  # N: Revealed type is "__main__.Foo"
+reveal_type(A().foo())  # N: Revealed type is "builtins.str"
+
+
+
+[case testNewReturnType14]
+from __future__ import annotations
+
+class A:
+    def __new__(cls) -> int: raise  # E: Incompatible return type for "__new__" (returns "int", but must return a subtype of "A")
+
+class B(A):
+    @classmethod
+    def foo(cls) -> int: raise
+
+reveal_type(B.foo())  # N: Revealed type is "builtins.int"
+[builtins fixtures/classmethod.pyi]
+
+[case testNewReturnType15]
+from typing import Generic, Type, TypeVar
+
+T = TypeVar("T")
+
+class A(Generic[T]):
+    def __new__(cls) -> B[int]: ...
+    @classmethod
+    def foo(cls: Type[T]) -> T: ...
+
+class B(A[T]): ...
+
+# These are incorrect, should be Any and str
+# See https://github.com/python/mypy/pull/16020
+reveal_type(B.foo())  # N: Revealed type is "builtins.int"
+reveal_type(B[str].foo())  # N: Revealed type is "builtins.int"
+
+class C(A[str]): ...
+
+reveal_type(C.foo())  # N: Revealed type is "builtins.str"
+[builtins fixtures/classmethod.pyi]
 
 [case testMetaclassPlaceholderNode]
 from sympy.assumptions import ManagedProperties