Do subtyping between self types with captured type in special way

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsTestGenerated.java

@@ -11765,6 +11765,12 @@ public class FirOldFrontendDiagnosticsTestGenerated extends AbstractFirDiagnosti
                 runTest("compiler/testData/diagnostics/tests/inference/capturedInProjectedFlexibleType.kt");
             }
 
+            @Test
+            @TestMetadata("capturedTypesInSelfType.kt")
+            public void testCapturedTypesInSelfType() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/inference/capturedTypesInSelfType.kt");
+            }
+
             @Test
             @TestMetadata("coerceFunctionLiteralToSuspend.kt")
             public void testCoerceFunctionLiteralToSuspend() throws Exception {

compiler/fir/cones/src/org/jetbrains/kotlin/fir/types/ConeTypes.kt

@@ -314,8 +314,8 @@ class ConeStubType(val variable: ConeTypeVariable, override val nullability: Con
     }
 }
 
-open class ConeTypeVariable(name: String) : TypeVariableMarker {
-    val typeConstructor = ConeTypeVariableTypeConstructor(name)
+open class ConeTypeVariable(name: String, originalTypeParameter: TypeParameterMarker? = null) : TypeVariableMarker {
+    val typeConstructor = ConeTypeVariableTypeConstructor(name, originalTypeParameter)
     val defaultType = ConeTypeVariableType(ConeNullability.NOT_NULL, typeConstructor)
 
     override fun toString(): String {
@@ -323,7 +323,10 @@ open class ConeTypeVariable(name: String) : TypeVariableMarker {
     }
 }
 
-class ConeTypeVariableTypeConstructor(val debugName: String) : ConeClassifierLookupTag(), TypeVariableTypeConstructorMarker {
+class ConeTypeVariableTypeConstructor(
+    val debugName: String,
+    val originalTypeParameter: TypeParameterMarker?
+) : ConeClassifierLookupTag(), TypeVariableTypeConstructorMarker {
     override val name: Name get() = Name.identifier(debugName)
 
     var isContainedInInvariantOrContravariantPositions: Boolean = false

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/inference/ConeTypeParameterBasedTypeVariable.kt

@@ -9,4 +9,4 @@ import org.jetbrains.kotlin.fir.symbols.impl.FirTypeParameterSymbol
 import org.jetbrains.kotlin.fir.types.ConeTypeVariable
 
 class ConeTypeParameterBasedTypeVariable(val typeParameterSymbol: FirTypeParameterSymbol) :
-    ConeTypeVariable(typeParameterSymbol.name.identifier)
+    ConeTypeVariable(typeParameterSymbol.name.identifier, typeParameterSymbol.toLookupTag())

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/types/ConeTypeContext.kt

@@ -47,6 +47,12 @@ interface ConeTypeContext : TypeSystemContext, TypeSystemOptimizationContext, Ty
         return classId.isLocal
     }
 
+    override val TypeVariableTypeConstructorMarker.typeParameter: TypeParameterMarker?
+        get() {
+            require(this is ConeTypeVariableTypeConstructor)
+            return this.originalTypeParameter
+        }
+
     override fun SimpleTypeMarker.possibleIntegerTypes(): Collection<KotlinTypeMarker> {
         return (this as? ConeIntegerLiteralType)?.possibleTypes ?: emptyList()
     }
@@ -274,6 +280,14 @@ interface ConeTypeContext : TypeSystemContext, TypeSystemOptimizationContext, Ty
         return this
     }
 
+    override fun TypeParameterMarker.doesFormSelfType(selfConstructor: TypeConstructorMarker): Boolean {
+        require(this is ConeTypeParameterLookupTag)
+        return this.typeParameterSymbol.fir.bounds.any { typeRef ->
+            typeRef.coneType.contains { it.typeConstructor() == this.getTypeConstructor() }
+                    && typeRef.coneType.typeConstructor() == selfConstructor
+        }
+    }
+
     override fun areEqualTypeConstructors(c1: TypeConstructorMarker, c2: TypeConstructorMarker): Boolean {
         if (c1 is ErrorTypeConstructor || c2 is ErrorTypeConstructor) return false
         return c1 == c2

compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/types/IrTypeSystemContext.kt

@@ -168,6 +168,28 @@ interface IrTypeSystemContext : TypeSystemContext, TypeSystemCommonSuperTypesCon
 
     override fun TypeParameterMarker.getTypeConstructor() = this as IrTypeParameterSymbol
 
+    private fun KotlinTypeMarker.containsTypeConstructor(constructor: TypeConstructorMarker): Boolean {
+        if (this.typeConstructor() == constructor) return true
+
+        for (i in 0 until this.argumentsCount()) {
+            val typeArgument = this.getArgument(i).takeIf { !it.isStarProjection() } ?: continue
+            if (typeArgument.getType().containsTypeConstructor(constructor)) return true
+        }
+
+        return false
+    }
+
+    override fun TypeParameterMarker.doesFormSelfType(selfConstructor: TypeConstructorMarker): Boolean {
+        for (i in 0 until this.upperBoundCount()) {
+            val upperBound = this.getUpperBound(i)
+            if (upperBound.containsTypeConstructor(selfConstructor) && upperBound.typeConstructor() == selfConstructor) {
+                return true
+            }
+        }
+
+        return false
+    }
+
     override fun areEqualTypeConstructors(c1: TypeConstructorMarker, c2: TypeConstructorMarker): Boolean =
         if (c1 is IrClassifierSymbol && c2 is IrClassifierSymbol) {
             FqNameEqualityChecker.areEqual(c1 , c2)
@@ -277,6 +299,9 @@ interface IrTypeSystemContext : TypeSystemContext, TypeSystemCommonSuperTypesCon
         return this.owner.classId?.isLocal == true
     }
 
+    override val TypeVariableTypeConstructorMarker.typeParameter: TypeParameterMarker?
+        get() = error("Type variables is unsupported in IR")
+
     override fun createFlexibleType(lowerBound: SimpleTypeMarker, upperBound: SimpleTypeMarker): KotlinTypeMarker {
         require(lowerBound.isNothing())
         require(upperBound is IrType && upperBound.isNullableAny())

compiler/resolution/src/org/jetbrains/kotlin/resolve/calls/inference/model/TypeVariable.kt

@@ -20,8 +20,6 @@ import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.descriptors.ClassifierDescriptor
 import org.jetbrains.kotlin.descriptors.TypeParameterDescriptor
 import org.jetbrains.kotlin.descriptors.annotations.Annotations
-import org.jetbrains.kotlin.resolve.calls.model.CallableReferenceKotlinCallArgument
-import org.jetbrains.kotlin.resolve.calls.model.LambdaKotlinCallArgument
 import org.jetbrains.kotlin.resolve.calls.model.PostponableKotlinCallArgument
 import org.jetbrains.kotlin.resolve.descriptorUtil.builtIns
 import org.jetbrains.kotlin.resolve.descriptorUtil.hasOnlyInputTypesAnnotation
@@ -36,7 +34,11 @@ import org.jetbrains.kotlin.types.model.TypeVariableTypeConstructorMarker
 import org.jetbrains.kotlin.types.refinement.TypeRefinement
 
 
-class TypeVariableTypeConstructor(private val builtIns: KotlinBuiltIns, val debugName: String) : TypeConstructor,
+class TypeVariableTypeConstructor(
+    private val builtIns: KotlinBuiltIns,
+    val debugName: String,
+    override val originalTypeParameter: TypeParameterDescriptor?
+) : TypeConstructor,
     NewTypeVariableConstructor, TypeVariableTypeConstructorMarker {
     override fun getParameters(): List<TypeParameterDescriptor> = emptyList()
     override fun getSupertypes(): Collection<KotlinType> = emptyList()
@@ -54,8 +56,12 @@ class TypeVariableTypeConstructor(private val builtIns: KotlinBuiltIns, val debu
     var isContainedInInvariantOrContravariantPositions: Boolean = false
 }
 
-sealed class NewTypeVariable(builtIns: KotlinBuiltIns, name: String) : TypeVariableMarker {
-    val freshTypeConstructor = TypeVariableTypeConstructor(builtIns, name)
+sealed class NewTypeVariable(
+    builtIns: KotlinBuiltIns,
+    name: String,
+    originalTypeParameter: TypeParameterDescriptor? = null
+) : TypeVariableMarker {
+    val freshTypeConstructor = TypeVariableTypeConstructor(builtIns, name, originalTypeParameter)
 
     // member scope is used if we have receiver with type TypeVariable(T)
     // todo add to member scope methods from supertypes for type variable
@@ -75,7 +81,7 @@ fun TypeConstructor.typeForTypeVariable(): SimpleType {
 
 class TypeVariableFromCallableDescriptor(
     val originalTypeParameter: TypeParameterDescriptor
-) : NewTypeVariable(originalTypeParameter.builtIns, originalTypeParameter.name.identifier) {
+) : NewTypeVariable(originalTypeParameter.builtIns, originalTypeParameter.name.identifier, originalTypeParameter) {
     override fun hasOnlyInputTypesAnnotation(): Boolean = originalTypeParameter.hasOnlyInputTypesAnnotation()
 }
 

compiler/testData/diagnostics/tests/inference/capturedTypesInSelfType.fir.kt

@@ -0,0 +1,8 @@
+// WITH_RUNTIME
+// !DIAGNOSTICS: -UNUSED_VARIABLE
+
+class Foo<T : Enum<T>>(val values: Array<T>)
+
+fun foo(x: Array<out Enum<*>>) {
+    val y = <!INAPPLICABLE_CANDIDATE!>Foo<!>(x)
+}

compiler/testData/diagnostics/tests/inference/capturedTypesInSelfType.kt

@@ -0,0 +1,8 @@
+// WITH_RUNTIME
+// !DIAGNOSTICS: -UNUSED_VARIABLE
+
+class Foo<T : Enum<T>>(val values: Array<T>)
+
+fun foo(x: Array<out Enum<*>>) {
+    val y = Foo(x)
+}

compiler/testData/diagnostics/tests/inference/capturedTypesInSelfType.txt

@@ -0,0 +1,11 @@
+package
+
+public fun foo(/*0*/ x: kotlin.Array<out kotlin.Enum<*>>): kotlin.Unit
+
+public final class Foo</*0*/ T : kotlin.Enum<T>> {
+    public constructor Foo</*0*/ T : kotlin.Enum<T>>(/*0*/ values: kotlin.Array<T>)
+    public final val values: kotlin.Array<T>
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/DiagnosticTestGenerated.java

@@ -11771,6 +11771,12 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
                 runTest("compiler/testData/diagnostics/tests/inference/capturedInProjectedFlexibleType.kt");
             }
 
+            @Test
+            @TestMetadata("capturedTypesInSelfType.kt")
+            public void testCapturedTypesInSelfType() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/inference/capturedTypesInSelfType.kt");
+            }
+
             @Test
             @TestMetadata("coerceFunctionLiteralToSuspend.kt")
             public void testCoerceFunctionLiteralToSuspend() throws Exception {

core/compiler.common/src/org/jetbrains/kotlin/types/AbstractTypeChecker.kt

@@ -360,6 +360,24 @@ object AbstractTypeChecker {
         }
     }
 
+    private fun TypeSystemContext.isTypeVariableAgainstStarProjectionForSelfType(
+        subArgumentType: KotlinTypeMarker,
+        superArgumentType: KotlinTypeMarker,
+        selfConstructor: TypeConstructorMarker
+    ): Boolean {
+        val simpleSubArgumentType = subArgumentType.asSimpleType()
+
+        if (simpleSubArgumentType !is CapturedTypeMarker || !simpleSubArgumentType.typeConstructor().projection().isStarProjection())
+            return false
+        // Only 'for subtyping' captured types are approximated before adding constraints (see ConstraintInjector.addNewIncorporatedConstraint)
+        // that can lead to adding problematic constraints like UPPER(Nothing) given by CapturedType(*) <: TypeVariable(A)
+        if (simpleSubArgumentType.captureStatus() != CaptureStatus.FOR_SUBTYPING) return false
+
+        val typeVariableConstructor = superArgumentType.typeConstructor() as? TypeVariableTypeConstructorMarker ?: return false
+
+        return typeVariableConstructor.typeParameter?.doesFormSelfType(selfConstructor) == true
+    }
+
     fun AbstractTypeCheckerContext.isSubtypeForSameConstructor(
         capturedSubArguments: TypeArgumentListMarker,
         superType: SimpleTypeMarker
@@ -379,6 +397,7 @@ object AbstractTypeChecker {
 
         for (index in 0 until parametersCount) {
             val superProjection = superType.getArgument(index) // todo error index
+
             if (superProjection.isStarProjection()) continue // A<B> <: A<*>
 
             val superArgumentType = superProjection.getType()
@@ -387,6 +406,19 @@ object AbstractTypeChecker {
                 it.getType()
             }
 
+            val isTypeVariableAgainstStarProjectionForSelfType =
+                isTypeVariableAgainstStarProjectionForSelfType(subArgumentType, superArgumentType, superTypeConstructor) ||
+                        isTypeVariableAgainstStarProjectionForSelfType(superArgumentType, subArgumentType, superTypeConstructor)
+
+            /*
+             * We don't check subtyping between types like CapturedType(*) and TypeVariable(E) if the corresponding type parameter forms self type, for instance, Enum<E: Enum<E>>.
+             * It can return false and produce unwanted constraints like UPPER(Nothing) (by CapturedType(*) <:> TypeVariable(E)) in the type inference context
+             * due to approximation captured types.
+             * Instead this type check we move on self-type level anyway: checking CapturedType(out Enum<*>) against TypeVariable(E).
+             * This subtyping can already be successful and not add unwanted constraints in the type inference context.
+             */
+            if (isTypeVariableAgainstStarProjectionForSelfType) continue
+
             val variance = effectiveVariance(superTypeConstructor.getParameter(index).getVariance(), superProjection.getVariance())
                 ?: return isErrorTypeEqualsToAnything // todo exception?
 
@@ -463,9 +495,39 @@ object AbstractTypeChecker {
             return superTypeConstructor.supertypes().all { isSubtypeOf(context, subType, it) }
         }
 
+        /*
+         * We handle cases like CapturedType(out Bar) <: Foo<CapturedType(out Bar)> separately here.
+         * If Foo is a self type i.g. Foo<E: Foo<E>>, then argument for E will certainly be subtype of Foo<same_argument_for_E>,
+         * so if CapturedType(out Bar) is the same as a type of Foo's argument and Foo is a self type, then subtyping should return true.
+         * If we don't handle this case separately, subtyping may not converge due to the nature of the capturing.
+         */
+        if (subType is CapturedTypeMarker) {
+            val typeParameter =
+                context.typeSystemContext.getTypeParameterForArgumentInBaseIfItEqualToTarget(baseType = superType, targetType = subType)
+            if (typeParameter != null && typeParameter.doesFormSelfType(superType.typeConstructor())) {
+                return true
+            }
+        }
+
         return null
     }
 
+    private fun TypeSystemContext.getTypeParameterForArgumentInBaseIfItEqualToTarget(
+        baseType: KotlinTypeMarker,
+        targetType: KotlinTypeMarker
+    ): TypeParameterMarker? {
+        for (i in 0 until baseType.argumentsCount()) {
+            val typeArgument = baseType.getArgument(i).takeIf { !it.isStarProjection() } ?: continue
+
+            if (typeArgument.getType() == targetType) {
+                return baseType.typeConstructor().getParameter(i)
+            }
+
+            getTypeParameterForArgumentInBaseIfItEqualToTarget(typeArgument.getType(), targetType)?.let { return it }
+        }
+
+        return null
+    }
 
     private fun collectAllSupertypesWithGivenTypeConstructor(
         context: AbstractTypeCheckerContext,

core/compiler.common/src/org/jetbrains/kotlin/types/model/TypeSystemContext.kt

@@ -306,10 +306,13 @@ interface TypeSystemContext : TypeSystemOptimizationContext {
     fun TypeConstructorMarker.isIntegerLiteralTypeConstructor(): Boolean
     fun TypeConstructorMarker.isLocalType(): Boolean
 
+    val TypeVariableTypeConstructorMarker.typeParameter: TypeParameterMarker?
+
     fun TypeParameterMarker.getVariance(): TypeVariance
     fun TypeParameterMarker.upperBoundCount(): Int
     fun TypeParameterMarker.getUpperBound(index: Int): KotlinTypeMarker
     fun TypeParameterMarker.getTypeConstructor(): TypeConstructorMarker
+    fun TypeParameterMarker.doesFormSelfType(selfConstructor: TypeConstructorMarker): Boolean
 
     fun areEqualTypeConstructors(c1: TypeConstructorMarker, c2: TypeConstructorMarker): Boolean
 

core/descriptors/src/org/jetbrains/kotlin/types/checker/ClassicTypeSystemContext.kt

@@ -44,6 +44,12 @@ interface ClassicTypeSystemContext : TypeSystemInferenceExtensionContext, TypeSy
         return declarationDescriptor?.classId?.isLocal == true
     }
 
+    override val TypeVariableTypeConstructorMarker.typeParameter: TypeParameterMarker?
+        get() {
+            require(this is NewTypeVariableConstructor, this::errorMessage)
+            return this.originalTypeParameter
+        }
+
     override fun SimpleTypeMarker.possibleIntegerTypes(): Collection<KotlinTypeMarker> {
         val typeConstructor = typeConstructor()
         require(typeConstructor is IntegerLiteralTypeConstructor, this::errorMessage)
@@ -212,6 +218,13 @@ interface ClassicTypeSystemContext : TypeSystemInferenceExtensionContext, TypeSy
         return this.typeConstructor
     }
 
+    override fun TypeParameterMarker.doesFormSelfType(selfConstructor: TypeConstructorMarker): Boolean {
+        require(this is TypeParameterDescriptor, this::errorMessage)
+        require(selfConstructor is TypeConstructor, this::errorMessage)
+
+        return doesTypeParameterFormSelfType(this, selfConstructor)
+    }
+
     override fun areEqualTypeConstructors(c1: TypeConstructorMarker, c2: TypeConstructorMarker): Boolean {
         require(c1 is TypeConstructor, c1::errorMessage)
         require(c2 is TypeConstructor, c2::errorMessage)

core/descriptors/src/org/jetbrains/kotlin/types/checker/utils.kt

@@ -17,9 +17,11 @@
 package org.jetbrains.kotlin.types.checker
 
 import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
+import org.jetbrains.kotlin.descriptors.TypeParameterDescriptor
 import org.jetbrains.kotlin.renderer.DescriptorRenderer
 import org.jetbrains.kotlin.resolve.calls.inference.wrapWithCapturingSubstitution
 import org.jetbrains.kotlin.types.*
+import org.jetbrains.kotlin.types.typeUtil.contains
 import org.jetbrains.kotlin.types.typesApproximation.approximateCapturedTypes
 import java.util.*
 
@@ -101,4 +103,11 @@ private fun TypeConstructor.debugInfo() = buildString {
     }
 }
 
-interface NewTypeVariableConstructor
+interface NewTypeVariableConstructor {
+    val originalTypeParameter: TypeParameterDescriptor?
+}
+
+fun doesTypeParameterFormSelfType(typeParameter: TypeParameterDescriptor, selfConstructor: TypeConstructor) =
+    typeParameter.upperBounds.any { upperBound ->
+        upperBound.contains { it.constructor == typeParameter.typeConstructor } && upperBound.constructor == selfConstructor
+    }