Do subtyping between self types with captured type in special way

core/compiler.common/src/org/jetbrains/kotlin/types/AbstractTypeChecker.kt

@@ -360,6 +360,24 @@ object AbstractTypeChecker {
         }
     }
 
+    private fun TypeSystemContext.isTypeVariableAgainstStarProjectionForSelfType(
+        subArgumentType: KotlinTypeMarker,
+        superArgumentType: KotlinTypeMarker,
+        selfConstructor: TypeConstructorMarker
+    ): Boolean {
+        val simpleSubArgumentType = subArgumentType.asSimpleType()
+
+        if (simpleSubArgumentType !is CapturedTypeMarker || !simpleSubArgumentType.typeConstructor().projection().isStarProjection())
+            return false
+        // Only 'for subtyping' captured types are approximated before adding constraints (see ConstraintInjector.addNewIncorporatedConstraint)
+        // that can lead to adding problematic constraints like UPPER(Nothing) given by CapturedType(*) <: TypeVariable(A)
+        if (simpleSubArgumentType.captureStatus() != CaptureStatus.FOR_SUBTYPING) return false
+
+        val typeVariableConstructor = superArgumentType.typeConstructor() as? TypeVariableTypeConstructorMarker ?: return false
+
+        return typeVariableConstructor.typeParameter?.doesFormSelfType(selfConstructor) == true
+    }
+
     fun AbstractTypeCheckerContext.isSubtypeForSameConstructor(
         capturedSubArguments: TypeArgumentListMarker,
         superType: SimpleTypeMarker
@@ -379,6 +397,7 @@ object AbstractTypeChecker {
 
         for (index in 0 until parametersCount) {
             val superProjection = superType.getArgument(index) // todo error index
+
             if (superProjection.isStarProjection()) continue // A<B> <: A<*>
 
             val superArgumentType = superProjection.getType()
@@ -387,6 +406,19 @@ object AbstractTypeChecker {
                 it.getType()
             }
 
+            val isTypeVariableAgainstStarProjectionForSelfType =
+                isTypeVariableAgainstStarProjectionForSelfType(subArgumentType, superArgumentType, superTypeConstructor) ||
+                        isTypeVariableAgainstStarProjectionForSelfType(superArgumentType, subArgumentType, superTypeConstructor)
+
+            /*
+             * We don't check subtyping between types like CapturedType(*) and TypeVariable(E) if the corresponding type parameter forms self type, for instance, Enum<E: Enum<E>>.
+             * It can return false and produce unwanted constraints like UPPER(Nothing) (by CapturedType(*) <:> TypeVariable(E)) in the type inference context
+             * due to approximation captured types.
+             * Instead this type check we move on self-type level anyway: checking CapturedType(out Enum<*>) against TypeVariable(E).
+             * This subtyping can already be successful and not add unwanted constraints in the type inference context.
+             */
+            if (isTypeVariableAgainstStarProjectionForSelfType) continue
+
             val variance = effectiveVariance(superTypeConstructor.getParameter(index).getVariance(), superProjection.getVariance())
                 ?: return isErrorTypeEqualsToAnything // todo exception?
 
@@ -463,9 +495,39 @@ object AbstractTypeChecker {
             return superTypeConstructor.supertypes().all { isSubtypeOf(context, subType, it) }
         }
 
+        /*
+         * We handle cases like CapturedType(out Bar) <: Foo<CapturedType(out Bar)> separately here.
+         * If Foo is a self type i.g. Foo<E: Foo<E>>, then argument for E will certainly be subtype of Foo<same_argument_for_E>,
+         * so if CapturedType(out Bar) is the same as a type of Foo's argument and Foo is a self type, then subtyping should return true.
+         * If we don't handle this case separately, subtyping may not converge due to the nature of the capturing.
+         */
+        if (subType is CapturedTypeMarker) {
+            val typeParameter =
+                context.typeSystemContext.getTypeParameterForArgumentInBaseIfItEqualToTarget(baseType = superType, targetType = subType)
+            if (typeParameter != null && typeParameter.doesFormSelfType(superType.typeConstructor())) {
+                return true
+            }
+        }
+
         return null
     }
 
+    private fun TypeSystemContext.getTypeParameterForArgumentInBaseIfItEqualToTarget(
+        baseType: KotlinTypeMarker,
+        targetType: KotlinTypeMarker
+    ): TypeParameterMarker? {
+        for (i in 0 until baseType.argumentsCount()) {
+            val typeArgument = baseType.getArgument(i).takeIf { !it.isStarProjection() } ?: continue
+
+            if (typeArgument.getType() == targetType) {
+                return baseType.typeConstructor().getParameter(i)
+            }
+
+            getTypeParameterForArgumentInBaseIfItEqualToTarget(typeArgument.getType(), targetType)?.let { return it }
+        }
+
+        return null
+    }
 
     private fun collectAllSupertypesWithGivenTypeConstructor(
         context: AbstractTypeCheckerContext,

core/compiler.common/src/org/jetbrains/kotlin/types/model/TypeSystemContext.kt

@@ -306,10 +306,13 @@ interface TypeSystemContext : TypeSystemOptimizationContext {
     fun TypeConstructorMarker.isIntegerLiteralTypeConstructor(): Boolean
     fun TypeConstructorMarker.isLocalType(): Boolean
 
+    val TypeVariableTypeConstructorMarker.typeParameter: TypeParameterMarker?
+
     fun TypeParameterMarker.getVariance(): TypeVariance
     fun TypeParameterMarker.upperBoundCount(): Int
     fun TypeParameterMarker.getUpperBound(index: Int): KotlinTypeMarker
     fun TypeParameterMarker.getTypeConstructor(): TypeConstructorMarker
+    fun TypeParameterMarker.doesFormSelfType(selfConstructor: TypeConstructorMarker): Boolean
 
     fun areEqualTypeConstructors(c1: TypeConstructorMarker, c2: TypeConstructorMarker): Boolean
 

core/descriptors/src/org/jetbrains/kotlin/types/checker/ClassicTypeSystemContext.kt

@@ -44,6 +44,12 @@ interface ClassicTypeSystemContext : TypeSystemInferenceExtensionContext, TypeSy
         return declarationDescriptor?.classId?.isLocal == true
     }
 
+    override val TypeVariableTypeConstructorMarker.typeParameter: TypeParameterMarker?
+        get() {
+            require(this is NewTypeVariableConstructor, this::errorMessage)
+            return this.originalTypeParameter
+        }
+
     override fun SimpleTypeMarker.possibleIntegerTypes(): Collection<KotlinTypeMarker> {
         val typeConstructor = typeConstructor()
         require(typeConstructor is IntegerLiteralTypeConstructor, this::errorMessage)
@@ -212,6 +218,13 @@ interface ClassicTypeSystemContext : TypeSystemInferenceExtensionContext, TypeSy
         return this.typeConstructor
     }
 
+    override fun TypeParameterMarker.doesFormSelfType(selfConstructor: TypeConstructorMarker): Boolean {
+        require(this is TypeParameterDescriptor, this::errorMessage)
+        require(selfConstructor is TypeConstructor, this::errorMessage)
+
+        return doesTypeParameterFormSelfType(this, selfConstructor)
+    }
+
     override fun areEqualTypeConstructors(c1: TypeConstructorMarker, c2: TypeConstructorMarker): Boolean {
         require(c1 is TypeConstructor, c1::errorMessage)
         require(c2 is TypeConstructor, c2::errorMessage)

core/descriptors/src/org/jetbrains/kotlin/types/checker/utils.kt

@@ -17,9 +17,11 @@
 package org.jetbrains.kotlin.types.checker
 
 import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
+import org.jetbrains.kotlin.descriptors.TypeParameterDescriptor
 import org.jetbrains.kotlin.renderer.DescriptorRenderer
 import org.jetbrains.kotlin.resolve.calls.inference.wrapWithCapturingSubstitution
 import org.jetbrains.kotlin.types.*
+import org.jetbrains.kotlin.types.typeUtil.contains
 import org.jetbrains.kotlin.types.typesApproximation.approximateCapturedTypes
 import java.util.*
 
@@ -101,4 +103,11 @@ private fun TypeConstructor.debugInfo() = buildString {
     }
 }
 
-interface NewTypeVariableConstructor
+interface NewTypeVariableConstructor {
+    val originalTypeParameter: TypeParameterDescriptor?
+}
+
+fun doesTypeParameterFormSelfType(typeParameter: TypeParameterDescriptor, selfConstructor: TypeConstructor) =
+    typeParameter.upperBounds.any { upperBound ->
+        upperBound.contains { it.constructor == typeParameter.typeConstructor } && upperBound.constructor == selfConstructor
+    }