Prevent recursion on erasion of raw types with interdependent type parameters

^KT-47480 Fixed

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsTestGenerated.java

@@ -21436,6 +21436,12 @@ public class FirOldFrontendDiagnosticsTestGenerated extends AbstractFirDiagnosti
                     runTest("compiler/testData/diagnostics/tests/platformTypes/rawTypes/interClassesRecursion.kt");
                 }
 
+                @Test
+                @TestMetadata("intermediateRecursion.kt")
+                public void testIntermediateRecursion() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/platformTypes/rawTypes/intermediateRecursion.kt");
+                }
+
                 @Test
                 @TestMetadata("nonGenericRawMember.kt")
                 public void testNonGenericRawMember() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsWithLightTreeTestGenerated.java

@@ -21436,6 +21436,12 @@ public class FirOldFrontendDiagnosticsWithLightTreeTestGenerated extends Abstrac
                     runTest("compiler/testData/diagnostics/tests/platformTypes/rawTypes/interClassesRecursion.kt");
                 }
 
+                @Test
+                @TestMetadata("intermediateRecursion.kt")
+                public void testIntermediateRecursion() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/platformTypes/rawTypes/intermediateRecursion.kt");
+                }
+
                 @Test
                 @TestMetadata("nonGenericRawMember.kt")
                 public void testNonGenericRawMember() throws Exception {

compiler/testData/diagnostics/tests/platformTypes/rawTypes/intermediateRecursion.kt

@@ -0,0 +1,14 @@
+// FIR_IDENTICAL
+// FILE: Boo.java
+public class Boo<N> {}
+
+// FILE: Foo.java
+public class Foo<T extends Boo<K>, K extends Boo<X>, X extends Boo<K>> {
+    T test2() { return null; }
+    static Foo test1() { return null; }
+}
+
+// FILE: main.kt
+fun main() {
+    val x = Foo.test1().test2()
+}

compiler/testData/diagnostics/tests/platformTypes/rawTypes/intermediateRecursion.txt

@@ -0,0 +1,21 @@
+package
+
+public fun main(): kotlin.Unit
+
+public open class Boo</*0*/ N : kotlin.Any!> {
+    public constructor Boo</*0*/ N : kotlin.Any!>()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public open class Foo</*0*/ T : Boo<K!>!, /*1*/ K : Boo<X!>!, /*2*/ X : Boo<K!>!> {
+    public constructor Foo</*0*/ T : Boo<K!>!, /*1*/ K : Boo<X!>!, /*2*/ X : Boo<K!>!>()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public/*package*/ open fun test2(): T!
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+
+    // Static members
+    public/*package*/ open fun test1(): (Foo<(raw) Boo<Boo<Boo<*>!>!>!, (raw) Boo<Boo<*>!>!, (raw) Boo<Boo<*>!>!>..Foo<out Boo<out Boo<out Boo<*>!>!>!, out Boo<out Boo<*>!>!, out Boo<out Boo<*>!>!>?)
+}

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/DiagnosticTestGenerated.java

@@ -21442,6 +21442,12 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
                     runTest("compiler/testData/diagnostics/tests/platformTypes/rawTypes/interClassesRecursion.kt");
                 }
 
+                @Test
+                @TestMetadata("intermediateRecursion.kt")
+                public void testIntermediateRecursion() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/platformTypes/rawTypes/intermediateRecursion.kt");
+                }
+
                 @Test
                 @TestMetadata("nonGenericRawMember.kt")
                 public void testNonGenericRawMember() throws Exception {

core/descriptors.jvm/src/org/jetbrains/kotlin/load/java/lazy/types/JavaTypeResolver.kt

@@ -213,7 +213,7 @@ class JavaTypeResolver(
                  * but it's wrong because Foo<*> isn't subtype of Foo<Foo<*>> in accordance with declared upper bound of Foo.
                  * So we should create Foo<*> in this case (CapturedType(*) is really subtype of Foo<CapturedType(*)>).
                  */
-                if (hasTypeParameterRecursiveBounds(parameter, selfConstructor = null, attr.upperBoundOfTypeParameter))
+                if (hasTypeParameterRecursiveBounds(parameter, selfConstructor = null, attr.visitedTypeParameters))
                     return@map StarProjectionImpl(parameter)
 
                 // Some activity for preventing recursion in cases like `class A<T extends A, F extends T>`
@@ -309,11 +309,12 @@ data class JavaTypeAttributes(
     val howThisTypeIsUsed: TypeUsage,
     val flexibility: JavaTypeFlexibility = INFLEXIBLE,
     val isForAnnotationParameter: Boolean = false,
-    // Current type is upper bound of this type parameter
+    // we use it to prevent happening a recursion while compute type parameter's upper bounds
-    val upperBoundOfTypeParameter: TypeParameterDescriptor? = null
+    val visitedTypeParameters: Set<TypeParameterDescriptor>? = null
 ) {
     fun withFlexibility(flexibility: JavaTypeFlexibility) = copy(flexibility = flexibility)
-    fun withTypeParameter(upperBoundOfTypeParameter: TypeParameterDescriptor) = copy(upperBoundOfTypeParameter = upperBoundOfTypeParameter)
+    fun withNewVisitedTypeParameter(typeParameter: TypeParameterDescriptor) =
+        copy(visitedTypeParameters = if (visitedTypeParameters != null) visitedTypeParameters + typeParameter else setOf(typeParameter))
 }
 
 enum class JavaTypeFlexibility {
@@ -328,7 +329,7 @@ fun TypeUsage.toAttributes(
 ) = JavaTypeAttributes(
     this,
     isForAnnotationParameter = isForAnnotationParameter,
-    upperBoundOfTypeParameter = upperBoundForTypeParameter
+    visitedTypeParameters = upperBoundForTypeParameter?.let(::setOf)
 )
 
 // Definition:
@@ -343,9 +344,9 @@ internal fun TypeParameterDescriptor.getErasedUpperBound(
     typeAttr: JavaTypeAttributes,
     defaultValue: (() -> KotlinType) = { ErrorUtils.createErrorType("Can't compute erased upper bound of type parameter `$this`") }
 ): KotlinType {
-    if (this === typeAttr.upperBoundOfTypeParameter) return defaultValue()
+    val visitedTypeParameters = typeAttr.visitedTypeParameters
 
-    val newTypeAttr = if (typeAttr.upperBoundOfTypeParameter == null) typeAttr.withTypeParameter(this) else typeAttr
+    if (visitedTypeParameters != null && this in visitedTypeParameters) return defaultValue()
 
     /*
      * We should do erasure of containing type parameters with their erasure to avoid creating inconsistent types.
@@ -353,13 +354,13 @@ internal fun TypeParameterDescriptor.getErasedUpperBound(
      * but it's wrong type: projection(*) != projection(Any).
      * So we should substitute erasure of the corresponding type parameter: `Foo<Foo<Any>, Any>` or `Foo<Foo<*>, *>`.
      */
-    val erasedUpperBounds = defaultType.extractTypeParametersFromUpperBounds(typeAttr.upperBoundOfTypeParameter).associate {
+    val erasedUpperBounds = defaultType.extractTypeParametersFromUpperBounds(visitedTypeParameters).associate {
-        val boundProjection = if (it !== typeAttr.upperBoundOfTypeParameter) {
+        val boundProjection = if (visitedTypeParameters == null || it !in visitedTypeParameters) {
             RawSubstitution.computeProjection(
                 it,
                 // if erasure happens due to invalid arguments number, use star projections instead
                 if (isRaw) typeAttr else typeAttr.withFlexibility(INFLEXIBLE),
-                if (it !== typeAttr.upperBoundOfTypeParameter) it.getErasedUpperBound(isRaw, newTypeAttr) else it.starProjectionType()
+                it.getErasedUpperBound(isRaw, typeAttr.withNewVisitedTypeParameter(this))
             )
         } else makeStarProjection(it, typeAttr)
 
@@ -374,21 +375,21 @@ internal fun TypeParameterDescriptor.getErasedUpperBound(
             erasedUpperBoundsSubstitutor,
             erasedUpperBounds,
             OUT_VARIANCE,
-            typeAttr.upperBoundOfTypeParameter
+            typeAttr.visitedTypeParameters
         )
     }
 
-    val stopAt = typeAttr.upperBoundOfTypeParameter ?: this
+    val stopAt = typeAttr.visitedTypeParameters ?: setOf(this)
     var current = firstUpperBound.constructor.declarationDescriptor as TypeParameterDescriptor
 
-    while (current != stopAt) {
+    while (current !in stopAt) {
         val nextUpperBound = current.upperBounds.first()
         if (nextUpperBound.constructor.declarationDescriptor is ClassDescriptor) {
             return nextUpperBound.replaceArgumentsWithStarProjectionOrMapped(
                 erasedUpperBoundsSubstitutor,
                 erasedUpperBounds,
                 OUT_VARIANCE,
-                typeAttr.upperBoundOfTypeParameter
+                typeAttr.visitedTypeParameters
             )
         }
 

core/descriptors/src/org/jetbrains/kotlin/types/TypeUtils.kt

@@ -185,13 +185,13 @@ fun KotlinType.contains(predicate: (UnwrappedType) -> Boolean) = TypeUtils.conta
 fun KotlinType.replaceArgumentsWithStarProjections() = replaceArgumentsWith(::StarProjectionImpl)
 fun KotlinType.replaceArgumentsWithNothing() = replaceArgumentsWith { it.builtIns.nothingType.asTypeProjection() }
 
-fun KotlinType.extractTypeParametersFromUpperBounds(upperBoundOfTypeParameter: TypeParameterDescriptor?): Set<TypeParameterDescriptor> =
+fun KotlinType.extractTypeParametersFromUpperBounds(visitedTypeParameters: Set<TypeParameterDescriptor>?): Set<TypeParameterDescriptor> =
-    mutableSetOf<TypeParameterDescriptor>().also { extractTypeParametersFromUpperBounds(this, it, upperBoundOfTypeParameter) }
+    mutableSetOf<TypeParameterDescriptor>().also { extractTypeParametersFromUpperBounds(this, it, visitedTypeParameters) }
 
 private fun KotlinType.extractTypeParametersFromUpperBounds(
     baseType: KotlinType,
     to: MutableSet<TypeParameterDescriptor>,
-    upperBoundOfTypeParameter: TypeParameterDescriptor?
+    visitedTypeParameters: Set<TypeParameterDescriptor>?
 ) {
     val declarationDescriptor = constructor.declarationDescriptor
 
@@ -200,16 +200,17 @@ private fun KotlinType.extractTypeParametersFromUpperBounds(
             to += declarationDescriptor
         } else {
             for (upperBound in declarationDescriptor.upperBounds) {
-                upperBound.extractTypeParametersFromUpperBounds(baseType, to, upperBoundOfTypeParameter)
+                upperBound.extractTypeParametersFromUpperBounds(baseType, to, visitedTypeParameters)
             }
         }
     } else {
         val typeParameters = (constructor.declarationDescriptor as? ClassifierDescriptorWithTypeParameters)?.declaredTypeParameters
         for ((i, argument) in arguments.withIndex()) {
             val typeParameter = typeParameters?.get(i)
-            if (argument.isStarProjection || (typeParameter != null && typeParameter == upperBoundOfTypeParameter)) continue
+            val isTypeParameterVisited = typeParameter != null && visitedTypeParameters != null && typeParameter in visitedTypeParameters
+            if (isTypeParameterVisited || argument.isStarProjection) continue
             if (argument.type.constructor.declarationDescriptor in to || argument.type.constructor == baseType.constructor) continue
-            argument.type.extractTypeParametersFromUpperBounds(baseType, to, upperBoundOfTypeParameter)
+            argument.type.extractTypeParametersFromUpperBounds(baseType, to, visitedTypeParameters)
         }
     }
 }
@@ -217,24 +218,26 @@ private fun KotlinType.extractTypeParametersFromUpperBounds(
 fun hasTypeParameterRecursiveBounds(
     typeParameter: TypeParameterDescriptor,
     selfConstructor: TypeConstructor? = null,
-    upperBoundOfTypeParameter: TypeParameterDescriptor? = null
+    visitedTypeParameters: Set<TypeParameterDescriptor>? = null
 ): Boolean =
     typeParameter.upperBounds.any { upperBound ->
-        upperBound.containsSelfTypeParameter(typeParameter.defaultType.constructor, upperBoundOfTypeParameter)
+        upperBound.containsSelfTypeParameter(typeParameter.defaultType.constructor, visitedTypeParameters)
                 && (selfConstructor == null || upperBound.constructor == selfConstructor)
     }
 
 private fun KotlinType.containsSelfTypeParameter(
     baseConstructor: TypeConstructor,
-    upperBoundOfTypeParameter: TypeParameterDescriptor?
+    visitedTypeParameters: Set<TypeParameterDescriptor>?
 ): Boolean {
     if (this.constructor == baseConstructor) return true
 
     val typeParameters = (constructor.declarationDescriptor as? ClassifierDescriptorWithTypeParameters)?.declaredTypeParameters
     return arguments.withIndex().any { (i, argument) ->
         val typeParameter = typeParameters?.getOrNull(i)
-        if ((typeParameter != null && typeParameter == upperBoundOfTypeParameter) || argument.isStarProjection) return@any false
+        val isTypeParameterVisited =
-        argument.type.containsSelfTypeParameter(baseConstructor, upperBoundOfTypeParameter)
+            typeParameter != null && visitedTypeParameters != null && typeParameter in visitedTypeParameters
+        if (isTypeParameterVisited || argument.isStarProjection) return@any false
+        argument.type.containsSelfTypeParameter(baseConstructor, visitedTypeParameters)
     }
 }
 
@@ -242,11 +245,12 @@ fun KotlinType.replaceArgumentsWithStarProjectionOrMapped(
     substitutor: TypeSubstitutor,
     substitutionMap: Map<TypeConstructor, TypeProjection>,
     variance: Variance,
-    upperBoundOfTypeParameter: TypeParameterDescriptor?
+    visitedTypeParameters: Set<TypeParameterDescriptor>?
 ) =
     replaceArgumentsWith { typeParameterDescriptor ->
         val argument = arguments.getOrNull(typeParameterDescriptor.index)
-        if (typeParameterDescriptor != upperBoundOfTypeParameter && argument != null && argument.type.constructor in substitutionMap) {
+        val isTypeParameterVisited = visitedTypeParameters != null && typeParameterDescriptor in visitedTypeParameters
+        if (!isTypeParameterVisited && argument != null && argument.type.constructor in substitutionMap) {
             argument
         } else StarProjectionImpl(typeParameterDescriptor)
     }.let { substitutor.safeSubstitute(it, variance) }