FIR2IR: convert recursive captured type properly

#KT-43346 Fixed

compiler/fir/fir2ir/src/org/jetbrains/kotlin/fir/backend/Fir2IrTypeConverter.kt

@@ -58,7 +58,11 @@ class Fir2IrTypeConverter(
         StandardClassIds.Char to irBuiltIns.charType
     )
 
+    private val capturedTypeCache = mutableMapOf<ConeCapturedType, IrType>()
+    private val capturedTypeStack = mutableSetOf<ConeCapturedType>()
+
     fun FirTypeRef.toIrType(typeContext: ConversionTypeContext = ConversionTypeContext.DEFAULT): IrType {
+        capturedTypeStack.clear()
         return when (this) {
             !is FirResolvedTypeRef -> createErrorType()
             !is FirImplicitBuiltinTypeRef -> type.toIrType(typeContext, annotations)
@@ -106,7 +110,16 @@ class Fir2IrTypeConverter(
                 upperBound.toIrType(typeContext)
             }
             is ConeCapturedType -> {
-                lowerType?.toIrType(typeContext) ?: constructor.supertypes!!.first().toIrType(typeContext)
+                if (capturedTypeStack.add(this)) {
+                    val irType = lowerType?.toIrType(typeContext) ?: constructor.supertypes!!.first().toIrType(typeContext)
+                    capturedTypeCache[this] = irType
+                    irType
+                } else {
+                    // Potentially recursive captured type, e.g., Recursive<R> where R : Recursive<R>, ...
+                    // That should have been handled during type argument conversion, though.
+                    // Or, simply repeated captured type, e.g., FunctionN<..., *, ..., *>, literally same captured types.
+                    capturedTypeCache[this] ?: createErrorType()
+                }
             }
             is ConeDefinitelyNotNullType -> {
                 original.toIrType(typeContext.definitelyNotNull())
@@ -132,12 +145,48 @@ class Fir2IrTypeConverter(
                 makeTypeProjection(irType, Variance.OUT_VARIANCE)
             }
             is ConeKotlinType -> {
-                val irType = toIrType(typeContext)
-                makeTypeProjection(irType, Variance.INVARIANT)
+                if (this is ConeCapturedType && capturedTypeStack.contains(this) && this.isRecursive(mutableSetOf())) {
+                    // Recursive captured type, e.g., Recursive<R> where R : Recursive<R>, ...
+                    // We can return * early here to avoid recursive type conversions.
+                    IrStarProjectionImpl
+                } else {
+                    val irType = toIrType(typeContext)
+                    makeTypeProjection(irType, Variance.INVARIANT)
+                }
             }
         }
     }
 
+    private fun ConeKotlinType.isRecursive(visited: MutableSet<ConeCapturedType>): Boolean =
+        when (this) {
+            is ConeLookupTagBasedType -> {
+                typeArguments.any {
+                    when (it) {
+                        is ConeKotlinType -> it.isRecursive(visited)
+                        is ConeKotlinTypeProjectionIn -> it.type.isRecursive(visited)
+                        is ConeKotlinTypeProjectionOut -> it.type.isRecursive(visited)
+                        else -> false
+                    }
+                }
+            }
+            is ConeFlexibleType -> {
+                lowerBound.isRecursive(visited) || upperBound.isRecursive(visited)
+            }
+            is ConeCapturedType -> {
+                if (visited.add(this)) {
+                    constructor.supertypes?.any { it.isRecursive(visited) } == true
+                } else
+                    true
+            }
+            is ConeDefinitelyNotNullType -> {
+                original.isRecursive(visited)
+            }
+            is ConeIntersectionType -> {
+                intersectedTypes.any { it.isRecursive(visited) }
+            }
+            else -> false
+        }
+
     private fun getArrayClassSymbol(classId: ClassId?): IrClassSymbol? {
         val primitiveId = StandardClassIds.elementTypeByPrimitiveArrayType[classId] ?: return null
         val irType = classIdToTypeMap[primitiveId]

compiler/fir/fir2ir/tests/org/jetbrains/kotlin/fir/Fir2IrTextTestGenerated.java

@@ -1807,6 +1807,11 @@ public class Fir2IrTextTestGenerated extends AbstractFir2IrTextTest {
             runTest("compiler/testData/ir/irText/firProblems/putIfAbsent.kt");
         }
 
+        @TestMetadata("recursiveCapturedTypeInPropertyReference.kt")
+        public void testRecursiveCapturedTypeInPropertyReference() throws Exception {
+            runTest("compiler/testData/ir/irText/firProblems/recursiveCapturedTypeInPropertyReference.kt");
+        }
+
         @TestMetadata("SameJavaFieldReferences.kt")
         public void testSameJavaFieldReferences() throws Exception {
             runTest("compiler/testData/ir/irText/firProblems/SameJavaFieldReferences.kt");

compiler/testData/ir/irText/firProblems/recursiveCapturedTypeInPropertyReference.fir.txt

@@ -0,0 +1,68 @@
+FILE fqName:<root> fileName:/recursiveCapturedTypeInPropertyReference.kt
+  CLASS INTERFACE name:Something modality:ABSTRACT visibility:public superTypes:[kotlin.Any]
+    $this: VALUE_PARAMETER INSTANCE_RECEIVER name:<this> type:<root>.Something
+    FUN FAKE_OVERRIDE name:equals visibility:public modality:OPEN <> ($this:kotlin.Any, other:kotlin.Any?) returnType:kotlin.Boolean [fake_override,operator]
+      overridden:
+        public open fun equals (other: kotlin.Any?): kotlin.Boolean [operator] declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+      VALUE_PARAMETER name:other index:0 type:kotlin.Any?
+    FUN FAKE_OVERRIDE name:hashCode visibility:public modality:OPEN <> ($this:kotlin.Any) returnType:kotlin.Int [fake_override]
+      overridden:
+        public open fun hashCode (): kotlin.Int declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+    FUN FAKE_OVERRIDE name:toString visibility:public modality:OPEN <> ($this:kotlin.Any) returnType:kotlin.String [fake_override]
+      overridden:
+        public open fun toString (): kotlin.String declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+  CLASS INTERFACE name:Recursive modality:ABSTRACT visibility:public superTypes:[kotlin.Any]
+    $this: VALUE_PARAMETER INSTANCE_RECEIVER name:<this> type:<root>.Recursive<R of <root>.Recursive>
+    TYPE_PARAMETER name:R index:0 variance: superTypes:[<root>.Recursive<R of <root>.Recursive>; <root>.Something]
+    PROPERTY name:symbol visibility:public modality:ABSTRACT [val]
+      FUN DEFAULT_PROPERTY_ACCESSOR name:<get-symbol> visibility:public modality:ABSTRACT <> ($this:<root>.Recursive<R of <root>.Recursive>) returnType:<root>.AbstractSymbol<R of <root>.Recursive>
+        correspondingProperty: PROPERTY name:symbol visibility:public modality:ABSTRACT [val]
+        $this: VALUE_PARAMETER name:<this> type:<root>.Recursive<R of <root>.Recursive>
+    FUN FAKE_OVERRIDE name:equals visibility:public modality:OPEN <> ($this:kotlin.Any, other:kotlin.Any?) returnType:kotlin.Boolean [fake_override,operator]
+      overridden:
+        public open fun equals (other: kotlin.Any?): kotlin.Boolean [operator] declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+      VALUE_PARAMETER name:other index:0 type:kotlin.Any?
+    FUN FAKE_OVERRIDE name:hashCode visibility:public modality:OPEN <> ($this:kotlin.Any) returnType:kotlin.Int [fake_override]
+      overridden:
+        public open fun hashCode (): kotlin.Int declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+    FUN FAKE_OVERRIDE name:toString visibility:public modality:OPEN <> ($this:kotlin.Any) returnType:kotlin.String [fake_override]
+      overridden:
+        public open fun toString (): kotlin.String declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+  CLASS CLASS name:AbstractSymbol modality:ABSTRACT visibility:public superTypes:[kotlin.Any]
+    $this: VALUE_PARAMETER INSTANCE_RECEIVER name:<this> type:<root>.AbstractSymbol<E of <root>.AbstractSymbol>
+    TYPE_PARAMETER name:E index:0 variance: superTypes:[<root>.Recursive<E of <root>.AbstractSymbol>; <root>.Something]
+    CONSTRUCTOR visibility:public <> () returnType:<root>.AbstractSymbol<E of <root>.AbstractSymbol> [primary]
+      BLOCK_BODY
+        DELEGATING_CONSTRUCTOR_CALL 'public constructor <init> () [primary] declared in kotlin.Any'
+        INSTANCE_INITIALIZER_CALL classDescriptor='CLASS CLASS name:AbstractSymbol modality:ABSTRACT visibility:public superTypes:[kotlin.Any]'
+    FUN name:foo visibility:public modality:FINAL <> ($this:<root>.AbstractSymbol<E of <root>.AbstractSymbol>, list:kotlin.collections.List<kotlin.Any>) returnType:kotlin.Unit
+      $this: VALUE_PARAMETER name:<this> type:<root>.AbstractSymbol<E of <root>.AbstractSymbol>
+      VALUE_PARAMETER name:list index:0 type:kotlin.collections.List<kotlin.Any>
+      BLOCK_BODY
+        VAR name:result type:kotlin.collections.List<<root>.AbstractSymbol<out <root>.Recursive<*>>> [val]
+          CALL 'public final fun map <T, R> (transform: kotlin.Function1<T of kotlin.collections.map, R of kotlin.collections.map>): kotlin.collections.List<R of kotlin.collections.map> [inline] declared in kotlin.collections' type=kotlin.collections.List<<root>.AbstractSymbol<out <root>.Recursive<*>>> origin=null
+            <T>: <root>.Recursive<*>
+            <R>: <root>.AbstractSymbol<out <root>.Recursive<*>>
+            $receiver: CALL 'public final fun filterIsInstance <R> (): kotlin.collections.List<R of kotlin.collections.filterIsInstance> [inline] declared in kotlin.collections' type=kotlin.collections.List<<root>.Recursive<*>> origin=null
+              <R>: <root>.Recursive<*>
+              $receiver: GET_VAR 'list: kotlin.collections.List<kotlin.Any> declared in <root>.AbstractSymbol.foo' type=kotlin.collections.List<kotlin.Any> origin=null
+            transform: PROPERTY_REFERENCE 'public abstract symbol: <root>.AbstractSymbol<R of <root>.Recursive> [val]' field=null getter='public abstract fun <get-symbol> (): <root>.AbstractSymbol<R of <root>.Recursive> declared in <root>.Recursive' setter=null type=kotlin.reflect.KProperty1<<root>.Recursive<*>, <root>.AbstractSymbol<<root>.Recursive<*>>> origin=null
+    FUN FAKE_OVERRIDE name:equals visibility:public modality:OPEN <> ($this:kotlin.Any, other:kotlin.Any?) returnType:kotlin.Boolean [fake_override,operator]
+      overridden:
+        public open fun equals (other: kotlin.Any?): kotlin.Boolean [operator] declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+      VALUE_PARAMETER name:other index:0 type:kotlin.Any?
+    FUN FAKE_OVERRIDE name:hashCode visibility:public modality:OPEN <> ($this:kotlin.Any) returnType:kotlin.Int [fake_override]
+      overridden:
+        public open fun hashCode (): kotlin.Int declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+    FUN FAKE_OVERRIDE name:toString visibility:public modality:OPEN <> ($this:kotlin.Any) returnType:kotlin.String [fake_override]
+      overridden:
+        public open fun toString (): kotlin.String declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any

compiler/testData/ir/irText/firProblems/recursiveCapturedTypeInPropertyReference.kt

@@ -0,0 +1,14 @@
+// WITH_RUNTIME
+// FULL_JDK
+
+interface Something
+
+interface Recursive<R> where R : Recursive<R>, R : Something {
+    val symbol: AbstractSymbol<R>
+}
+
+abstract class AbstractSymbol<E> where E : Recursive<E>, E : Something {
+    fun foo(list: List<Any>) {
+        val result = list.filterIsInstance<Recursive<*>>().map(Recursive<*>::symbol)
+    }
+}

compiler/testData/ir/irText/firProblems/recursiveCapturedTypeInPropertyReference.txt

@@ -0,0 +1,68 @@
+FILE fqName:<root> fileName:/recursiveCapturedTypeInPropertyReference.kt
+  CLASS INTERFACE name:Something modality:ABSTRACT visibility:public superTypes:[kotlin.Any]
+    $this: VALUE_PARAMETER INSTANCE_RECEIVER name:<this> type:<root>.Something
+    FUN FAKE_OVERRIDE name:equals visibility:public modality:OPEN <> ($this:kotlin.Any, other:kotlin.Any?) returnType:kotlin.Boolean [fake_override,operator]
+      overridden:
+        public open fun equals (other: kotlin.Any?): kotlin.Boolean [operator] declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+      VALUE_PARAMETER name:other index:0 type:kotlin.Any?
+    FUN FAKE_OVERRIDE name:hashCode visibility:public modality:OPEN <> ($this:kotlin.Any) returnType:kotlin.Int [fake_override]
+      overridden:
+        public open fun hashCode (): kotlin.Int declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+    FUN FAKE_OVERRIDE name:toString visibility:public modality:OPEN <> ($this:kotlin.Any) returnType:kotlin.String [fake_override]
+      overridden:
+        public open fun toString (): kotlin.String declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+  CLASS INTERFACE name:Recursive modality:ABSTRACT visibility:public superTypes:[kotlin.Any]
+    $this: VALUE_PARAMETER INSTANCE_RECEIVER name:<this> type:<root>.Recursive<R of <root>.Recursive>
+    TYPE_PARAMETER name:R index:0 variance: superTypes:[<root>.Recursive<R of <root>.Recursive>; <root>.Something]
+    PROPERTY name:symbol visibility:public modality:ABSTRACT [val]
+      FUN DEFAULT_PROPERTY_ACCESSOR name:<get-symbol> visibility:public modality:ABSTRACT <> ($this:<root>.Recursive<R of <root>.Recursive>) returnType:<root>.AbstractSymbol<R of <root>.Recursive>
+        correspondingProperty: PROPERTY name:symbol visibility:public modality:ABSTRACT [val]
+        $this: VALUE_PARAMETER name:<this> type:<root>.Recursive<R of <root>.Recursive>
+    FUN FAKE_OVERRIDE name:equals visibility:public modality:OPEN <> ($this:kotlin.Any, other:kotlin.Any?) returnType:kotlin.Boolean [fake_override,operator]
+      overridden:
+        public open fun equals (other: kotlin.Any?): kotlin.Boolean [operator] declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+      VALUE_PARAMETER name:other index:0 type:kotlin.Any?
+    FUN FAKE_OVERRIDE name:hashCode visibility:public modality:OPEN <> ($this:kotlin.Any) returnType:kotlin.Int [fake_override]
+      overridden:
+        public open fun hashCode (): kotlin.Int declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+    FUN FAKE_OVERRIDE name:toString visibility:public modality:OPEN <> ($this:kotlin.Any) returnType:kotlin.String [fake_override]
+      overridden:
+        public open fun toString (): kotlin.String declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+  CLASS CLASS name:AbstractSymbol modality:ABSTRACT visibility:public superTypes:[kotlin.Any]
+    $this: VALUE_PARAMETER INSTANCE_RECEIVER name:<this> type:<root>.AbstractSymbol<E of <root>.AbstractSymbol>
+    TYPE_PARAMETER name:E index:0 variance: superTypes:[<root>.Recursive<E of <root>.AbstractSymbol>; <root>.Something]
+    CONSTRUCTOR visibility:public <> () returnType:<root>.AbstractSymbol<E of <root>.AbstractSymbol> [primary]
+      BLOCK_BODY
+        DELEGATING_CONSTRUCTOR_CALL 'public constructor <init> () [primary] declared in kotlin.Any'
+        INSTANCE_INITIALIZER_CALL classDescriptor='CLASS CLASS name:AbstractSymbol modality:ABSTRACT visibility:public superTypes:[kotlin.Any]'
+    FUN name:foo visibility:public modality:FINAL <> ($this:<root>.AbstractSymbol<E of <root>.AbstractSymbol>, list:kotlin.collections.List<kotlin.Any>) returnType:kotlin.Unit
+      $this: VALUE_PARAMETER name:<this> type:<root>.AbstractSymbol<E of <root>.AbstractSymbol>
+      VALUE_PARAMETER name:list index:0 type:kotlin.collections.List<kotlin.Any>
+      BLOCK_BODY
+        VAR name:result type:kotlin.collections.List<<root>.AbstractSymbol<out <root>.Recursive<*>>> [val]
+          CALL 'public final fun map <T, R> (transform: kotlin.Function1<T of kotlin.collections.map, R of kotlin.collections.map>): kotlin.collections.List<R of kotlin.collections.map> [inline] declared in kotlin.collections' type=kotlin.collections.List<<root>.AbstractSymbol<out <root>.Recursive<*>>> origin=null
+            <T>: <root>.Recursive<*>
+            <R>: <root>.AbstractSymbol<out <root>.Recursive<*>>
+            $receiver: CALL 'public final fun filterIsInstance <R> (): kotlin.collections.List<@[NoInfer] R of kotlin.collections.filterIsInstance> [inline] declared in kotlin.collections' type=kotlin.collections.List<@[NoInfer] <root>.Recursive<*>> origin=null
+              <R>: <root>.Recursive<*>
+              $receiver: GET_VAR 'list: kotlin.collections.List<kotlin.Any> declared in <root>.AbstractSymbol.foo' type=kotlin.collections.List<kotlin.Any> origin=null
+            transform: PROPERTY_REFERENCE 'public abstract symbol: <root>.AbstractSymbol<R of <root>.Recursive> [val]' field=null getter='public abstract fun <get-symbol> (): <root>.AbstractSymbol<R of <root>.Recursive> declared in <root>.Recursive' setter=null type=kotlin.reflect.KProperty1<<root>.Recursive<*>, <root>.AbstractSymbol<out <root>.Recursive<*>>> origin=null
+    FUN FAKE_OVERRIDE name:equals visibility:public modality:OPEN <> ($this:kotlin.Any, other:kotlin.Any?) returnType:kotlin.Boolean [fake_override,operator]
+      overridden:
+        public open fun equals (other: kotlin.Any?): kotlin.Boolean [operator] declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+      VALUE_PARAMETER name:other index:0 type:kotlin.Any?
+    FUN FAKE_OVERRIDE name:hashCode visibility:public modality:OPEN <> ($this:kotlin.Any) returnType:kotlin.Int [fake_override]
+      overridden:
+        public open fun hashCode (): kotlin.Int declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any
+    FUN FAKE_OVERRIDE name:toString visibility:public modality:OPEN <> ($this:kotlin.Any) returnType:kotlin.String [fake_override]
+      overridden:
+        public open fun toString (): kotlin.String declared in kotlin.Any
+      $this: VALUE_PARAMETER name:<this> type:kotlin.Any

compiler/tests/org/jetbrains/kotlin/ir/IrTextTestCaseGenerated.java

@@ -1806,6 +1806,11 @@ public class IrTextTestCaseGenerated extends AbstractIrTextTestCase {
             runTest("compiler/testData/ir/irText/firProblems/putIfAbsent.kt");
         }
 
+        @TestMetadata("recursiveCapturedTypeInPropertyReference.kt")
+        public void testRecursiveCapturedTypeInPropertyReference() throws Exception {
+            runTest("compiler/testData/ir/irText/firProblems/recursiveCapturedTypeInPropertyReference.kt");
+        }
+
         @TestMetadata("SameJavaFieldReferences.kt")
         public void testSameJavaFieldReferences() throws Exception {
             runTest("compiler/testData/ir/irText/firProblems/SameJavaFieldReferences.kt");