K2: handle recursive types properly in approximation

This commit is intended to deal with inconsistency in K1/K2
star projection handling.
K1 star projection includes a 'type' property.
This type from a star projection can be used for relevant
functions / properties return types,
and already includes some approximation for recursive generics.
In contrast, K2 star projection is an object,
and return types of relevant functions / properties are
represented as captured types.
To prevent recursion in them in recursive generic case,
this commit includes additional replacement of their type arguments.
See more details in added comments.

#KT-65057 Fixed

compiler/resolution.common/src/org/jetbrains/kotlin/types/AbstractTypeApproximator.kt

@@ -318,8 +318,32 @@ abstract class AbstractTypeApproximator(
             }
         }
         val baseSubType = type.lowerType() ?: nothingType()
+        // This replacement is important for resolving the code like below in K2.
+        //     fun bar(y: FieldOrRef<*>) = y.field
+        //     interface FieldOrRef<FF : AbstractField<FF>> { val field: FF }
+        //     abstract class AbstractField<out F : AbstractField<F>>
+        // During resolving the value parameter y type, K1 also builds a type for a star projection *.
+        // See fun TypeParameterDescriptor.starProjectionType(): KotlinType and fun buildStarProjectionTypeByTypeParameters.
+        // Thanks to it, K1 builds the star projection type as AbstractField<*> and no other approximation is needed.
+        //
+        // In turn, K2 never makes such a thing (K2 star projection has no associated type).
+        // Instead, it resolves y.field as CapturedType(*) C (see usage one line below                      v ),
+        // and the constructor of this captured type has a star projection and a supertype of AbstractField(C)
+        // Without this replacement, the type approximator currently cannot handle such a situation properly
+        // and builds AbstractField<AbstractField<AbstractField<Any?>>>.
+        // The check it == type here is intended to find a recursion inside a captured type.
+        // A similar replacement for baseSubType looks unnecessary, no hits in the tests.
+        val replacedSuperType = if (isK2 && toSuper && baseSuperType.getArguments().any { it == type }) {
+            baseSuperType.replaceArguments {
+                // It's possible to use the stub here, because K2 star projection is an object and
+                // in fact this parameter is never used
+                if (it != type) it else createStarProjection(TypeParameterMarkerStubForK2StarProjection)
+            }
+        } else baseSuperType
 
-        val approximatedSuperType by lazy(LazyThreadSafetyMode.NONE) { approximateToSuperType(baseSuperType, conf, depth) }
+        val approximatedSuperType by lazy(LazyThreadSafetyMode.NONE) {
+            approximateToSuperType(replacedSuperType, conf, depth)
+        }
         val approximatedSubType by lazy(LazyThreadSafetyMode.NONE) { approximateToSubType(baseSubType, conf, depth) }
 
         if (!conf.capturedType(ctx, type)) {
@@ -336,7 +360,7 @@ abstract class AbstractTypeApproximator(
                 return null
             }
         }
-        val baseResult = if (toSuper) approximatedSuperType ?: baseSuperType else approximatedSubType ?: baseSubType
+        val baseResult = if (toSuper) approximatedSuperType ?: replacedSuperType else approximatedSubType ?: baseSubType
 
         // C = in Int, Int <: C => Int? <: C?
         // C = out Number, C <: Number => C? <: Number?
@@ -359,6 +383,8 @@ abstract class AbstractTypeApproximator(
         }
     }
 
+    private object TypeParameterMarkerStubForK2StarProjection : TypeParameterMarker
+
     private fun approximateSimpleToSuperType(type: SimpleTypeMarker, conf: TypeApproximatorConfiguration, depth: Int) =
         approximateTo(type, conf, toSuper = true, depth = depth)
 

compiler/testData/diagnostics/tests/inference/recursiveTypes/complexTypeUnwrapping.fir.txt

@@ -16,7 +16,7 @@ FILE: complexTypeUnwrapping.kt
             public get(): R|RE|
 
     }
-    public final fun foo(x: R|ElementOrRef<*, *>|): R|AbstractElement<out AbstractElement<out AbstractElement<out AbstractElement<out AbstractElement<out kotlin/Any?, out kotlin/Any?>, out AbstractField<kotlin/Any?>>, out AbstractField<AbstractField<kotlin/Any?>>>, out AbstractField<AbstractField<AbstractField<kotlin/Any?>>>>, out AbstractField<AbstractField<AbstractField<AbstractField<kotlin/Any?>>>>>| {
+    public final fun foo(x: R|ElementOrRef<*, *>|): R|AbstractElement<*, out AbstractField<*>>| {
         ^foo R|<local>/x|.R|SubstitutionOverride</ElementOrRef.element: R|CapturedType(*)|>|
     }
     public abstract interface FieldOrRef<FF : R|AbstractField<FF>|> : R|kotlin/Any| {
@@ -24,6 +24,6 @@ FILE: complexTypeUnwrapping.kt
             public get(): R|FF|
 
     }
-    public final fun bar(y: R|FieldOrRef<*>|): R|AbstractField<AbstractField<AbstractField<AbstractField<AbstractField<kotlin/Any?>>>>>| {
+    public final fun bar(y: R|FieldOrRef<*>|): R|AbstractField<*>| {
         ^bar R|<local>/y|.R|SubstitutionOverride</FieldOrRef.field: R|CapturedType(*)|>|
     }

compiler/testData/diagnostics/tests/inference/recursiveTypes/selfTypes/enabledInferenceOnSelfTypes/overridingJKCases.fir.kt

@@ -37,9 +37,9 @@ fun javaInterfaceTest(a: A1<*>, b: B1<*>, c: C1<*>) {
     val x = a.foo()
     <!DEBUG_INFO_EXPRESSION_TYPE("A1<*>")!>x<!>
     val y = b.foo()
-    <!DEBUG_INFO_EXPRESSION_TYPE("I<out I<out I<out I<out I<out kotlin.Any?>>>>>")!>y<!>
+    <!DEBUG_INFO_EXPRESSION_TYPE("I<*>")!>y<!>
     val z = c.foo()
-    <!DEBUG_INFO_EXPRESSION_TYPE("I<out I<out I<out I<out I<out kotlin.Any?>>>>>..I<out I<out I<out I<out I<out kotlin.Any?>>>>>?!")!>z<!>
+    <!DEBUG_INFO_EXPRESSION_TYPE("I<*>..I<*>?!")!>z<!>
 }
 
 fun javaClassTest(a: A2<*>, b: B2<*>, c: C2<*>, d: D2<*>) {

compiler/testData/diagnostics/tests/inference/recursiveTypes/selfTypes/enabledInferenceOnSelfTypes/whereAndWithCases.fir.kt

@@ -1,39 +0,0 @@
-// ISSUE: KT-59012
-// !LANGUAGE: +TypeInferenceOnCallsWithSelfTypes
-
-interface I1<G : I1<G>> {
-    fun <T : G> foo() : T
-    fun <T : G> bar(vararg args: T)
-}
-
-interface I2<in G : I2<G>> {
-    fun <T : G> foo() : T
-    fun <T : G> bar(vararg args: T)
-}
-
-interface I3<G> where G : I3<G> {
-    fun <T> foo() : T where T : G
-    fun <T> bar(vararg args: T) where T : G
-}
-
-fun test(a: I2<*>, b: I3<*>) {
-    val x = a.foo()
-    <!DEBUG_INFO_EXPRESSION_TYPE("I2<kotlin.Nothing>")!>x<!>
-    a.bar()
-    val y = b.foo()
-    <!DEBUG_INFO_EXPRESSION_TYPE("I3<*>")!>y<!>
-    b.bar()
-}
-
-fun withTest(a: I1<*>, b: I3<*>) {
-    with(a) {
-        val x = foo()
-        <!DEBUG_INFO_EXPRESSION_TYPE("I1<*>")!>x<!>
-        bar()
-    }
-    with(b) {
-        val y = foo()
-        <!DEBUG_INFO_EXPRESSION_TYPE("I3<*>")!>y<!>
-        bar()
-    }
-}

compiler/testData/diagnostics/tests/inference/recursiveTypes/selfTypes/enabledInferenceOnSelfTypes/whereAndWithCases.kt

@@ -1,3 +1,4 @@
+// FIR_IDENTICAL
 // ISSUE: KT-59012
 // !LANGUAGE: +TypeInferenceOnCallsWithSelfTypes
 

compiler/testData/ir/irText/firProblems/ClashResolutionDescriptor.fir.ir.txt

@@ -155,7 +155,7 @@ FILE fqName:<root> fileName:/ClashResolutionDescriptor.kt
                     TYPE_OP type=kotlin.collections.Collection<<root>.ComponentDescriptor>? origin=SAFE_CAST typeOperand=kotlin.collections.Collection<<root>.ComponentDescriptor>
                       CALL 'public open fun get (p0: @[EnhancedNullability] K of java.util.HashMap): V of java.util.HashMap? declared in java.util.HashMap' type=kotlin.Any? origin=GET_ARRAY_ELEMENT
                         $this: CALL 'private final fun <get-registrationMap> (): java.util.HashMap<java.lang.reflect.Type, kotlin.Any> declared in <root>' type=java.util.HashMap<java.lang.reflect.Type, kotlin.Any> origin=GET_PROPERTY
-                        p0: CALL 'public final fun <get-applicableTo> (): java.lang.Class<E of <root>.PlatformExtensionsClashResolver> declared in <root>.PlatformExtensionsClashResolver' type=java.lang.Class<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out kotlin.Any?>>>>>> origin=GET_PROPERTY
+                        p0: CALL 'public final fun <get-applicableTo> (): java.lang.Class<E of <root>.PlatformExtensionsClashResolver> declared in <root>.PlatformExtensionsClashResolver' type=java.lang.Class<out <root>.PlatformSpecificExtension<*>> origin=GET_PROPERTY
                           $this: GET_VAR 'val resolver: <root>.PlatformExtensionsClashResolver<*> declared in <root>.resolveClashesIfAny' type=<root>.PlatformExtensionsClashResolver<*> origin=null
                   WHEN type=kotlin.collections.Collection<<root>.ComponentDescriptor> origin=null
                     BRANCH
@@ -166,9 +166,9 @@ FILE fqName:<root> fileName:/ClashResolutionDescriptor.kt
                     BRANCH
                       if: CONST Boolean type=kotlin.Boolean value=true
                       then: GET_VAR 'val tmp_1: kotlin.collections.Collection<<root>.ComponentDescriptor>? declared in <root>.resolveClashesIfAny' type=kotlin.collections.Collection<<root>.ComponentDescriptor>? origin=null
-              VAR name:substituteDescriptor type:<root>.ClashResolutionDescriptor<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out kotlin.Any?>>>>>> [val]
+              VAR name:substituteDescriptor type:<root>.ClashResolutionDescriptor<out <root>.PlatformSpecificExtension<*>> [val]
-                CONSTRUCTOR_CALL 'public constructor <init> (container: <root>.ComponentContainer, resolver: <root>.PlatformExtensionsClashResolver<E of <root>.ClashResolutionDescriptor>, clashedComponents: kotlin.collections.List<<root>.ComponentDescriptor>) declared in <root>.ClashResolutionDescriptor' type=<root>.ClashResolutionDescriptor<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out kotlin.Any?>>>>>> origin=null
+                CONSTRUCTOR_CALL 'public constructor <init> (container: <root>.ComponentContainer, resolver: <root>.PlatformExtensionsClashResolver<E of <root>.ClashResolutionDescriptor>, clashedComponents: kotlin.collections.List<<root>.ComponentDescriptor>) declared in <root>.ClashResolutionDescriptor' type=<root>.ClashResolutionDescriptor<out <root>.PlatformSpecificExtension<*>> origin=null
-                  <class: E>: <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out <root>.PlatformSpecificExtension<out kotlin.Any?>>>>>
+                  <class: E>: <root>.PlatformSpecificExtension<*>
                   container: GET_VAR 'container: <root>.ComponentContainer declared in <root>.resolveClashesIfAny' type=<root>.ComponentContainer origin=null
                   resolver: GET_VAR 'val resolver: <root>.PlatformExtensionsClashResolver<*> declared in <root>.resolveClashesIfAny' type=<root>.PlatformExtensionsClashResolver<*> origin=null
                   clashedComponents: CALL 'public final fun toList <T> (): kotlin.collections.List<T of kotlin.collections.toList> declared in kotlin.collections' type=kotlin.collections.List<<root>.ComponentDescriptor> origin=null

compiler/testData/ir/irText/firProblems/ClashResolutionDescriptor.fir.kt.txt

@@ -54,7 +54,7 @@ fun resolveClashesIfAny(container: ComponentContainer, clashResolvers: List<Plat
             else -> tmp_1
           }
         }
-        val substituteDescriptor: ClashResolutionDescriptor<out PlatformSpecificExtension<out PlatformSpecificExtension<out PlatformSpecificExtension<out PlatformSpecificExtension<out PlatformSpecificExtension<out Any?>>>>>> = ClashResolutionDescriptor<PlatformSpecificExtension<out PlatformSpecificExtension<out PlatformSpecificExtension<out PlatformSpecificExtension<out PlatformSpecificExtension<out Any?>>>>>>(container = container, resolver = resolver, clashedComponents = clashedComponents.toList<ComponentDescriptor>())
+        val substituteDescriptor: ClashResolutionDescriptor<out PlatformSpecificExtension<*>> = ClashResolutionDescriptor<PlatformSpecificExtension<*>>(container = container, resolver = resolver, clashedComponents = clashedComponents.toList<ComponentDescriptor>())
       }
     }
   }