[FIR] Improve approximation of captured types

- Handle flexible types in recursion check
- Handle intersected supertypes separately
- Make check when not to approximate captured types in type argument
  position more fine-grained.
  Only apply it to case when the captured type is replaced by a star
  projection.
  All other cases are handled by recursive calls to
  approximateCapturedType

#KT-65377 Fixed

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/DiagnosticCompilerTestFE10TestdataTestGenerated.java

@@ -18059,6 +18059,12 @@ public class DiagnosticCompilerTestFE10TestdataTestGenerated extends AbstractDia
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/dontCheckNewCapturedTypeSpecificChecksForOldOnes.kt");
                 }
 
+                @Test
+                @TestMetadata("expectedTypeHasCapturedStarArgument.kt")
+                public void testExpectedTypeHasCapturedStarArgument() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/expectedTypeHasCapturedStarArgument.kt");
+                }
+
                 @Test
                 @TestMetadata("expectedTypeMismatchWithInVariance.kt")
                 public void testExpectedTypeMismatchWithInVariance() throws Exception {

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirPreresolvedReversedDiagnosticCompilerFE10TestDataTestGenerated.java

@@ -18059,6 +18059,12 @@ public class LLFirPreresolvedReversedDiagnosticCompilerFE10TestDataTestGenerated
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/dontCheckNewCapturedTypeSpecificChecksForOldOnes.kt");
                 }
 
+                @Test
+                @TestMetadata("expectedTypeHasCapturedStarArgument.kt")
+                public void testExpectedTypeHasCapturedStarArgument() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/expectedTypeHasCapturedStarArgument.kt");
+                }
+
                 @Test
                 @TestMetadata("expectedTypeMismatchWithInVariance.kt")
                 public void testExpectedTypeMismatchWithInVariance() throws Exception {

compiler/fir/analysis-tests/testData/resolve/types/castToBareType.fir.txt

@@ -16,6 +16,6 @@ FILE: castToBareType.kt
     public final fun foo(firAdaptee: R|FirFunction<*>|): R|kotlin/Unit| {
     }
     public final fun test(symbol: R|AbstractFirBasedSymbol<*>|): R|kotlin/Unit| {
-        lval firAdaptee: R|FirFunction<FirSymbolOwner<out it(FirSymbolOwner<out it(FirSymbolOwner<out it(FirSymbolOwner<out kotlin/Any?> & FirDeclaration)> & FirDeclaration)> & FirDeclaration)>>| = (R|<local>/symbol|.R|SubstitutionOverride</AbstractFirBasedSymbol.fir: R|CapturedType(*)|>| as R|FirFunction<FirSymbolOwner<out it(FirSymbolOwner<out it(FirSymbolOwner<out it(FirSymbolOwner<out kotlin/Any?> & FirDeclaration)> & FirDeclaration)> & FirDeclaration)>>|)
+        lval firAdaptee: R|FirFunction<*>| = (R|<local>/symbol|.R|SubstitutionOverride</AbstractFirBasedSymbol.fir: R|CapturedType(*)|>| as R|FirFunction<*>|)
         R|/foo|(R|<local>/firAdaptee|)
     }

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirLightTreeOldFrontendDiagnosticsTestGenerated.java

@@ -18053,6 +18053,12 @@ public class FirLightTreeOldFrontendDiagnosticsTestGenerated extends AbstractFir
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/dontCheckNewCapturedTypeSpecificChecksForOldOnes.kt");
                 }
 
+                @Test
+                @TestMetadata("expectedTypeHasCapturedStarArgument.kt")
+                public void testExpectedTypeHasCapturedStarArgument() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/expectedTypeHasCapturedStarArgument.kt");
+                }
+
                 @Test
                 @TestMetadata("expectedTypeMismatchWithInVariance.kt")
                 public void testExpectedTypeMismatchWithInVariance() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirPsiOldFrontendDiagnosticsTestGenerated.java

@@ -18059,6 +18059,12 @@ public class FirPsiOldFrontendDiagnosticsTestGenerated extends AbstractFirPsiDia
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/dontCheckNewCapturedTypeSpecificChecksForOldOnes.kt");
                 }
 
+                @Test
+                @TestMetadata("expectedTypeHasCapturedStarArgument.kt")
+                public void testExpectedTypeHasCapturedStarArgument() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/expectedTypeHasCapturedStarArgument.kt");
+                }
+
                 @Test
                 @TestMetadata("expectedTypeMismatchWithInVariance.kt")
                 public void testExpectedTypeMismatchWithInVariance() throws Exception {

compiler/resolution.common/src/org/jetbrains/kotlin/types/AbstractTypeApproximator.kt

@@ -8,7 +8,6 @@ package org.jetbrains.kotlin.types
 import org.jetbrains.kotlin.config.LanguageFeature
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.resolve.calls.NewCommonSuperTypeCalculator.commonSuperType
-import org.jetbrains.kotlin.resolve.calls.inference.hasRecursiveTypeParametersWithGivenSelfType
 import org.jetbrains.kotlin.types.model.*
 import org.jetbrains.kotlin.utils.addToStdlib.runIf
 import java.util.concurrent.ConcurrentHashMap
@@ -289,10 +288,45 @@ abstract class AbstractTypeApproximator(
         toSuper: Boolean,
         depth: Int
     ): KotlinTypeMarker? {
+        fun KotlinTypeMarker.replaceRecursionWithStarProjection(capturedType: CapturedTypeMarker, toSuper: Boolean): KotlinTypeMarker {
+            // This replacement is important for resolving the code like below in K2.
+            //     fun bar(y: FieldOrRef<*>) = y.field
+            //     interface FieldOrRef<FF : AbstractField<FF>> { val field: FF }
+            //     abstract class AbstractField<out F : AbstractField<F>>
+            // During resolving the value parameter y type, K1 also builds a type for a star projection *.
+            // See fun TypeParameterDescriptor.starProjectionType(): KotlinType and fun buildStarProjectionTypeByTypeParameters.
+            // Thanks to it, K1 builds the star projection type as AbstractField<*> and no other approximation is needed.
+            //
+            // In turn, K2 never makes such a thing (K2 star projection has no associated type).
+            // Instead, it resolves y.field as CapturedType(*) C (see usage one line below),
+            // and the constructor of this captured type has a star projection and a supertype of `AbstractField<C>`.
+            //
+            // Without this replacement, the type approximator currently cannot handle such a situation properly
+            // and builds AbstractField<AbstractField<AbstractField<Any?>>>.
+            // The check it == type here is intended to find a recursion inside a captured type.
+            // A similar replacement for baseSubType looks unnecessary, no hits in the tests.
+
+            fun TypeArgumentMarker.unwrapForComparison(): KotlinTypeMarker? {
+                if (this.isStarProjection()) return null
+                return getType().lowerBoundIfFlexible().originalIfDefinitelyNotNullable()
+            }
+
+            return if (isK2 && toSuper && getArguments().any { it.unwrapForComparison() == capturedType }) {
+                replaceArguments {
+                    when {
+                        it.unwrapForComparison() != capturedType -> it
+                        // It's possible to use the stub here, because K2 star projection is an object and
+                        // in fact this parameter is never used
+                        else -> createStarProjection(TypeParameterMarkerStubForK2StarProjection)
+                    }
+                }
+            } else this
+        }
+
         val supertypes = capturedType.typeConstructor().supertypes()
         val baseSuperType = when (supertypes.size) {
             0 -> nullableAnyType() // Let C = in Int, then superType for C and C? is Any?
-            1 -> supertypes.single()
+            1 -> supertypes.single().replaceRecursionWithStarProjection(capturedType, toSuper)
 
             // Consider the following example:
             // A.getA()::class.java, where `getA()` returns some class from Java
@@ -313,40 +347,14 @@ abstract class AbstractTypeApproximator(
 
             else -> {
                 val projection = capturedType.typeConstructorProjection()
-                if (projection.isStarProjection()) intersectTypes(supertypes.toList())
+                if (projection.isStarProjection()) intersectTypes(supertypes.map { it.replaceRecursionWithStarProjection(capturedType, toSuper) })
                 else projection.getType()
             }
         }
         val baseSubType = capturedType.lowerType() ?: nothingType()
-        // This replacement is important for resolving the code like below in K2.
-        //     fun bar(y: FieldOrRef<*>) = y.field
-        //     interface FieldOrRef<FF : AbstractField<FF>> { val field: FF }
-        //     abstract class AbstractField<out F : AbstractField<F>>
-        // During resolving the value parameter y type, K1 also builds a type for a star projection *.
-        // See fun TypeParameterDescriptor.starProjectionType(): KotlinType and fun buildStarProjectionTypeByTypeParameters.
-        // Thanks to it, K1 builds the star projection type as AbstractField<*> and no other approximation is needed.
-        //
-        // In turn, K2 never makes such a thing (K2 star projection has no associated type).
-        // Instead, it resolves y.field as CapturedType(*) C (see usage one line below),
-        // and the constructor of this captured type has a star projection and a supertype of `AbstractField<C>`.
-        //
-        // Without this replacement, the type approximator currently cannot handle such a situation properly
-        // and builds AbstractField<AbstractField<AbstractField<Any?>>>.
-        // The check it == type here is intended to find a recursion inside a captured type.
-        // A similar replacement for baseSubType looks unnecessary, no hits in the tests.
-        val replacedSuperType = if (isK2 && toSuper && baseSuperType.getArguments().any { it == capturedType }) {
-            baseSuperType.replaceArguments {
-                when {
-                    it != capturedType -> it
-                    // It's possible to use the stub here, because K2 star projection is an object and
-                    // in fact this parameter is never used
-                    else -> createStarProjection(TypeParameterMarkerStubForK2StarProjection)
-                }
-            }
-        } else baseSuperType
 
         val approximatedSuperType by lazy(LazyThreadSafetyMode.NONE) {
-            approximateToSuperType(replacedSuperType, conf, depth)
+            approximateToSuperType(baseSuperType, conf, depth)
         }
         val approximatedSubType by lazy(LazyThreadSafetyMode.NONE) { approximateToSubType(baseSubType, conf, depth) }
 
@@ -364,7 +372,7 @@ abstract class AbstractTypeApproximator(
                 return null
             }
         }
-        val baseResult = if (toSuper) approximatedSuperType ?: replacedSuperType else approximatedSubType ?: baseSubType
+        val baseResult = if (toSuper) approximatedSuperType ?: baseSuperType else approximatedSubType ?: baseSubType
 
         // C = in Int, Int <: C => Int? <: C?
         // C = out Number, C <: Number => C? <: Number?
@@ -508,26 +516,14 @@ abstract class AbstractTypeApproximator(
 
             val capturedType = argumentType.lowerBoundIfFlexible().originalIfDefinitelyNotNullable().asCapturedType()
 
-            // When capturing recursive types with self upper bounds, their super types can contain captured types.
-            // In approximateCapturedType, we check if the super/subtypes of captured types need approximation even if captured types
-            // themselves don't need approximation, and will land here.
-            // To support this case, we also don't want to approximate captured types here if the configuration says so.
-            // TODO rework captured types approximation KT-65228
-            if (capturedType != null &&
-                isK2 &&
-                !conf.capturedType(ctx, capturedType) &&
-                ctx.hasRecursiveTypeParametersWithGivenSelfType(capturedType.typeConstructor())
-            ) {
-                continue@loop
-            }
-
             val capturedStarProjectionOrNull =
                 capturedType?.typeConstructorProjection()?.takeIf { it.isStarProjection() }
 
             if (capturedStarProjectionOrNull != null &&
                 (effectiveVariance == TypeVariance.OUT || effectiveVariance == TypeVariance.INV) &&
                 toSuper &&
-                capturedType.typeParameter() == parameter
+                capturedType.typeParameter() == parameter &&
+                (!isK2 || conf.capturedType(ctx, capturedType))
             ) {
                 newArguments[index] = capturedStarProjectionOrNull
                 continue@loop

compiler/testData/diagnostics/tests/inference/capturedTypes/expectedTypeHasCapturedStarArgument.kt

@@ -0,0 +1,23 @@
+// FIR_IDENTICAL
+// DIAGNOSTICS: -UNUSED_PARAMETER
+interface A<V>
+
+fun interface F<O> {
+  fun apply(): A<O>
+}
+
+object C {
+  fun <V> createA(): A<V> = TODO()
+}
+
+class B<V> {
+  fun bar(function: F<out V>): B<V> = TODO()
+
+  companion object {
+    fun <X> from(a: A<X>): B<X> = TODO()
+  }
+}
+
+fun foo(a: A<*>) {
+  B.from(a).bar { C.createA() }
+}

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/DiagnosticTestGenerated.java

@@ -18059,6 +18059,12 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/dontCheckNewCapturedTypeSpecificChecksForOldOnes.kt");
                 }
 
+                @Test
+                @TestMetadata("expectedTypeHasCapturedStarArgument.kt")
+                public void testExpectedTypeHasCapturedStarArgument() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/expectedTypeHasCapturedStarArgument.kt");
+                }
+
                 @Test
                 @TestMetadata("expectedTypeMismatchWithInVariance.kt")
                 public void testExpectedTypeMismatchWithInVariance() throws Exception {