[FIR] Only don't approximate nested captured arguments if they have recursive supertypes

This fixes a compiler crash
IllegalStateException: Captured type for incorporation shouldn't escape
from incorporation

The crash occurs when a captured type with status FOR_INCORPORATION
is two layers deep inside a captured type with status FROM_EXPRESSION.
We first check if approximation is required for the most outer captured
type in AbstractTypeApproximator.approximateCapturedType.
Then we encounter the second captured type with status FROM_EXPRESSION
in AbstractTypeApproximator.approximateParametrizedType.
At this point, we stop checking and miss the third captured type with
status FOR_INCORPORATION.

Unfortunately, we can't check recursively if nested captured types
need to be approximated because of types with recursive super types
(the original reason why the extra check was introduced).
That's why we restrict the second check to types with recursive
super types, effectively restoring the previous behavior for all other
types.

#KT-65050 Fixed

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/DiagnosticCompilerTestFE10TestdataTestGenerated.java

@@ -17899,6 +17899,12 @@ public class DiagnosticCompilerTestFE10TestdataTestGenerated extends AbstractDia
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/capturedFlexibleIntersectionTypesWithDifferentConstructors.kt");
                 }
 
+                @Test
+                @TestMetadata("capturedForIncorporationEscapes.kt")
+                public void testCapturedForIncorporationEscapes() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/capturedForIncorporationEscapes.kt");
+                }
+
                 @Test
                 @TestMetadata("capturedInTypeInference.kt")
                 public void testCapturedInTypeInference() throws Exception {

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirPreresolvedReversedDiagnosticCompilerFE10TestDataTestGenerated.java

@@ -17899,6 +17899,12 @@ public class LLFirPreresolvedReversedDiagnosticCompilerFE10TestDataTestGenerated
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/capturedFlexibleIntersectionTypesWithDifferentConstructors.kt");
                 }
 
+                @Test
+                @TestMetadata("capturedForIncorporationEscapes.kt")
+                public void testCapturedForIncorporationEscapes() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/capturedForIncorporationEscapes.kt");
+                }
+
                 @Test
                 @TestMetadata("capturedInTypeInference.kt")
                 public void testCapturedInTypeInference() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirLightTreeOldFrontendDiagnosticsTestGenerated.java

@@ -17893,6 +17893,12 @@ public class FirLightTreeOldFrontendDiagnosticsTestGenerated extends AbstractFir
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/capturedFlexibleIntersectionTypesWithDifferentConstructors.kt");
                 }
 
+                @Test
+                @TestMetadata("capturedForIncorporationEscapes.kt")
+                public void testCapturedForIncorporationEscapes() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/capturedForIncorporationEscapes.kt");
+                }
+
                 @Test
                 @TestMetadata("capturedInTypeInference.kt")
                 public void testCapturedInTypeInference() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirPsiOldFrontendDiagnosticsTestGenerated.java

@@ -17899,6 +17899,12 @@ public class FirPsiOldFrontendDiagnosticsTestGenerated extends AbstractFirPsiDia
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/capturedFlexibleIntersectionTypesWithDifferentConstructors.kt");
                 }
 
+                @Test
+                @TestMetadata("capturedForIncorporationEscapes.kt")
+                public void testCapturedForIncorporationEscapes() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/capturedForIncorporationEscapes.kt");
+                }
+
                 @Test
                 @TestMetadata("capturedInTypeInference.kt")
                 public void testCapturedInTypeInference() throws Exception {

compiler/resolution.common/src/org/jetbrains/kotlin/types/AbstractTypeApproximator.kt

@@ -8,6 +8,7 @@ package org.jetbrains.kotlin.types
 import org.jetbrains.kotlin.config.LanguageFeature
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.resolve.calls.NewCommonSuperTypeCalculator.commonSuperType
+import org.jetbrains.kotlin.resolve.calls.inference.hasRecursiveTypeParametersWithGivenSelfType
 import org.jetbrains.kotlin.types.model.*
 import org.jetbrains.kotlin.utils.addToStdlib.runIf
 import java.util.concurrent.ConcurrentHashMap
@@ -481,7 +482,11 @@ abstract class AbstractTypeApproximator(
             // In approximateCapturedType, we check if the super/subtypes of captured types need approximation even if captured types
             // themselves don't need approximation, and will land here.
             // To support this case, we also don't want to approximate captured types here if the configuration says so.
-            if (capturedType != null && isK2 && !conf.capturedType(ctx, capturedType)) {
+            if (capturedType != null &&
+                isK2 &&
+                !conf.capturedType(ctx, capturedType) &&
+                ctx.hasRecursiveTypeParametersWithGivenSelfType(capturedType.typeConstructor())
+            ) {
                 continue@loop
             }
 

compiler/testData/diagnostics/tests/inference/capturedTypes/capturedForIncorporationEscapes.kt

@@ -0,0 +1,18 @@
+// FIR_IDENTICAL
+// ISSUE: KT-65050
+
+interface HttpResponse<T>
+
+class GitLabMergeRequestShortRestDTO
+
+fun loadMergeRequests(): HttpResponse<out List<GitLabMergeRequestShortRestDTO>> = TODO()
+
+inline fun <reified T> loadList(): HttpResponse<out List<T>> = TODO()
+
+fun loadMergeRequests(boolean: Boolean) {
+    val response = if (boolean) {
+        loadMergeRequests()
+    } else {
+        loadList()
+    }
+}

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/DiagnosticTestGenerated.java

@@ -17899,6 +17899,12 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/capturedFlexibleIntersectionTypesWithDifferentConstructors.kt");
                 }
 
+                @Test
+                @TestMetadata("capturedForIncorporationEscapes.kt")
+                public void testCapturedForIncorporationEscapes() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/capturedForIncorporationEscapes.kt");
+                }
+
                 @Test
                 @TestMetadata("capturedInTypeInference.kt")
                 public void testCapturedInTypeInference() throws Exception {