K2: Avoid inappropriate approximation of captured type to Nothing?

There's a heuristic for approximation of a captured type that once
it has non-trivial lower bound (other than Nothing), it's worth
approximating it to sub-type even while the containing
top-level type is being approximated to super-type.

And that sounds reasonable in case the lower bound is indeed non-trivial,
but that's not the case because nullability here comes from
the nullability of captured type position.

So, the fix is basically not to treat such approximations as non-trivial.
And while that seems to be a bit of a change in the language semantics,
it still looks reasonable (see other changes in test data and KT-58087)

^KT-57958 Fixed
^KT-58087 Fixed

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/DiagnosticCompilerTestFE10TestdataTestGenerated.java

@@ -15417,6 +15417,12 @@ public class DiagnosticCompilerTestFE10TestdataTestGenerated extends AbstractDia
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/approximateContravariantCapturedTypes.kt");
                 }
 
+                @Test
+                @TestMetadata("approximationLeavesNonTrivialLowerBound.kt")
+                public void testApproximationLeavesNonTrivialLowerBound() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/approximationLeavesNonTrivialLowerBound.kt");
+                }
+
                 @Test
                 @TestMetadata("avoidCreatingUselessCapturedTypes.kt")
                 public void testAvoidCreatingUselessCapturedTypes() throws Exception {
@@ -15441,6 +15447,18 @@ public class DiagnosticCompilerTestFE10TestdataTestGenerated extends AbstractDia
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureForPlatformTypes.kt");
                 }
 
+                @Test
+                @TestMetadata("captureFromNullableTypeInScope.kt")
+                public void testCaptureFromNullableTypeInScope() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureFromNullableTypeInScope.kt");
+                }
+
+                @Test
+                @TestMetadata("captureFromNullableTypeInScopeAny.kt")
+                public void testCaptureFromNullableTypeInScopeAny() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureFromNullableTypeInScopeAny.kt");
+                }
+
                 @Test
                 @TestMetadata("captureFromNullableTypeVariable.kt")
                 public void testCaptureFromNullableTypeVariable() throws Exception {

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirPreresolvedReversedDiagnosticCompilerFE10TestDataTestGenerated.java

@@ -15417,6 +15417,12 @@ public class LLFirPreresolvedReversedDiagnosticCompilerFE10TestDataTestGenerated
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/approximateContravariantCapturedTypes.kt");
                 }
 
+                @Test
+                @TestMetadata("approximationLeavesNonTrivialLowerBound.kt")
+                public void testApproximationLeavesNonTrivialLowerBound() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/approximationLeavesNonTrivialLowerBound.kt");
+                }
+
                 @Test
                 @TestMetadata("avoidCreatingUselessCapturedTypes.kt")
                 public void testAvoidCreatingUselessCapturedTypes() throws Exception {
@@ -15441,6 +15447,18 @@ public class LLFirPreresolvedReversedDiagnosticCompilerFE10TestDataTestGenerated
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureForPlatformTypes.kt");
                 }
 
+                @Test
+                @TestMetadata("captureFromNullableTypeInScope.kt")
+                public void testCaptureFromNullableTypeInScope() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureFromNullableTypeInScope.kt");
+                }
+
+                @Test
+                @TestMetadata("captureFromNullableTypeInScopeAny.kt")
+                public void testCaptureFromNullableTypeInScopeAny() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureFromNullableTypeInScopeAny.kt");
+                }
+
                 @Test
                 @TestMetadata("captureFromNullableTypeVariable.kt")
                 public void testCaptureFromNullableTypeVariable() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirLightTreeOldFrontendDiagnosticsTestGenerated.java

@@ -15417,6 +15417,12 @@ public class FirLightTreeOldFrontendDiagnosticsTestGenerated extends AbstractFir
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/approximateContravariantCapturedTypes.kt");
                 }
 
+                @Test
+                @TestMetadata("approximationLeavesNonTrivialLowerBound.kt")
+                public void testApproximationLeavesNonTrivialLowerBound() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/approximationLeavesNonTrivialLowerBound.kt");
+                }
+
                 @Test
                 @TestMetadata("avoidCreatingUselessCapturedTypes.kt")
                 public void testAvoidCreatingUselessCapturedTypes() throws Exception {
@@ -15441,6 +15447,18 @@ public class FirLightTreeOldFrontendDiagnosticsTestGenerated extends AbstractFir
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureForPlatformTypes.kt");
                 }
 
+                @Test
+                @TestMetadata("captureFromNullableTypeInScope.kt")
+                public void testCaptureFromNullableTypeInScope() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureFromNullableTypeInScope.kt");
+                }
+
+                @Test
+                @TestMetadata("captureFromNullableTypeInScopeAny.kt")
+                public void testCaptureFromNullableTypeInScopeAny() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureFromNullableTypeInScopeAny.kt");
+                }
+
                 @Test
                 @TestMetadata("captureFromNullableTypeVariable.kt")
                 public void testCaptureFromNullableTypeVariable() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirPsiOldFrontendDiagnosticsTestGenerated.java

@@ -15423,6 +15423,12 @@ public class FirPsiOldFrontendDiagnosticsTestGenerated extends AbstractFirPsiDia
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/approximateContravariantCapturedTypes.kt");
                 }
 
+                @Test
+                @TestMetadata("approximationLeavesNonTrivialLowerBound.kt")
+                public void testApproximationLeavesNonTrivialLowerBound() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/approximationLeavesNonTrivialLowerBound.kt");
+                }
+
                 @Test
                 @TestMetadata("avoidCreatingUselessCapturedTypes.kt")
                 public void testAvoidCreatingUselessCapturedTypes() throws Exception {
@@ -15447,6 +15453,18 @@ public class FirPsiOldFrontendDiagnosticsTestGenerated extends AbstractFirPsiDia
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureForPlatformTypes.kt");
                 }
 
+                @Test
+                @TestMetadata("captureFromNullableTypeInScope.kt")
+                public void testCaptureFromNullableTypeInScope() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureFromNullableTypeInScope.kt");
+                }
+
+                @Test
+                @TestMetadata("captureFromNullableTypeInScopeAny.kt")
+                public void testCaptureFromNullableTypeInScopeAny() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureFromNullableTypeInScopeAny.kt");
+                }
+
                 @Test
                 @TestMetadata("captureFromNullableTypeVariable.kt")
                 public void testCaptureFromNullableTypeVariable() throws Exception {

compiler/resolution.common/src/org/jetbrains/kotlin/types/AbstractTypeApproximator.kt

@@ -531,25 +531,27 @@ abstract class AbstractTypeApproximator(
                         continue@loop
                     }
 
-                    /**
-                     * Example with non-trivial both type approximations:
-                     * Inv<In<C>> where C = in Int
-                     * Inv<In<C>> <: Inv<out In<Int>>
-                     * Inv<In<C>> <: Inv<in In<Any?>>
-                     *
-                     * So such case is rare and we will chose Inv<out In<Int>> for now.
-                     *
-                     * Note that for case Inv<C> we will chose Inv<in Int>, because it is more informative then Inv<out Any?>.
-                     * May be we should do the same for deeper types, but not now.
-                     */
+                    // In case of Inv<C> and C = Captured(in Int), we choose Inv<in Int> as resulting approximation
+                    // NB: Inv<C> <: Inv<in Int> because Int <: C (as Int is a lower bound of the C captured type)
+                    //
+                    // That behavior of choosing non-trivial lower bound is crucial when there's also non-trivial upper bound,
+                    // like if Inv would be declared as `interface Inv<T : CharSequence>` (see test approximationLeavesNonTrivialLowerBound.kt)
+                    //
+                    // In that case the next condition after that doesn't help because in case of both non-trivial bounds, it chooses the upper one
                     if (argumentType.typeConstructor().isCapturedTypeConstructor()) {
                         val subType = approximateToSubType(argumentType, conf, depth) ?: continue@loop
-                        if (!subType.isTrivialSub()) {
+                        if (shouldUseSubTypeForCapturedArgument(subType, argumentType, conf, depth)) {
                             newArguments[index] = createTypeArgument(subType, TypeVariance.IN)
                             continue@loop
                         }
                     }
 
+                    // Example with non-trivial both type approximations:
+                    //  Inv<In<C>> where C = Captured(in Int)
+                    //  Inv<In<C>> <: Inv<out In<Int>>
+                    //  Inv<In<C>> <: Inv<in In<Any?>>
+                    //
+                    // So, both of the options are possible, but since such case is rare we will chose Inv<out In<Int>> for now
                     val approximatedSuperType =
                         approximateToSuperType(argumentType, conf, depth) ?: continue@loop // null means that this type we can leave as is
                     if (approximatedSuperType.isTrivialSuper()) {
@@ -577,6 +579,36 @@ abstract class AbstractTypeApproximator(
         return approximateLocalTypes(approximatedType, conf, toSuper) ?: approximatedType
     }
 
+    private fun shouldUseSubTypeForCapturedArgument(
+        subType: KotlinTypeMarker,
+        capturedArgumentType: KotlinTypeMarker,
+        conf: TypeApproximatorConfiguration,
+        depth: Int,
+    ): Boolean {
+        if (subType.isTrivialSub()) return false
+        // For K1, the result is always `!subType.isTrivialSub()` (leaving the old behavior)
+        if (!isK2) return true
+
+        // Basically, what's written further might be simplified like
+        // return !approximateToSubType(capturedArgumentType.withNullability(false), conf, depth)!!.isTrivialSub()
+        // But it seems that now it looks a bit more clear and probably performant, thus first two if's are basically fast paths
+
+        // If it's not `Nothing?`, then the lower bound is indeed non-trivial
+        if (!subType.lowerBoundIfFlexible().isNullableNothing()) return true
+
+        // Here the subType is `Nothing?`, and it might be trivial only in cause the nullability is caused by nullability of captured type itself
+
+        // If captured type is not marked as nullable, then nullability of subType came from the lower bound of the captured type.
+        // Thus, the lower bound is non-trivial for sure
+        if (!capturedArgumentType.isMarkedNullable()) return true
+
+        val notMarkedNullableSubType =
+            approximateToSubType(capturedArgumentType.withNullability(false), conf, depth)
+                ?: error("Not-marked-nullable version of captured type approximation should also return not-null")
+
+        return !notMarkedNullableSubType.isTrivialSub()
+    }
+
     private fun KotlinTypeMarker.defaultResult(toSuper: Boolean) = if (toSuper) nullableAnyType() else {
         if (this is SimpleTypeMarker && isMarkedNullable()) nullableNothingType() else nothingType()
     }

compiler/testData/diagnostics/tests/inference/capturedTypes/approximationLeavesNonTrivialLowerBound.kt

@@ -0,0 +1,19 @@
+// FIR_IDENTICAL
+interface Inv<T : CharSequence>
+
+fun <E : CharSequence> id(i: Inv<E>): Inv<E> = i
+
+fun foo1(x: Inv<in String>) {
+    val y = <!DEBUG_INFO_EXPRESSION_TYPE("Inv<in kotlin.String>")!>id(x)<!> // Should return Inv<in String>
+    bar1(y)
+}
+
+fun bar1(i: Inv<in String>) {}
+
+fun foo2(x: Inv<in Nothing>) {
+    // Inv<out kotlin.CharSequence> is OK here, because it's still a subtype of Inv<in Nothing>
+    val y = <!DEBUG_INFO_EXPRESSION_TYPE("Inv<out kotlin.CharSequence>")!>id(x)<!>
+    bar2(y)
+}
+
+fun bar2(i: Inv<in Nothing>) {}

compiler/testData/diagnostics/tests/inference/capturedTypes/captureForNullableTypes.fir.kt

@@ -4,15 +4,15 @@
 fun <T: Any> bar(a: Array<T>): Array<T?> =  null!!
 
 fun test1(a: Array<out Int>) {
-    val r: Array<out Int?> = <!INITIALIZER_TYPE_MISMATCH!>bar(a)<!>
+    val r: Array<out Int?> = bar(a)
     val t = bar(a)
-    t checkType { <!UNRESOLVED_REFERENCE_WRONG_RECEIVER!>_<!><Array<out Int?>>() }
+    t checkType { _<Array<out Int?>>() }
 }
 
 fun <T: Any> foo(l: Array<T>): Array<Array<T?>> = null!!
 
 fun test2(a: Array<out Int>) {
-    val r: Array<out Array<out Int?>> = <!INITIALIZER_TYPE_MISMATCH!>foo(a)<!>
+    val r: Array<out Array<out Int?>> = foo(a)
     val t = foo(a)
-    t checkType { <!UNRESOLVED_REFERENCE_WRONG_RECEIVER!>_<!><Array<out Array<out Int?>>>() }
+    t checkType { _<Array<out Array<out Int?>>>() }
 }

compiler/testData/diagnostics/tests/inference/capturedTypes/captureFromNullableTypeInScope.kt

@@ -0,0 +1,18 @@
+// FIR_IDENTICAL
+// ISSUE: KT-57958
+
+fun ListVM<*>.foo() {
+    val currentItem1: MutableProperty<out ListItemVM<*>?> = currentItem
+}
+
+interface MutableProperty<T> {
+    var value: T
+}
+
+interface ListItemVM<out TItem> {
+    val value: TItem
+}
+
+interface ListVM<TItemVM : ListItemVM<*>> {
+    val currentItem: MutableProperty<TItemVM?>
+}

compiler/testData/diagnostics/tests/inference/capturedTypes/captureFromNullableTypeInScopeAny.kt

@@ -0,0 +1,14 @@
+// FIR_IDENTICAL
+// ISSUE: KT-57958
+
+fun ListVM<*>.foo() {
+    val currentItem1: MutableProperty<out Any?> = <!DEBUG_INFO_EXPRESSION_TYPE("MutableProperty<in kotlin.Nothing?>")!>currentItem<!>
+}
+
+interface MutableProperty<T> {
+    var value: T
+}
+
+interface ListVM<TItemVM> {
+    val currentItem: MutableProperty<TItemVM?>
+}

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/DiagnosticTestGenerated.java

@@ -15423,6 +15423,12 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/approximateContravariantCapturedTypes.kt");
                 }
 
+                @Test
+                @TestMetadata("approximationLeavesNonTrivialLowerBound.kt")
+                public void testApproximationLeavesNonTrivialLowerBound() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/approximationLeavesNonTrivialLowerBound.kt");
+                }
+
                 @Test
                 @TestMetadata("avoidCreatingUselessCapturedTypes.kt")
                 public void testAvoidCreatingUselessCapturedTypes() throws Exception {
@@ -15447,6 +15453,18 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
                     runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureForPlatformTypes.kt");
                 }
 
+                @Test
+                @TestMetadata("captureFromNullableTypeInScope.kt")
+                public void testCaptureFromNullableTypeInScope() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureFromNullableTypeInScope.kt");
+                }
+
+                @Test
+                @TestMetadata("captureFromNullableTypeInScopeAny.kt")
+                public void testCaptureFromNullableTypeInScopeAny() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/inference/capturedTypes/captureFromNullableTypeInScopeAny.kt");
+                }
+
                 @Test
                 @TestMetadata("captureFromNullableTypeVariable.kt")
                 public void testCaptureFromNullableTypeVariable() throws Exception {