Preserve nullability while substituting captured types

compiler/testData/capturedTypeApproximation/nullableTypeVariable.kt

@@ -0,0 +1,11 @@
+class Inv<T>
+
+fun foo<T>(a: Inv<T?>) {}
+
+// T captures 'in Int'
+// lower: Nothing
+// upper: Inv<in Int?>
+
+// T captures 'out Int'
+// lower: Nothing
+// upper: Inv<out Int?>

compiler/testData/diagnostics/tests/inference/capturedTypes/captureForNullableTypes.kt

@@ -0,0 +1,24 @@
+// !DIAGNOSTICS: -UNUSED_VARIABLE -UNUSED_PARAMETER
+// !CHECK_TYPE
+
+fun <T: Any> bar(a: Array<T>): Array<T?> =  null!!
+
+fun test1(a: Array<Int>) {
+    val r: Array<Int?> = bar(a)
+    val t = bar(a)
+    t checkType { it : _<Array<Int?>> }
+}
+
+fun <T: Any> foo(l: Array<T>): Array<Array<T?>> = null!!
+
+fun test2(a: Array<out Int>) {
+    val r: Array<out Array<out Int?>> = foo(a)
+    val t = foo(a)
+    t checkType { it : _<Array<out Array<out Int?>>> }
+}
+
+fun test3(a: Array<in Int>) {
+    val r: Array<out Array<in Int?>> = foo(a)
+    val t = foo(a)
+    t checkType { it : _<Array<out Array<in Int?>>> }
+}

compiler/testData/diagnostics/tests/inference/capturedTypes/captureForNullableTypes.txt

@@ -0,0 +1,7 @@
+package
+
+internal fun </*0*/ T : kotlin.Any> bar(/*0*/ a: kotlin.Array<T>): kotlin.Array<T?>
+internal fun </*0*/ T : kotlin.Any> foo(/*0*/ l: kotlin.Array<T>): kotlin.Array<kotlin.Array<T?>>
+internal fun test1(/*0*/ a: kotlin.Array<kotlin.Int>): kotlin.Unit
+internal fun test2(/*0*/ a: kotlin.Array<out kotlin.Int>): kotlin.Unit
+internal fun test3(/*0*/ a: kotlin.Array<in kotlin.Int>): kotlin.Unit

compiler/tests/org/jetbrains/jet/checkers/JetDiagnosticsTestGenerated.java

@@ -5094,6 +5094,12 @@ public class JetDiagnosticsTestGenerated extends AbstractJetDiagnosticsTest {
                     JetTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("compiler/testData/diagnostics/tests/inference/capturedTypes"), Pattern.compile("^(.+)\\.kt$"), true);
                 }
 
+                @TestMetadata("captureForNullableTypes.kt")
+                public void testCaptureForNullableTypes() throws Exception {
+                    String fileName = JetTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/inference/capturedTypes/captureForNullableTypes.kt");
+                    doTest(fileName);
+                }
+
                 @TestMetadata("captureForPlatformTypes.kt")
                 public void testCaptureForPlatformTypes() throws Exception {
                     String fileName = JetTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/inference/capturedTypes/captureForPlatformTypes.kt");

compiler/tests/org/jetbrains/jet/resolve/typeApproximation/CapturedTypeApproximationTestGenerated.java

@@ -72,6 +72,12 @@ public class CapturedTypeApproximationTestGenerated extends AbstractCapturedType
         doTest(fileName);
     }
 
+    @TestMetadata("nullableTypeVariable.kt")
+    public void testNullableTypeVariable() throws Exception {
+        String fileName = JetTestUtils.navigationMetadata("compiler/testData/capturedTypeApproximation/nullableTypeVariable.kt");
+        doTest(fileName);
+    }
+
     @TestMetadata("useSiteVarianceIn.kt")
     public void testUseSiteVarianceIn() throws Exception {
         String fileName = JetTestUtils.navigationMetadata("compiler/testData/capturedTypeApproximation/useSiteVarianceIn.kt");

core/descriptors/src/org/jetbrains/jet/lang/types/CapturedTypeApproximation.kt

@@ -51,12 +51,14 @@ private val NULLABLE_ANY = KotlinBuiltIns.getInstance().getNullableAnyType()
 
 private val NOTHING = KotlinBuiltIns.getInstance().getNothingType()
 
+private val NULLABLE_NOTHING = KotlinBuiltIns.getInstance().getNullableNothingType()
+
 private fun TypeArgument.toTypeProjection(): TypeProjection {
     assert(isConsistent) { "Only consistent enhanced type propection can be converted to type projection" }
     fun removeProjectionIfRedundant(variance: Variance) = if (variance == typeParameter.getVariance()) Variance.INVARIANT else variance
     return when {
         inProjection == outProjection -> TypeProjectionImpl(inProjection)
-        inProjection == NOTHING -> TypeProjectionImpl(removeProjectionIfRedundant(Variance.OUT_VARIANCE), outProjection)
+        inProjection == NOTHING || inProjection == NULLABLE_NOTHING -> TypeProjectionImpl(removeProjectionIfRedundant(Variance.OUT_VARIANCE), outProjection)
         outProjection == NULLABLE_ANY -> TypeProjectionImpl(removeProjectionIfRedundant(Variance.IN_VARIANCE), inProjection)
         else -> throw AssertionError("Enhanced type projection can't be converted to type projection: $this")
     }
@@ -99,10 +101,13 @@ public fun approximateCapturedTypes(type: JetType): ApproximationBounds<JetType>
     val typeConstructor = type.getConstructor()
     if (type.isCaptured()) {
         val typeProjection = (typeConstructor as CapturedTypeConstructor).typeProjection
+        // todo: preserve flexibility as well
+        fun JetType.makeNullableIfNeeded() = TypeUtils.makeNullableIfNeeded(this, type.isMarkedNullable())
+        val bound = typeProjection.getType().makeNullableIfNeeded()
 
         return when (typeProjection.getProjectionKind()) {
-            Variance.IN_VARIANCE -> ApproximationBounds(typeProjection.getType(), NULLABLE_ANY)
+            Variance.IN_VARIANCE -> ApproximationBounds(bound, NULLABLE_ANY)
-            Variance.OUT_VARIANCE -> ApproximationBounds(NOTHING, typeProjection.getType())
+            Variance.OUT_VARIANCE -> ApproximationBounds(NOTHING.makeNullableIfNeeded(), bound)
             else -> throw AssertionError("Only nontrivial projections should have been captured, not: $typeProjection")
         }
     }