NI: Try to solve constraint system with another flexibility from a type variable if couldn't solve

^KT-36254 Fixed

compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/FirOldFrontendDiagnosticsTestGenerated.java

@@ -16476,6 +16476,16 @@ public class FirOldFrontendDiagnosticsTestGenerated extends AbstractFirOldFronte
                 runTest("compiler/testData/diagnostics/tests/platformTypes/methodCall/entrySet.kt");
             }
 
+            @TestMetadata("flexibilityThroughTypeVariable.kt")
+            public void testFlexibilityThroughTypeVariable() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/platformTypes/methodCall/flexibilityThroughTypeVariable.kt");
+            }
+
+            @TestMetadata("flexibilityThroughTypeVariableOut.kt")
+            public void testFlexibilityThroughTypeVariableOut() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/platformTypes/methodCall/flexibilityThroughTypeVariableOut.kt");
+            }
+
             @TestMetadata("genericsAndArrays.kt")
             public void testGenericsAndArrays() throws Exception {
                 runTest("compiler/testData/diagnostics/tests/platformTypes/methodCall/genericsAndArrays.kt");

compiler/resolution/src/org/jetbrains/kotlin/resolve/calls/components/ResolutionParts.kt

@@ -173,7 +173,7 @@ internal object CreateFreshVariablesSubstitutor : ResolutionPart() {
         }
     }
 
-    private fun TypeParameterDescriptor.shouldBeFlexible(): Boolean {
+    fun TypeParameterDescriptor.shouldBeFlexible(): Boolean {
         return upperBounds.any {
             it.isFlexible() || ((it.constructor.declarationDescriptor as? TypeParameterDescriptor)?.run { shouldBeFlexible() } ?: false)
         }

compiler/resolution/src/org/jetbrains/kotlin/resolve/calls/inference/components/ConstraintIncorporator.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.resolve.calls.inference.components
 
+import org.jetbrains.kotlin.resolve.calls.components.CreateFreshVariablesSubstitutor.shouldBeFlexible
 import org.jetbrains.kotlin.resolve.calls.inference.model.*
 import org.jetbrains.kotlin.types.*
 import org.jetbrains.kotlin.types.model.*
@@ -27,7 +28,11 @@ class ConstraintIncorporator(
 
         fun getConstraintsForVariable(typeVariable: TypeVariableMarker): Collection<Constraint>
 
-        fun addNewIncorporatedConstraint(lowerType: KotlinTypeMarker, upperType: KotlinTypeMarker)
+        fun addNewIncorporatedConstraint(
+            lowerType: KotlinTypeMarker,
+            upperType: KotlinTypeMarker,
+            shouldTryUseDifferentFlexibilityForUpperType: Boolean
+        )
 
         fun addNewIncorporatedConstraint(typeVariable: TypeVariableMarker, type: KotlinTypeMarker, constraintContext: ConstraintContext)
     }
@@ -49,11 +54,14 @@ class ConstraintIncorporator(
         typeVariable: TypeVariableMarker,
         constraint: Constraint
     ) {
+        val shouldBeTypeVariableFlexible =
+            typeVariable is TypeVariableFromCallableDescriptor && typeVariable.originalTypeParameter.shouldBeFlexible()
+
         // \alpha <: constraint.type
         if (constraint.kind != ConstraintKind.LOWER) {
             getConstraintsForVariable(typeVariable).forEach {
                 if (it.kind != ConstraintKind.UPPER) {
-                    addNewIncorporatedConstraint(it.type, constraint.type)
+                    addNewIncorporatedConstraint(it.type, constraint.type, shouldBeTypeVariableFlexible)
                 }
             }
         }
@@ -62,7 +70,7 @@ class ConstraintIncorporator(
         if (constraint.kind != ConstraintKind.UPPER) {
             getConstraintsForVariable(typeVariable).forEach {
                 if (it.kind != ConstraintKind.LOWER) {
-                    addNewIncorporatedConstraint(constraint.type, it.type)
+                    addNewIncorporatedConstraint(constraint.type, it.type, shouldBeTypeVariableFlexible)
                 }
             }
         }

compiler/resolution/src/org/jetbrains/kotlin/resolve/calls/inference/components/ConstraintInjector.kt

@@ -148,10 +148,33 @@ class ConstraintInjector(
             }
         }
 
-        fun runIsSubtypeOf(lowerType: KotlinTypeMarker, upperType: KotlinTypeMarker) {
+        fun runIsSubtypeOf(
-            if (!AbstractTypeChecker.isSubtypeOf(this@TypeCheckerContext as AbstractTypeCheckerContext, lowerType, upperType)) {
+            lowerType: KotlinTypeMarker,
+            upperType: KotlinTypeMarker,
+            shouldTryUseDifferentFlexibilityForUpperType: Boolean = false
+        ) {
+            fun isSubtypeOf(upperType: KotlinTypeMarker) =
+                AbstractTypeChecker.isSubtypeOf(
+                    this@TypeCheckerContext as AbstractTypeCheckerContext,
+                    lowerType,
+                    upperType
+                )
+
+            if (!isSubtypeOf(upperType)) {
                 // todo improve error reporting -- add information about base types
-                c.addError(NewConstraintError(lowerType, upperType, position))
+                if (shouldTryUseDifferentFlexibilityForUpperType && upperType is SimpleType) {
+                    /*
+                     * Please don't reuse this logic.
+                     * It's necessary to solve constraint systems when flexibility isn't propagated through a type variable.
+                     * It's OK in the old inference because it uses already substituted types, that are with the correct flexibility.
+                     */
+                    val flexibleUpperType = createFlexibleType(upperType, upperType.withNullability(true))
+                    if (!isSubtypeOf(flexibleUpperType)) {
+                        c.addError(NewConstraintError(lowerType, flexibleUpperType, position))
+                    }
+                } else {
+                    c.addError(NewConstraintError(lowerType, upperType, position))
+                }
             }
         }
 
@@ -181,10 +204,14 @@ class ConstraintInjector(
         }
 
         // from ConstraintIncorporator.Context
-        override fun addNewIncorporatedConstraint(lowerType: KotlinTypeMarker, upperType: KotlinTypeMarker) {
+        override fun addNewIncorporatedConstraint(
+            lowerType: KotlinTypeMarker,
+            upperType: KotlinTypeMarker,
+            shouldTryUseDifferentFlexibilityForUpperType: Boolean
+        ) {
             if (lowerType === upperType) return
             if (c.isAllowedType(lowerType) && c.isAllowedType(upperType)) {
-                runIsSubtypeOf(lowerType, upperType)
+                runIsSubtypeOf(lowerType, upperType, shouldTryUseDifferentFlexibilityForUpperType)
             }
         }
 

compiler/testData/diagnostics/tests/platformTypes/methodCall/flexibilityThroughTypeVariable.kt

@@ -0,0 +1,18 @@
+// !LANGUAGE: +NewInference
+// !DIAGNOSTICS: -UNUSED_PARAMETER
+
+// Issue: KT-36254
+
+// FILE: Convertor.java
+
+public interface Convertor<Src, Dst> {
+    Dst convert(Src o);
+}
+
+// FILE: main.kt
+
+fun takeConvertor(c: Convertor<String, String>) {}
+
+fun main() {
+    takeConvertor(Convertor { null })
+}

compiler/testData/diagnostics/tests/platformTypes/methodCall/flexibilityThroughTypeVariable.txt

@@ -0,0 +1,11 @@
+package
+
+public fun main(): kotlin.Unit
+public fun takeConvertor(/*0*/ c: Convertor<kotlin.String, kotlin.String>): kotlin.Unit
+
+public interface Convertor</*0*/ Src : kotlin.Any!, /*1*/ Dst : kotlin.Any!> {
+    public abstract fun convert(/*0*/ o: Src!): Dst!
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/testData/diagnostics/tests/platformTypes/methodCall/flexibilityThroughTypeVariableOut.kt

@@ -0,0 +1,20 @@
+// !LANGUAGE: +NewInference
+// !DIAGNOSTICS: -UNUSED_PARAMETER
+
+// Issue: KT-36254
+
+// FILE: Convertor.java
+
+public interface Convertor<Src, Dst> {
+    Out<Dst> convert(Out<Src> o);
+}
+
+// FILE: main.kt
+
+fun takeConvertor(c: Convertor<String, String>) {}
+
+class Out<out T> {}
+
+fun main(o: Out<Nothing?>) {
+    takeConvertor(Convertor { o })
+}

compiler/testData/diagnostics/tests/platformTypes/methodCall/flexibilityThroughTypeVariableOut.txt

@@ -0,0 +1,18 @@
+package
+
+public fun main(/*0*/ o: Out<kotlin.Nothing?>): kotlin.Unit
+public fun takeConvertor(/*0*/ c: Convertor<kotlin.String, kotlin.String>): kotlin.Unit
+
+public interface Convertor</*0*/ Src : kotlin.Any!, /*1*/ Dst : kotlin.Any!> {
+    public abstract fun convert(/*0*/ o: Out<Src!>!): Out<Dst!>!
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public final class Out</*0*/ out T> {
+    public constructor Out</*0*/ out T>()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/tests/org/jetbrains/kotlin/checkers/DiagnosticsTestGenerated.java

@@ -16483,6 +16483,16 @@ public class DiagnosticsTestGenerated extends AbstractDiagnosticsTest {
                     runTest("compiler/testData/diagnostics/tests/platformTypes/methodCall/entrySet.kt");
                 }
 
+                @TestMetadata("flexibilityThroughTypeVariable.kt")
+                public void testFlexibilityThroughTypeVariable() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/platformTypes/methodCall/flexibilityThroughTypeVariable.kt");
+                }
+
+                @TestMetadata("flexibilityThroughTypeVariableOut.kt")
+                public void testFlexibilityThroughTypeVariableOut() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/platformTypes/methodCall/flexibilityThroughTypeVariableOut.kt");
+                }
+
                 @TestMetadata("genericsAndArrays.kt")
                 public void testGenericsAndArrays() throws Exception {
                     runTest("compiler/testData/diagnostics/tests/platformTypes/methodCall/genericsAndArrays.kt");

compiler/tests/org/jetbrains/kotlin/checkers/javac/DiagnosticsUsingJavacTestGenerated.java

@@ -16478,6 +16478,16 @@ public class DiagnosticsUsingJavacTestGenerated extends AbstractDiagnosticsUsing
                     runTest("compiler/testData/diagnostics/tests/platformTypes/methodCall/entrySet.kt");
                 }
 
+                @TestMetadata("flexibilityThroughTypeVariable.kt")
+                public void testFlexibilityThroughTypeVariable() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/platformTypes/methodCall/flexibilityThroughTypeVariable.kt");
+                }
+
+                @TestMetadata("flexibilityThroughTypeVariableOut.kt")
+                public void testFlexibilityThroughTypeVariableOut() throws Exception {
+                    runTest("compiler/testData/diagnostics/tests/platformTypes/methodCall/flexibilityThroughTypeVariableOut.kt");
+                }
+
                 @TestMetadata("genericsAndArrays.kt")
                 public void testGenericsAndArrays() throws Exception {
                     runTest("compiler/testData/diagnostics/tests/platformTypes/methodCall/genericsAndArrays.kt");