[NI] Approximate captured types before type variable fixation.

We prefer denotable types when we solve constraint system.
I.e. if for T we have not equality constraint with captured type we can approximate captured type to denotable type.

compiler/resolution/src/org/jetbrains/kotlin/resolve/calls/inference/components/ConstraintIncorporator.kt

@@ -148,14 +148,8 @@ class ConstraintIncorporator(val typeApproximator: TypeApproximator) {
     }
 
     private fun approximateCapturedTypes(type: UnwrappedType, toSuper: Boolean): UnwrappedType =
-            if (toSuper) typeApproximator.approximateToSuperType(type, CapturedTypesApproximatorConfiguration) ?: type
-            else typeApproximator.approximateToSubType(type, CapturedTypesApproximatorConfiguration) ?: type
+            if (toSuper) typeApproximator.approximateToSuperType(type, TypeApproximatorConfiguration.IncorporationConfiguration) ?: type
+            else typeApproximator.approximateToSubType(type, TypeApproximatorConfiguration.IncorporationConfiguration) ?: type
 
 
-    private object CapturedTypesApproximatorConfiguration : TypeApproximatorConfiguration.AllFlexibleSameValue() {
-        override val allFlexible get() = true
-        override val capturedType get() = { it: NewCapturedType -> it.captureStatus != CaptureStatus.FOR_INCORPORATION }
-        override val intersection get() = IntersectionStrategy.ALLOWED
-        override val typeVariable: (TypeVariableTypeConstructor) -> Boolean get() = { true }
-    }
 }

compiler/resolution/src/org/jetbrains/kotlin/resolve/calls/inference/components/ResultTypeResolver.kt

@@ -27,7 +27,10 @@ import org.jetbrains.kotlin.types.*
 import org.jetbrains.kotlin.types.checker.intersectTypes
 import java.util.*
 
-class ResultTypeResolver(val commonSupertypeCalculator: CommonSupertypeCalculator) {
+class ResultTypeResolver(
+        val commonSupertypeCalculator: CommonSupertypeCalculator,
+        val typeApproximator: TypeApproximator
+) {
     interface Context {
         fun isProperType(type: UnwrappedType): Boolean
     }
@@ -38,11 +41,31 @@ class ResultTypeResolver(val commonSupertypeCalculator: CommonSupertypeCalculato
         if (direction == ResolveDirection.TO_SUBTYPE || direction == ResolveDirection.UNKNOWN) {
             val lowerConstraints = variableWithConstraints.constraints.filter { it.kind == ConstraintKind.LOWER && c.isProperType(it.type) }
             if (lowerConstraints.isNotEmpty()) {
-                return commonSupertypeCalculator(convertLowerTypesWithKnowledgeOfNumberTypes(lowerConstraints))
+                val commonSupertype = commonSupertypeCalculator(convertLowerTypesWithKnowledgeOfNumberTypes(lowerConstraints))
+                /**
+                 *
+                 * fun <T> Array<out T>.intersect(other: Iterable<T>) {
+                 *      val set = toMutableSet()
+                 *      set.retainAll(other)
+                 * }
+                 * fun <X> Array<out X>.toMutableSet(): MutableSet<X> = ...
+                 * fun <Y> MutableCollection<in Y>.retainAll(elements: Iterable<Y>) {}
+                 *
+                 * Here, when we solve type system for `toMutableSet` we have the following constrains:
+                 * Array<C(out T)> <: Array<out X> => C(out X) <: T.
+                 * If we fix it to T = C(out X) then return type of `toMutableSet()` will be `MutableSet<C(out X)>`
+                 * and type of variable `set` will be `MutableSet<out T>` and the following line will have contradiction.
+                 *
+                 * To fix this problem when we fix variable, we will approximate captured types before fixation.
+                 *
+                 * todo: may be for TO_SUPER direction we should do the same
+                 */
+
+                return typeApproximator.approximateToSuperType(commonSupertype, TypeApproximatorConfiguration.CapturedTypesApproximation) ?: commonSupertype
             }
         }
 
-        // direction == TO_LOWER or there is no LOWER bounds
+        // direction == TO_SUPER or there is no LOWER bounds
         val upperConstraints = variableWithConstraints.constraints.filter { it.kind == ConstraintKind.UPPER && c.isProperType(it.type) }
         if (upperConstraints.isNotEmpty()) {
             return intersectTypes(upperConstraints.map { it.type })

compiler/resolution/src/org/jetbrains/kotlin/types/TypeApproximator.kt

@@ -16,14 +16,19 @@
 
 package org.jetbrains.kotlin.types
 
-import org.jetbrains.kotlin.types.TypeApproximatorConfiguration.IntersectionStrategy.*
 import org.jetbrains.kotlin.resolve.calls.components.CommonSupertypeCalculator
 import org.jetbrains.kotlin.resolve.calls.inference.model.TypeVariableTypeConstructor
+import org.jetbrains.kotlin.types.TypeApproximatorConfiguration.IntersectionStrategy.*
+import org.jetbrains.kotlin.types.checker.CaptureStatus.FOR_INCORPORATION
+import org.jetbrains.kotlin.types.checker.CaptureStatus.FROM_EXPRESSION
 import org.jetbrains.kotlin.types.checker.NewCapturedType
 import org.jetbrains.kotlin.types.checker.NewCapturedTypeConstructor
 import org.jetbrains.kotlin.types.checker.NewKotlinTypeChecker
 import org.jetbrains.kotlin.types.checker.intersectTypes
-import org.jetbrains.kotlin.types.typeUtil.*
+import org.jetbrains.kotlin.types.typeUtil.asTypeProjection
+import org.jetbrains.kotlin.types.typeUtil.builtIns
+import org.jetbrains.kotlin.types.typeUtil.isNothing
+import org.jetbrains.kotlin.types.typeUtil.isNullableAny
 
 
 open class TypeApproximatorConfiguration {
@@ -58,6 +63,25 @@ open class TypeApproximatorConfiguration {
     object PublicDeclaration : AllFlexibleSameValue() {
         override val allFlexible get() = true
     }
+
+    object IncorporationConfiguration : TypeApproximatorConfiguration.AllFlexibleSameValue() {
+        override val allFlexible get() = true
+
+        // i.e. will be approximated only FOR_INCORPORATION captured types
+        override val capturedType get() = { it: NewCapturedType -> it.captureStatus != FOR_INCORPORATION }
+        override val intersection get() = IntersectionStrategy.ALLOWED
+        override val typeVariable: (TypeVariableTypeConstructor) -> Boolean get() = { true }
+    }
+
+    object CapturedTypesApproximation : TypeApproximatorConfiguration.AllFlexibleSameValue() {
+        override val allFlexible get() = true
+
+        // i.e. will be approximated only FROM_EXPRESSION captured types
+        override val capturedType get() = { it: NewCapturedType -> it.captureStatus != FROM_EXPRESSION }
+        override val intersection get() = IntersectionStrategy.ALLOWED
+        override val typeVariable: (TypeVariableTypeConstructor) -> Boolean get() = { true }
+    }
+
 }
 
 class TypeApproximator(private val commonSupertypeCalculator: CommonSupertypeCalculator) {

compiler/testData/diagnostics/tests/inference/capturedTypes/approximateBeforeFixation.kt

@@ -0,0 +1,8 @@
+
+fun <T> Array<out T>.intersect(other: Iterable<T>) {
+    val set = toMutableSet()
+    set.retainAll(other)
+}
+
+fun <X> Array<out X>.toMutableSet(): MutableSet<X> = TODO()
+fun <Y> MutableCollection<in Y>.retainAll(<!UNUSED_PARAMETER!>elements<!>: Iterable<Y>) {}

compiler/testData/diagnostics/tests/inference/capturedTypes/approximateBeforeFixation.txt

@@ -0,0 +1,5 @@
+package
+
+public fun </*0*/ T> kotlin.Array<out T>.intersect(/*0*/ other: kotlin.collections.Iterable<T>): kotlin.Unit
+public fun </*0*/ Y> kotlin.collections.MutableCollection<in Y>.retainAll(/*0*/ elements: kotlin.collections.Iterable<Y>): kotlin.Unit
+public fun </*0*/ X> kotlin.Array<out X>.toMutableSet(): kotlin.collections.MutableSet<X>

compiler/tests/org/jetbrains/kotlin/checkers/DiagnosticsTestGenerated.java

@@ -10212,6 +10212,12 @@ public class DiagnosticsTestGenerated extends AbstractDiagnosticsTest {
                     KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("compiler/testData/diagnostics/tests/inference/capturedTypes"), Pattern.compile("^(.+)\\.kt$"), TargetBackend.ANY, true);
                 }
 
+                @TestMetadata("approximateBeforeFixation.kt")
+                public void testApproximateBeforeFixation() throws Exception {
+                    String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/inference/capturedTypes/approximateBeforeFixation.kt");
+                    doTest(fileName);
+                }
+
                 @TestMetadata("cannotCaptureInProjection.kt")
                 public void testCannotCaptureInProjection() throws Exception {
                     String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/inference/capturedTypes/cannotCaptureInProjection.kt");