Fix check for instance for types with compatible upper bounds

#KT-5246 Fixed

compiler/frontend/src/org/jetbrains/kotlin/types/TypeIntersector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2016 JetBrains s.r.o.
+ * Copyright 2010-2017 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -182,8 +182,8 @@ public class TypeIntersector {
                                parameterUsage.howTheTypeParameterIsUsed.superpose(howTheTypeIsUsedBefore));
                 return Unit.INSTANCE;
             };
-            processAllTypeParameters(withParameters, Variance.INVARIANT, processor);
-            processAllTypeParameters(expected, Variance.INVARIANT, processor);
+            processAllTypeParameters(withParameters, Variance.INVARIANT, processor, parameters::containsKey);
+            processAllTypeParameters(expected, Variance.INVARIANT, processor, parameters::containsKey);
             ConstraintSystem.Builder constraintSystem = new ConstraintSystemBuilderImpl();
             TypeSubstitutor substitutor = constraintSystem.registerTypeVariables(CallHandle.NONE.INSTANCE, parameters.keySet(), false);
             constraintSystem.addSubtypeConstraint(withParameters, substitutor.substitute(expected, Variance.INVARIANT), SPECIAL.position());
@@ -191,14 +191,25 @@ public class TypeIntersector {
             return constraintSystem.build().getStatus().isSuccessful();
         }
 
-        private static void processAllTypeParameters(KotlinType type, Variance howThisTypeIsUsed, Function1<TypeParameterUsage, Unit> result) {
+        private static void processAllTypeParameters(
+                KotlinType type,
+                Variance howThisTypeIsUsed,
+                Function1<TypeParameterUsage, Unit> result,
+                Function1<TypeParameterDescriptor, Boolean> containsParameter
+        ) {
             ClassifierDescriptor descriptor = type.getConstructor().getDeclarationDescriptor();
             if (descriptor instanceof TypeParameterDescriptor) {
+                if (containsParameter.invoke((TypeParameterDescriptor) descriptor)) return;
+
                 result.invoke(new TypeParameterUsage((TypeParameterDescriptor) descriptor, howThisTypeIsUsed));
+
+                for (KotlinType superType : type.getConstructor().getSupertypes()) {
+                    processAllTypeParameters(superType, howThisTypeIsUsed, result, containsParameter);
+                }
             }
             for (TypeProjection projection : type.getArguments()) {
                 if (projection.isStarProjection()) continue;
-                processAllTypeParameters(projection.getType(), projection.getProjectionKind(), result);
+                processAllTypeParameters(projection.getType(), projection.getProjectionKind(), result, containsParameter);
             }
         }
     }

compiler/testData/diagnostics/tests/cast/IsForTypeWithComplexUpperBound.kt

@@ -0,0 +1,34 @@
+interface A<T>
+class B : A<String> {}
+
+fun <T, U : A<T>> test1(a: U) {
+    a is B
+}
+
+fun <T, S : A<T>, U : S> test2(a: U) {
+    a is B
+}
+
+fun <T : A<String>, V : A<Int>> test3(a: T, b: V) {
+    a is B
+    b is <!INCOMPATIBLE_TYPES!>B<!>
+}
+
+fun <T, V : A<out T>> test4(a: T, b: V) {
+    a is B
+    b is B
+}
+
+interface Out<out T>
+class OutNothing : Out<Nothing>
+
+fun <T, S : Out<T>> test5(a: S) {
+    a is OutNothing
+}
+
+interface In<in T>
+class InNothing : In<Nothing>
+
+fun <T, S : In<T>> test6(a: S) {
+    a is InNothing
+}

compiler/testData/diagnostics/tests/cast/IsForTypeWithComplexUpperBound.txt

@@ -0,0 +1,47 @@
+package
+
+public fun </*0*/ T, /*1*/ U : A<T>> test1(/*0*/ a: U): kotlin.Unit
+public fun </*0*/ T, /*1*/ S : A<T>, /*2*/ U : S> test2(/*0*/ a: U): kotlin.Unit
+public fun </*0*/ T : A<kotlin.String>, /*1*/ V : A<kotlin.Int>> test3(/*0*/ a: T, /*1*/ b: V): kotlin.Unit
+public fun </*0*/ T, /*1*/ V : A<out T>> test4(/*0*/ a: T, /*1*/ b: V): kotlin.Unit
+public fun </*0*/ T, /*1*/ S : Out<T>> test5(/*0*/ a: S): kotlin.Unit
+public fun </*0*/ T, /*1*/ S : In<T>> test6(/*0*/ a: S): kotlin.Unit
+
+public interface A</*0*/ T> {
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public final class B : A<kotlin.String> {
+    public constructor B()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public interface In</*0*/ in T> {
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public final class InNothing : In<kotlin.Nothing> {
+    public constructor InNothing()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public interface Out</*0*/ out T> {
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public final class OutNothing : Out<kotlin.Nothing> {
+    public constructor OutNothing()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/testData/diagnostics/tests/cast/IsWithCycleUpperBounds.kt

@@ -0,0 +1,14 @@
+// check that there is no SOE on checking for instance
+interface Visitor<T>
+interface Acceptor<T>
+
+class Word : Acceptor<Visitor<Word>>
+
+class V : Visitor<Word>
+
+class S<T : Acceptor<U>, U : Visitor<T>>(val visitor: U, val acceptor: T) {
+    fun test() {
+        visitor is V
+        acceptor is Word
+    }
+}

compiler/testData/diagnostics/tests/cast/IsWithCycleUpperBounds.txt

@@ -0,0 +1,37 @@
+package
+
+public interface Acceptor</*0*/ T> {
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public final class S</*0*/ T : Acceptor<U>, /*1*/ U : Visitor<T>> {
+    public constructor S</*0*/ T : Acceptor<U>, /*1*/ U : Visitor<T>>(/*0*/ visitor: U, /*1*/ acceptor: T)
+    public final val acceptor: T
+    public final val visitor: U
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public final fun test(): kotlin.Unit
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public final class V : Visitor<Word> {
+    public constructor V()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public interface Visitor</*0*/ T> {
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public final class Word : Acceptor<Visitor<Word>> {
+    public constructor Word()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/tests/org/jetbrains/kotlin/checkers/DiagnosticsTestGenerated.java

@@ -2902,6 +2902,12 @@ public class DiagnosticsTestGenerated extends AbstractDiagnosticsTest {
                 doTest(fileName);
             }
 
+            @TestMetadata("IsForTypeWithComplexUpperBound.kt")
+            public void testIsForTypeWithComplexUpperBound() throws Exception {
+                String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/cast/IsForTypeWithComplexUpperBound.kt");
+                doTest(fileName);
+            }
+
             @TestMetadata("IsRecursionSustainable.kt")
             public void testIsRecursionSustainable() throws Exception {
                 String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/cast/IsRecursionSustainable.kt");
@@ -2914,6 +2920,12 @@ public class DiagnosticsTestGenerated extends AbstractDiagnosticsTest {
                 doTest(fileName);
             }
 
+            @TestMetadata("IsWithCycleUpperBounds.kt")
+            public void testIsWithCycleUpperBounds() throws Exception {
+                String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/cast/IsWithCycleUpperBounds.kt");
+                doTest(fileName);
+            }
+
             @TestMetadata("kt614.kt")
             public void testKt614() throws Exception {
                 String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/cast/kt614.kt");