Add resolution status to report about unsuccessful smartcast

#KT-10248 Fixed
 #KT-11119 Fixed

compiler/frontend/src/org/jetbrains/kotlin/resolve/calls/CandidateResolver.kt

@@ -559,7 +559,7 @@ class CandidateResolver(
                 }
                 if (!smartCastResult.isCorrect) {
                     // Error about unstable smart cast reported within checkAndRecordPossibleCast
-                    return OTHER_ERROR
+                    return UNSTABLE_SMARTCAST_ERROR
                 }
             }
         }

compiler/frontend/src/org/jetbrains/kotlin/resolve/calls/results/ResolutionStatus.java

@@ -24,6 +24,7 @@ public enum ResolutionStatus {
     UNKNOWN_STATUS,
     UNSAFE_CALL_ERROR,
     WRONG_NUMBER_OF_TYPE_ARGUMENTS_ERROR,
+    UNSTABLE_SMARTCAST_ERROR,
     OTHER_ERROR,
     ARGUMENTS_MAPPING_ERROR,
     // '1.foo()' shouldn't be resolved to 'fun String.foo()'
@@ -40,6 +41,7 @@ public enum ResolutionStatus {
     public static final EnumSet<ResolutionStatus>[] SEVERITY_LEVELS = new EnumSet[] {
             EnumSet.of(UNSAFE_CALL_ERROR), // weakest
             EnumSet.of(WRONG_NUMBER_OF_TYPE_ARGUMENTS_ERROR),
+            EnumSet.of(UNSTABLE_SMARTCAST_ERROR),
             EnumSet.of(OTHER_ERROR),
             EnumSet.of(ARGUMENTS_MAPPING_ERROR),
             EnumSet.of(RECEIVER_TYPE_ERROR),

compiler/resolution/src/org/jetbrains/kotlin/resolve/calls/results/OverloadingConflictResolver.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2016 JetBrains s.r.o.
+ * Copyright 2010-2017 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,8 +20,10 @@ import gnu.trove.THashSet
 import gnu.trove.TObjectHashingStrategy
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.descriptors.CallableDescriptor
+import org.jetbrains.kotlin.descriptors.CallableMemberDescriptor
 import org.jetbrains.kotlin.descriptors.ScriptDescriptor
 import org.jetbrains.kotlin.descriptors.Visibilities
+import org.jetbrains.kotlin.descriptors.synthetic.SyntheticMemberDescriptor
 import org.jetbrains.kotlin.resolve.DescriptorEquivalenceForOverrides
 import org.jetbrains.kotlin.resolve.OverridingUtil
 import org.jetbrains.kotlin.resolve.calls.context.CheckArgumentTypesMode
@@ -70,7 +72,22 @@ class OverloadingConflictResolver<C : Any>(
         }
 
         val noEquivalentCalls = filterOutEquivalentCalls(fixedCandidates)
-        val noOverrides = OverridingUtil.filterOverrides(noEquivalentCalls) { it.resultingDescriptor }
+        val noOverrides = OverridingUtil.filterOverrides(noEquivalentCalls) { a, b ->
+            val aDescriptor = a.resultingDescriptor
+            val bDescriptor = b.resultingDescriptor
+            // Here we'd like to handle situation when we have two synthetic descriptors as in syntheticSAMExtensions.kt
+
+            // Without this, we'll pick all synthetic descriptors as they don't have overridden descriptors and
+            // then report ambiguity, which isn't very convenient
+            if (aDescriptor is SyntheticMemberDescriptor<*> && bDescriptor is SyntheticMemberDescriptor<*>) {
+                val aBaseDescriptor = aDescriptor.baseDescriptorForSynthetic
+                val bBaseDescriptor = bDescriptor.baseDescriptorForSynthetic
+                if (aBaseDescriptor is CallableMemberDescriptor && bBaseDescriptor is CallableMemberDescriptor) {
+                    return@filterOverrides Pair(aBaseDescriptor, bBaseDescriptor)
+                }
+            }
+            Pair(aDescriptor, bDescriptor)
+        }
         if (noOverrides.size == 1) {
             return noOverrides
         }

compiler/testData/diagnostics/tests/nullabilityAndSmartCasts/nullableReceiverWithOverloadedMethod.kt

@@ -0,0 +1,29 @@
+// !DIAGNOSTICS: -UNUSED_PARAMETER
+
+class A {
+    fun f(x: Boolean): Int = 0
+
+    fun f(y: String): Int = 0
+}
+
+class B {
+    private var a: A? = null
+
+    fun takeInt(i: Int) {}
+
+    fun f() {
+        a = A()
+        <!SMARTCAST_IMPOSSIBLE!>a<!>.f(true)
+        takeInt(<!SMARTCAST_IMPOSSIBLE!>a<!>.f(""))
+        a.<!NONE_APPLICABLE!>f<!>()
+    }
+
+    fun g() {
+        takeInt(if (3 > 2) {
+            a = A()
+            <!SMARTCAST_IMPOSSIBLE!>a<!>.f(true)
+        } else {
+            6
+        })
+    }
+}

compiler/testData/diagnostics/tests/nullabilityAndSmartCasts/nullableReceiverWithOverloadedMethod.txt

@@ -0,0 +1,21 @@
+package
+
+public final class A {
+    public constructor A()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public final fun f(/*0*/ x: kotlin.Boolean): kotlin.Int
+    public final fun f(/*0*/ y: kotlin.String): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public final class B {
+    public constructor B()
+    private final var a: A?
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public final fun f(): kotlin.Unit
+    public final fun g(): kotlin.Unit
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public final fun takeInt(/*0*/ i: kotlin.Int): kotlin.Unit
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/testData/diagnostics/tests/nullabilityAndSmartCasts/unstableSmartcastWhenOpenGetterWithOverloading.kt

@@ -0,0 +1,15 @@
+// !DIAGNOSTICS: -UNUSED_PARAMETER
+
+interface Ctx
+class CtxImpl : Ctx {
+    fun doJob(a: Int) {}
+    fun doJob(s: String) {}
+}
+
+open class Test(open val ctx: Ctx) {
+    fun test() {
+        when (ctx) {
+            is CtxImpl -> <!SMARTCAST_IMPOSSIBLE!>ctx<!>.doJob(2)
+        }
+    }
+}

compiler/testData/diagnostics/tests/nullabilityAndSmartCasts/unstableSmartcastWhenOpenGetterWithOverloading.txt

@@ -0,0 +1,25 @@
+package
+
+public interface Ctx {
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public final class CtxImpl : Ctx {
+    public constructor CtxImpl()
+    public final fun doJob(/*0*/ a: kotlin.Int): kotlin.Unit
+    public final fun doJob(/*0*/ s: kotlin.String): kotlin.Unit
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public open class Test {
+    public constructor Test(/*0*/ ctx: Ctx)
+    public open val ctx: Ctx
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public final fun test(): kotlin.Unit
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/testData/diagnostics/tests/nullabilityAndSmartCasts/unstableSmartcastWithOverloadedExtensions.kt

@@ -0,0 +1,14 @@
+// !DIAGNOSTICS: -UNUSED_PARAMETER
+
+class A
+class B
+
+fun A.foo(a: A) {}
+fun A.foo(b: B) {}
+var a: A? = null
+
+fun smartCastInterference(b: B) {
+    if (a != null) {
+        <!SMARTCAST_IMPOSSIBLE!>a<!>.foo(b)
+    }
+}

compiler/testData/diagnostics/tests/nullabilityAndSmartCasts/unstableSmartcastWithOverloadedExtensions.txt

@@ -0,0 +1,20 @@
+package
+
+public var a: A?
+public fun smartCastInterference(/*0*/ b: B): kotlin.Unit
+public fun A.foo(/*0*/ a: A): kotlin.Unit
+public fun A.foo(/*0*/ b: B): kotlin.Unit
+
+public final class A {
+    public constructor A()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public final class B {
+    public constructor B()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/testData/diagnostics/tests/scopes/protectedVisibility/syntheticSAMExtensions.kt

@@ -20,7 +20,7 @@ class B : A() {
         }
 
         if (d.x is B) {
-            <!SMARTCAST_IMPOSSIBLE!>d.x<!>.foo <!TYPE_MISMATCH!>{}<!>
+            <!SMARTCAST_IMPOSSIBLE!>d.x<!>.foo {}
         }
     }
 }

compiler/tests/org/jetbrains/kotlin/checkers/DiagnosticsTestGenerated.java

@@ -13912,6 +13912,12 @@ public class DiagnosticsTestGenerated extends AbstractDiagnosticsTest {
                 doTest(fileName);
             }
 
+            @TestMetadata("nullableReceiverWithOverloadedMethod.kt")
+            public void testNullableReceiverWithOverloadedMethod() throws Exception {
+                String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/nullabilityAndSmartCasts/nullableReceiverWithOverloadedMethod.kt");
+                doTest(fileName);
+            }
+
             @TestMetadata("PreferExtensionsOnNullableReceiver.kt")
             public void testPreferExtensionsOnNullableReceiver() throws Exception {
                 String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/nullabilityAndSmartCasts/PreferExtensionsOnNullableReceiver.kt");
@@ -13959,6 +13965,18 @@ public class DiagnosticsTestGenerated extends AbstractDiagnosticsTest {
                 String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/nullabilityAndSmartCasts/unnecessaryNotNullAssertion.kt");
                 doTest(fileName);
             }
+
+            @TestMetadata("unstableSmartcastWhenOpenGetterWithOverloading.kt")
+            public void testUnstableSmartcastWhenOpenGetterWithOverloading() throws Exception {
+                String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/nullabilityAndSmartCasts/unstableSmartcastWhenOpenGetterWithOverloading.kt");
+                doTest(fileName);
+            }
+
+            @TestMetadata("unstableSmartcastWithOverloadedExtensions.kt")
+            public void testUnstableSmartcastWithOverloadedExtensions() throws Exception {
+                String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/tests/nullabilityAndSmartCasts/unstableSmartcastWithOverloadedExtensions.kt");
+                doTest(fileName);
+            }
         }
 
         @TestMetadata("compiler/testData/diagnostics/tests/nullableTypes")

core/descriptors/src/org/jetbrains/kotlin/resolve/OverridingUtil.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2016 JetBrains s.r.o.
+ * Copyright 2010-2017 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,9 +16,11 @@
 
 package org.jetbrains.kotlin.resolve;
 
+import kotlin.Pair;
 import kotlin.Unit;
 import kotlin.collections.CollectionsKt;
 import kotlin.jvm.functions.Function1;
+import kotlin.jvm.functions.Function2;
 import org.jetbrains.annotations.Mutable;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -33,7 +35,6 @@ import org.jetbrains.kotlin.types.KotlinTypeKt;
 import org.jetbrains.kotlin.types.TypeConstructor;
 import org.jetbrains.kotlin.types.checker.KotlinTypeChecker;
 import org.jetbrains.kotlin.types.checker.KotlinTypeCheckerImpl;
-import org.jetbrains.kotlin.utils.FunctionsKt;
 import org.jetbrains.kotlin.utils.SmartSet;
 
 import java.util.*;
@@ -72,23 +73,29 @@ public class OverridingUtil {
      */
     @NotNull
     public static <D extends CallableDescriptor> Set<D> filterOutOverridden(@NotNull Set<D> candidateSet) {
-        return filterOverrides(candidateSet, FunctionsKt.<CallableDescriptor>identity());
+        return filterOverrides(candidateSet, new Function2<D, D, Pair<CallableDescriptor, CallableDescriptor>>() {
+            @Override
+            public Pair<CallableDescriptor, CallableDescriptor> invoke(D a, D b) {
+                return new Pair<CallableDescriptor, CallableDescriptor>(a, b);
+            }
+        });
     }
 
     @NotNull
     public static <D> Set<D> filterOverrides(
             @NotNull Set<D> candidateSet,
-            @NotNull Function1<? super D, ? extends CallableDescriptor> transform
+            @NotNull Function2<? super D, ? super D, Pair<CallableDescriptor, CallableDescriptor>> transformFirst
     ) {
         if (candidateSet.size() <= 1) return candidateSet;
 
         Set<D> result = new LinkedHashSet<D>();
         outerLoop:
         for (D meD : candidateSet) {
-            CallableDescriptor me = transform.invoke(meD);
             for (Iterator<D> iterator = result.iterator(); iterator.hasNext(); ) {
                 D otherD = iterator.next();
-                CallableDescriptor other = transform.invoke(otherD);
+                Pair<CallableDescriptor, CallableDescriptor> meAndOther = transformFirst.invoke(meD, otherD);
+                CallableDescriptor me = meAndOther.component1();
+                CallableDescriptor other = meAndOther.component2();
                 if (overrides(me, other)) {
                     iterator.remove();
                 }