[NI] Fix ambiguity when there are no applicable candidates

#KT-35064 Fixed

compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/FirOldFrontendDiagnosticsTestGenerated.java

@@ -1884,6 +1884,11 @@ public class FirOldFrontendDiagnosticsTestGenerated extends AbstractFirOldFronte
             runTest("compiler/testData/diagnostics/tests/callableReference/memberExtensionsImportedFromObjectsUnsupported.kt");
         }
 
+        @TestMetadata("noAmbiguityWhenAllReferencesAreInapplicable.kt")
+        public void testNoAmbiguityWhenAllReferencesAreInapplicable() throws Exception {
+            runTest("compiler/testData/diagnostics/tests/callableReference/noAmbiguityWhenAllReferencesAreInapplicable.kt");
+        }
+
         @TestMetadata("noExceptionOnRedCodeWithArrayLikeCall.kt")
         public void testNoExceptionOnRedCodeWithArrayLikeCall() throws Exception {
             runTest("compiler/testData/diagnostics/tests/callableReference/noExceptionOnRedCodeWithArrayLikeCall.kt");

compiler/frontend/src/org/jetbrains/kotlin/resolve/calls/DiagnosticReporterByTrackingStrategy.kt

@@ -148,7 +148,7 @@ class DiagnosticReporterByTrackingStrategy(
                 val expression = ambiguityDiagnostic.argument.psiExpression.safeAs<KtCallableReferenceExpression>()
                 val candidates = ambiguityDiagnostic.candidates.map { it.candidate }
                 reportIfNonNull(expression) {
-                    trace.report(CALLABLE_REFERENCE_RESOLUTION_AMBIGUITY.on(it.callableReference, candidates))
+                    trace.reportDiagnosticOnce(CALLABLE_REFERENCE_RESOLUTION_AMBIGUITY.on(it.callableReference, candidates))
                     trace.record(BindingContext.AMBIGUOUS_REFERENCE_TARGET, it.callableReference, candidates)
                 }
             }

compiler/resolution/src/org/jetbrains/kotlin/resolve/calls/components/CallableReferenceResolver.kt

@@ -30,6 +30,7 @@ import org.jetbrains.kotlin.resolve.calls.results.PlatformOverloadsSpecificityCo
 import org.jetbrains.kotlin.resolve.calls.results.TypeSpecificityComparator
 import org.jetbrains.kotlin.resolve.calls.tower.ImplicitScopeTower
 import org.jetbrains.kotlin.resolve.calls.tower.TowerResolver
+import org.jetbrains.kotlin.resolve.calls.tower.isInapplicable
 import org.jetbrains.kotlin.types.UnwrappedType
 
 
@@ -79,6 +80,10 @@ class CallableReferenceResolver(
         }
 
         if (candidates.size > 1 && resolvedAtom is EagerCallableReferenceAtom) {
+            if (candidates.all { it.resultingApplicability.isInapplicable }) {
+                diagnosticsHolder.addDiagnostic(CallableReferenceCandidatesAmbiguity(argument, candidates))
+            }
+
             resolvedAtom.setAnalyzedResults(
                 candidate = null,
                 subResolvedAtoms = listOf(resolvedAtom.transformToPostponed())

compiler/testData/diagnostics/tests/callableReference/noAmbiguityWhenAllReferencesAreInapplicable.fir.kt

@@ -0,0 +1,20 @@
+// !DIAGNOSTICS: -UNUSED_PARAMETER -UNUSED_VARIABLE
+// !LANGUAGE: +NewInference
+
+import kotlin.reflect.KFunction1
+import kotlin.reflect.KFunction2
+
+object Foo {
+    fun <T> bar(fn: KFunction1<T, Boolean>): String = ""
+    fun <T, U> bar(fn: KFunction2<T, U, Boolean>): Int = 10
+}
+
+class A
+class B
+
+fun A.test() = true // everything is OK without this line
+fun B.test() = true
+
+fun main() {
+    val x = Foo.bar(B::test) // ambiguity in NI
+}

compiler/testData/diagnostics/tests/callableReference/noAmbiguityWhenAllReferencesAreInapplicable.kt

@@ -0,0 +1,20 @@
+// !DIAGNOSTICS: -UNUSED_PARAMETER -UNUSED_VARIABLE
+// !LANGUAGE: +NewInference
+
+import kotlin.reflect.KFunction1
+import kotlin.reflect.KFunction2
+
+object Foo {
+    fun <T> bar(fn: KFunction1<T, Boolean>): String = ""
+    fun <T, U> bar(fn: KFunction2<T, U, Boolean>): Int = 10
+}
+
+class A
+class B
+
+fun A.test() = true // everything is OK without this line
+fun B.test() = true
+
+fun main() {
+    val x = Foo.bar(B::test) // ambiguity in NI
+}

compiler/testData/diagnostics/tests/callableReference/noAmbiguityWhenAllReferencesAreInapplicable.txt

@@ -0,0 +1,28 @@
+package
+
+public fun main(): kotlin.Unit
+public fun A.test(): kotlin.Boolean
+public fun B.test(): kotlin.Boolean
+
+public final class A {
+    public constructor A()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public final class B {
+    public constructor B()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}
+
+public object Foo {
+    private constructor Foo()
+    public final fun </*0*/ T> bar(/*0*/ fn: kotlin.reflect.KFunction1<T, kotlin.Boolean>): kotlin.String
+    public final fun </*0*/ T, /*1*/ U> bar(/*0*/ fn: kotlin.reflect.KFunction2<T, U, kotlin.Boolean>): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/testData/diagnostics/tests/callableReference/resolve/multipleOutersAndMultipleCallableReferences.kt

@@ -12,5 +12,5 @@ fun <T, R : Number> bar(f: (T) -> Inv<R>, p: String = "") {}
 fun <T, R : Base> bar(f: (T) -> Inv<R>, p: Int = 4) {}
 
 fun test() {
-    <!OVERLOAD_RESOLUTION_AMBIGUITY!>bar<!>(::<!DEBUG_INFO_MISSING_UNRESOLVED!>foo<!>)
+    bar(::foo)
 }

compiler/tests/org/jetbrains/kotlin/checkers/DiagnosticsTestGenerated.java

@@ -1891,6 +1891,11 @@ public class DiagnosticsTestGenerated extends AbstractDiagnosticsTest {
                 runTest("compiler/testData/diagnostics/tests/callableReference/memberExtensionsImportedFromObjectsUnsupported.kt");
             }
 
+            @TestMetadata("noAmbiguityWhenAllReferencesAreInapplicable.kt")
+            public void testNoAmbiguityWhenAllReferencesAreInapplicable() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/callableReference/noAmbiguityWhenAllReferencesAreInapplicable.kt");
+            }
+
             @TestMetadata("noExceptionOnRedCodeWithArrayLikeCall.kt")
             public void testNoExceptionOnRedCodeWithArrayLikeCall() throws Exception {
                 runTest("compiler/testData/diagnostics/tests/callableReference/noExceptionOnRedCodeWithArrayLikeCall.kt");

compiler/tests/org/jetbrains/kotlin/checkers/javac/DiagnosticsUsingJavacTestGenerated.java

@@ -1886,6 +1886,11 @@ public class DiagnosticsUsingJavacTestGenerated extends AbstractDiagnosticsUsing
                 runTest("compiler/testData/diagnostics/tests/callableReference/memberExtensionsImportedFromObjectsUnsupported.kt");
             }
 
+            @TestMetadata("noAmbiguityWhenAllReferencesAreInapplicable.kt")
+            public void testNoAmbiguityWhenAllReferencesAreInapplicable() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/callableReference/noAmbiguityWhenAllReferencesAreInapplicable.kt");
+            }
+
             @TestMetadata("noExceptionOnRedCodeWithArrayLikeCall.kt")
             public void testNoExceptionOnRedCodeWithArrayLikeCall() throws Exception {
                 runTest("compiler/testData/diagnostics/tests/callableReference/noExceptionOnRedCodeWithArrayLikeCall.kt");