[FE 1.0] Check callable reference return type safety during resolution

^KT-51844
^KT-52503 Fixed

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/DiagnosisCompilerTestFE10TestdataTestGenerated.java

@@ -2826,6 +2826,12 @@ public class DiagnosisCompilerTestFE10TestdataTestGenerated extends AbstractDiag
                 runTest("compiler/testData/diagnostics/tests/callableReference/kt49038.kt");
             }
 
+            @Test
+            @TestMetadata("kt52503.kt")
+            public void testKt52503() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/callableReference/kt52503.kt");
+            }
+
             @Test
             @TestMetadata("kt7430_wrongClassOnLHS.kt")
             public void testKt7430_wrongClassOnLHS() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsTestGenerated.java

@@ -2826,6 +2826,12 @@ public class FirOldFrontendDiagnosticsTestGenerated extends AbstractFirDiagnosti
                 runTest("compiler/testData/diagnostics/tests/callableReference/kt49038.kt");
             }
 
+            @Test
+            @TestMetadata("kt52503.kt")
+            public void testKt52503() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/callableReference/kt52503.kt");
+            }
+
             @Test
             @TestMetadata("kt7430_wrongClassOnLHS.kt")
             public void testKt7430_wrongClassOnLHS() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirOldFrontendDiagnosticsWithLightTreeTestGenerated.java

@@ -2826,6 +2826,12 @@ public class FirOldFrontendDiagnosticsWithLightTreeTestGenerated extends Abstrac
                 runTest("compiler/testData/diagnostics/tests/callableReference/kt49038.kt");
             }
 
+            @Test
+            @TestMetadata("kt52503.kt")
+            public void testKt52503() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/callableReference/kt52503.kt");
+            }
+
             @Test
             @TestMetadata("kt7430_wrongClassOnLHS.kt")
             public void testKt7430_wrongClassOnLHS() throws Exception {

compiler/fir/fir2ir/tests-gen/org/jetbrains/kotlin/test/runners/codegen/FirBlackBoxCodegenTestGenerated.java

@@ -3129,6 +3129,12 @@ public class FirBlackBoxCodegenTestGenerated extends AbstractFirBlackBoxCodegenT
             runTest("compiler/testData/codegen/box/callableReference/kt50172.kt");
         }
 
+        @Test
+        @TestMetadata("kt51844.kt")
+        public void testKt51844() throws Exception {
+            runTest("compiler/testData/codegen/box/callableReference/kt51844.kt");
+        }
+
         @Test
         @TestMetadata("nested.kt")
         public void testNested() throws Exception {

compiler/frontend/src/org/jetbrains/kotlin/util/ReenteringLazyValueComputationException.java

@@ -19,8 +19,9 @@ package org.jetbrains.kotlin.util;
 import com.intellij.openapi.application.Application;
 import com.intellij.openapi.application.ApplicationManager;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.kotlin.types.error.LazyWrappedTypeComputationException;
 
-public class ReenteringLazyValueComputationException extends RuntimeException {
+public class ReenteringLazyValueComputationException extends LazyWrappedTypeComputationException {
     public ReenteringLazyValueComputationException() {
     }
 

compiler/resolution/src/org/jetbrains/kotlin/resolve/calls/model/CallableReferencesCandidateFactory.kt

@@ -27,6 +27,7 @@ import org.jetbrains.kotlin.types.*
 import org.jetbrains.kotlin.types.error.ErrorScopeKind
 import org.jetbrains.kotlin.types.error.ErrorUtils
 import org.jetbrains.kotlin.types.error.ErrorTypeKind
+import org.jetbrains.kotlin.types.error.LazyWrappedTypeComputationException
 import org.jetbrains.kotlin.types.expressions.CoercionStrategy
 import org.jetbrains.kotlin.types.typeUtil.isUnit
 import org.jetbrains.kotlin.utils.SmartList
@@ -239,8 +240,15 @@ class CallableReferencesCandidateFactory(
         // lower(Unit!) = Unit
         val returnExpectedType = inputOutputTypes.outputType
 
+        fun isReturnTypeNonUnitSafe(): Boolean =
+            try {
+                descriptor.returnType?.isUnit() == false
+            } catch (e: LazyWrappedTypeComputationException) {
+                false
+            }
+
         val coercion =
-            if (returnExpectedType.isUnit() && descriptor.returnType?.isUnit() == false)
+            if (returnExpectedType.isUnit() && isReturnTypeNonUnitSafe())
                 CoercionStrategy.COERCION_TO_UNIT
             else
                 CoercionStrategy.NO_COERCION

compiler/testData/codegen/box/callableReference/kt51844.kt

@@ -0,0 +1,15 @@
+// WITH_STDLIB
+
+abstract class Foo {
+    abstract fun contains(x: Int);
+}
+
+// ERROR: Type checking has run into a recursive problem. Easiest workaround: specify types of your declarations explicitly
+fun Foo.contains(vararg xs: Int) = xs.forEach(this::contains)
+
+fun box(): String {
+    object : Foo() {
+        override fun contains(x: Int) {}
+    }.contains(1)
+    return "OK"
+}

compiler/testData/diagnostics/tests/callableReference/kt52503.kt

@@ -0,0 +1,16 @@
+// FIR_IDENTICAL
+// WITH_STDLIB
+
+abstract class Foo
+
+object A {
+    fun Foo.contains(vararg xs: Int) = // 1
+        xs.forEach(this::<!EXTENSION_IN_CLASS_REFERENCE_NOT_ALLOWED!>contains<!>) // resolved to (2) in 1.7.0-RC, should be error "Type checking has run into a recursive problem"
+}
+
+fun Any.contains(vararg xs: Int) {} // 2
+
+fun box(): String {
+    object : Foo() {}.contains(1)
+    return "OK"
+}

compiler/testData/diagnostics/tests/callableReference/kt52503.txt

@@ -0,0 +1,19 @@
+package
+
+public fun box(): kotlin.String
+public fun kotlin.Any.contains(/*0*/ vararg xs: kotlin.Int /*kotlin.IntArray*/): kotlin.Unit
+
+public object A {
+    private constructor A()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+    public final fun Foo.contains(/*0*/ vararg xs: kotlin.Int /*kotlin.IntArray*/): kotlin.Unit
+}
+
+public abstract class Foo {
+    public constructor Foo()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/testData/diagnostics/tests/platformTypes/kt50877.fir.kt

@@ -1,31 +0,0 @@
-// FULL_JDK
-// WITH_STDLIB
-
-// FILE: Schematic.kt
-class Schematic {
-    var name: String? = null
-
-    var error: String? = null
-
-    override fun toString(): String {
-        return name!!
-    }
-}
-
-// FILE: SortedListModel.java
-import java.util.Comparator;
-
-public class SortedListModel<T> {
-    public SortedListModel(Comparator<? super T> comparator) {
-    }
-}
-
-
-// FILE: main.kt
-val model = SortedListModel<Schematic>(Comparator.comparing { b1: Schematic ->
-    when {
-        b1.error != null -> 2
-        b1.name!!.contains(":") -> 1
-        else -> 0
-    }
-}.thenComparing { b1: Schematic -> b1.name!! })

compiler/testData/diagnostics/tests/platformTypes/kt50877.kt

@@ -1,3 +1,4 @@
+// FIR_IDENTICAL
 // FULL_JDK
 // WITH_STDLIB
 

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/DiagnosticTestGenerated.java

@@ -2832,6 +2832,12 @@ public class DiagnosticTestGenerated extends AbstractDiagnosticTest {
                 runTest("compiler/testData/diagnostics/tests/callableReference/kt49038.kt");
             }
 
+            @Test
+            @TestMetadata("kt52503.kt")
+            public void testKt52503() throws Exception {
+                runTest("compiler/testData/diagnostics/tests/callableReference/kt52503.kt");
+            }
+
             @Test
             @TestMetadata("kt7430_wrongClassOnLHS.kt")
             public void testKt7430_wrongClassOnLHS() throws Exception {

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/BlackBoxCodegenTestGenerated.java

@@ -3033,6 +3033,12 @@ public class BlackBoxCodegenTestGenerated extends AbstractBlackBoxCodegenTest {
             runTest("compiler/testData/codegen/box/callableReference/kt50172.kt");
         }
 
+        @Test
+        @TestMetadata("kt51844.kt")
+        public void testKt51844() throws Exception {
+            runTest("compiler/testData/codegen/box/callableReference/kt51844.kt");
+        }
+
         @Test
         @TestMetadata("nested.kt")
         public void testNested() throws Exception {

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/IrBlackBoxCodegenTestGenerated.java

@@ -3129,6 +3129,12 @@ public class IrBlackBoxCodegenTestGenerated extends AbstractIrBlackBoxCodegenTes
             runTest("compiler/testData/codegen/box/callableReference/kt50172.kt");
         }
 
+        @Test
+        @TestMetadata("kt51844.kt")
+        public void testKt51844() throws Exception {
+            runTest("compiler/testData/codegen/box/callableReference/kt51844.kt");
+        }
+
         @Test
         @TestMetadata("nested.kt")
         public void testNested() throws Exception {

compiler/tests-gen/org/jetbrains/kotlin/codegen/LightAnalysisModeTestGenerated.java

@@ -2658,6 +2658,11 @@ public class LightAnalysisModeTestGenerated extends AbstractLightAnalysisModeTes
             runTest("compiler/testData/codegen/box/callableReference/kt50172.kt");
         }
 
+        @TestMetadata("kt51844.kt")
+        public void testKt51844() throws Exception {
+            runTest("compiler/testData/codegen/box/callableReference/kt51844.kt");
+        }
+
         @TestMetadata("nested.kt")
         public void testNested() throws Exception {
             runTest("compiler/testData/codegen/box/callableReference/nested.kt");

core/descriptors/src/org/jetbrains/kotlin/types/error/LazyWrappedTypeComputationException.kt

@@ -0,0 +1,8 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.types.error
+
+abstract class LazyWrappedTypeComputationException : RuntimeException()

js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/JsCodegenBoxTestGenerated.java

@@ -1983,6 +1983,12 @@ public class JsCodegenBoxTestGenerated extends AbstractJsCodegenBoxTest {
             runTest("compiler/testData/codegen/box/callableReference/kt50172.kt");
         }
 
+        @Test
+        @TestMetadata("kt51844.kt")
+        public void testKt51844() throws Exception {
+            runTest("compiler/testData/codegen/box/callableReference/kt51844.kt");
+        }
+
         @Test
         @TestMetadata("nested.kt")
         public void testNested() throws Exception {

js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/ir/IrJsCodegenBoxTestGenerated.java

@@ -2025,6 +2025,12 @@ public class IrJsCodegenBoxTestGenerated extends AbstractIrJsCodegenBoxTest {
             runTest("compiler/testData/codegen/box/callableReference/kt50172.kt");
         }
 
+        @Test
+        @TestMetadata("kt51844.kt")
+        public void testKt51844() throws Exception {
+            runTest("compiler/testData/codegen/box/callableReference/kt51844.kt");
+        }
+
         @Test
         @TestMetadata("nested.kt")
         public void testNested() throws Exception {

js/js.tests/tests-gen/org/jetbrains/kotlin/js/testOld/wasm/semantics/IrCodegenBoxWasmTestGenerated.java

@@ -1808,6 +1808,11 @@ public class IrCodegenBoxWasmTestGenerated extends AbstractIrCodegenBoxWasmTest
             runTest("compiler/testData/codegen/box/callableReference/kt50172.kt");
         }
 
+        @TestMetadata("kt51844.kt")
+        public void testKt51844() throws Exception {
+            runTest("compiler/testData/codegen/box/callableReference/kt51844.kt");
+        }
+
         @TestMetadata("nested.kt")
         public void testNested() throws Exception {
             runTest("compiler/testData/codegen/box/callableReference/nested.kt");

native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/NativeCodegenBoxTestGenerated.java

@@ -2077,6 +2077,12 @@ public class NativeCodegenBoxTestGenerated extends AbstractNativeCodegenBoxTest
                 runTest("compiler/testData/codegen/box/callableReference/kt50172.kt");
             }
 
+            @Test
+            @TestMetadata("kt51844.kt")
+            public void testKt51844() throws Exception {
+                runTest("compiler/testData/codegen/box/callableReference/kt51844.kt");
+            }
+
             @Test
             @TestMetadata("nested.kt")
             public void testNested() throws Exception {