Introduce AdaptedFunctionReference runtime class

It's used as a superclass for anonymous classes for adapted function
references. Its main feature is that it _doesn't_ inherit from KFunction
(as opposed to FunctionReference), as per the decision to postpone
reflection support for adapted function references in KT-36024.

 #KT-36024 Fixed

compiler/backend/src/org/jetbrains/kotlin/codegen/ClosureCodegen.java

@@ -74,7 +74,7 @@ public class ClosureCodegen extends MemberCodegen<KtElement> {
     protected final Type asmType;
     protected final int visibilityFlag;
     private final boolean shouldHaveBoundReferenceReceiver;
-    private final boolean isRegularFunctionReference;
+    private final boolean isLegacyFunctionReference;
     private final boolean isOptimizedFunctionReference;
     private final boolean isAdaptedFunctionReference;
 
@@ -129,7 +129,7 @@ public class ClosureCodegen extends MemberCodegen<KtElement> {
         this.shouldHaveBoundReferenceReceiver = CallableReferenceUtilKt.isForBoundCallableReference(closure);
 
         ClassifierDescriptor superClassDescriptor = superClassType.getConstructor().getDeclarationDescriptor();
-        this.isRegularFunctionReference =
+        this.isLegacyFunctionReference =
                 functionReferenceTarget != null &&
                 superClassDescriptor == state.getJvmRuntimeTypes().getFunctionReference();
         this.isOptimizedFunctionReference =
@@ -137,7 +137,7 @@ public class ClosureCodegen extends MemberCodegen<KtElement> {
                 superClassDescriptor == state.getJvmRuntimeTypes().getFunctionReferenceImpl();
         this.isAdaptedFunctionReference =
                 functionReferenceTarget != null &&
-                superClassDescriptor == state.getJvmRuntimeTypes().getLambda();
+                superClassDescriptor == state.getJvmRuntimeTypes().getAdaptedFunctionReference();
 
         this.asmType = typeMapper.mapClass(classDescriptor);
 
@@ -199,29 +199,15 @@ public class ClosureCodegen extends MemberCodegen<KtElement> {
     protected void generateClosureBody() {
         functionCodegen.generateMethod(JvmDeclarationOriginKt.OtherOrigin(element, funDescriptor), funDescriptor, strategy);
 
-        if (isRegularFunctionReference) {
+        if (isLegacyFunctionReference) {
             generateFunctionReferenceMethods(functionReferenceTarget);
         }
 
-        if (shouldHaveBoundReferenceReceiver && isAdaptedFunctionReference) {
-            generateBoundAdaptedCallableReferenceReceiverField();
-        }
-
         functionCodegen.generateDefaultIfNeeded(
                 context.intoFunction(funDescriptor), funDescriptor, context.getContextKind(), DefaultParameterValueLoader.DEFAULT, null
         );
     }
 
-    private void generateBoundAdaptedCallableReferenceReceiverField() {
-        v.newField(
-                JvmDeclarationOriginKt.OtherOrigin(element, funDescriptor),
-                ACC_PRIVATE,
-                BOUND_REFERENCE_RECEIVER,
-                OBJECT_TYPE.getDescriptor(),
-                null, null
-        );
-    }
-
     protected void generateBridges() {
         FunctionDescriptor erasedInterfaceFunction;
         if (samType == null) {
@@ -522,20 +508,19 @@ public class ClosureCodegen extends MemberCodegen<KtElement> {
             iv.load(0, superClassAsmType);
 
             List<Type> superCtorArgTypes = new ArrayList<>();
-            if (superClassAsmType.equals(LAMBDA) || superClassAsmType.equals(FUNCTION_REFERENCE) ||
+            if (superClassAsmType.equals(LAMBDA) || functionReferenceTarget != null ||
-                superClassAsmType.equals(FUNCTION_REFERENCE_IMPL) ||
                 CoroutineCodegenUtilKt.isCoroutineSuperClass(state.getLanguageVersionSettings(), superClassAsmType.getInternalName())
             ) {
                 int arity = calculateArity();
                 iv.iconst(arity);
                 superCtorArgTypes.add(Type.INT_TYPE);
-                if (shouldHaveBoundReferenceReceiver && !isAdaptedFunctionReference) {
+                if (shouldHaveBoundReferenceReceiver) {
                     CallableReferenceUtilKt.loadBoundReferenceReceiverParameter(
                             iv, boundReceiverParameterIndex, boundReceiverType, boundReceiverKotlinType
                     );
                     superCtorArgTypes.add(OBJECT_TYPE);
                 }
-                if (isOptimizedFunctionReference) {
+                if (isOptimizedFunctionReference || isAdaptedFunctionReference) {
                     assert functionReferenceTarget != null : "No function reference target: " + funDescriptor;
                     generateCallableReferenceDeclarationContainerClass(iv, functionReferenceTarget, state);
                     iv.aconst(functionReferenceTarget.getName().asString());
@@ -558,19 +543,6 @@ public class ClosureCodegen extends MemberCodegen<KtElement> {
                     Type.getMethodDescriptor(Type.VOID_TYPE, superCtorArgTypes.toArray(new Type[0])), false
             );
 
-            // Bound adapted function references store receiver in a separate field.
-            if (shouldHaveBoundReferenceReceiver && isAdaptedFunctionReference) {
-                iv.load(0, superClassAsmType);
-                CallableReferenceUtilKt.loadBoundReferenceReceiverParameter(
-                        iv, boundReceiverParameterIndex, boundReceiverType, boundReceiverKotlinType
-                );
-                iv.putfield(
-                        asmType.getInternalName(),
-                        BOUND_REFERENCE_RECEIVER,
-                        OBJECT_TYPE.getDescriptor()
-                );
-            }
-
             iv.visitInsn(RETURN);
 
             FunctionCodegen.endVisit(iv, "constructor", element);

compiler/backend/src/org/jetbrains/kotlin/codegen/JvmRuntimeTypes.kt

@@ -43,9 +43,10 @@ class JvmRuntimeTypes(
     private fun propertyClasses(prefix: String, suffix: String): Lazy<List<ClassDescriptor>> =
         lazy { (0..2).map { i -> createClass(kotlinJvmInternalPackage, prefix + i + suffix) } }
 
-    val lambda: ClassDescriptor by internal("Lambda")
+    private val lambda: ClassDescriptor by internal("Lambda")
     val functionReference: ClassDescriptor by internal("FunctionReference")
     val functionReferenceImpl: ClassDescriptor by internal("FunctionReferenceImpl")
+    val adaptedFunctionReference: ClassDescriptor by internal("AdaptedFunctionReference")
 
     private val localVariableReference: ClassDescriptor by internal("LocalVariableReference")
     private val mutableLocalVariableReference: ClassDescriptor by internal("MutableLocalVariableReference")
@@ -152,7 +153,7 @@ class JvmRuntimeTypes(
 
         val suspendFunctionType = if (referencedFunction.isSuspend) suspendFunctionInterface?.defaultType else null
         val superClass = when {
-            isAdaptedCallableReference -> lambda
+            isAdaptedCallableReference -> adaptedFunctionReference
             generateOptimizedCallableReferenceSuperClasses -> functionReferenceImpl
             else -> functionReference
         }

compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/JvmSymbols.kt

@@ -21,6 +21,7 @@ import org.jetbrains.kotlin.descriptors.impl.EmptyPackageFragmentDescriptor
 import org.jetbrains.kotlin.ir.builders.declarations.*
 import org.jetbrains.kotlin.ir.declarations.IrClass
 import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
+import org.jetbrains.kotlin.ir.declarations.IrFunction
 import org.jetbrains.kotlin.ir.declarations.IrPackageFragment
 import org.jetbrains.kotlin.ir.declarations.impl.IrExternalPackageFragmentImpl
 import org.jetbrains.kotlin.ir.symbols.*
@@ -314,21 +315,33 @@ class JvmSymbols(
             klass.superTypes = listOf(functionReference.defaultType)
 
             if (generateOptimizedCallableReferenceSuperClasses) {
-                for (hasBoundReceiver in listOf(false, true)) {
+                klass.generateCallableReferenceSuperclassConstructors(withArity = true)
-                    klass.addConstructor().apply {
-                        addValueParameter("arity", irBuiltIns.intType)
-                        if (hasBoundReceiver) {
-                            addValueParameter("receiver", irBuiltIns.anyNType)
-                        }
-                        addValueParameter("owner", javaLangClass.starProjectedType)
-                        addValueParameter("name", irBuiltIns.stringType)
-                        addValueParameter("signature", irBuiltIns.stringType)
-                        addValueParameter("flags", irBuiltIns.intType)
-                    }
-                }
             }
         }
 
+    val adaptedFunctionReference: IrClassSymbol =
+        createClass(FqName("kotlin.jvm.internal.AdaptedFunctionReference"), classModality = Modality.OPEN) { klass ->
+            klass.superTypes = listOf(irBuiltIns.anyType)
+            klass.generateCallableReferenceSuperclassConstructors(withArity = true)
+        }
+
+    private fun IrClass.generateCallableReferenceSuperclassConstructors(withArity: Boolean) {
+        for (hasBoundReceiver in listOf(false, true)) {
+            addConstructor().apply {
+                if (withArity) {
+                    addValueParameter("arity", irBuiltIns.intType)
+                }
+                if (hasBoundReceiver) {
+                    addValueParameter("receiver", irBuiltIns.anyNType)
+                }
+                addValueParameter("owner", javaLangClass.starProjectedType)
+                addValueParameter("name", irBuiltIns.stringType)
+                addValueParameter("signature", irBuiltIns.stringType)
+                addValueParameter("flags", irBuiltIns.intType)
+            }
+        }
+    }
+
     fun getFunction(parameterCount: Int): IrClassSymbol =
         symbolTable.referenceClass(builtIns.getFunction(parameterCount))
 
@@ -403,17 +416,7 @@ class JvmSymbols(
                     }
 
                     if (generateOptimizedCallableReferenceSuperClasses) {
-                        for (hasBoundReceiver in listOf(false, true)) {
+                        klass.generateCallableReferenceSuperclassConstructors(withArity = false)
-                            klass.addConstructor().apply {
-                                if (hasBoundReceiver) {
-                                    addValueParameter("receiver", irBuiltIns.anyNType)
-                                }
-                                addValueParameter("owner", javaLangClass.starProjectedType)
-                                addValueParameter("name", irBuiltIns.stringType)
-                                addValueParameter("signature", irBuiltIns.stringType)
-                                addValueParameter("flags", irBuiltIns.intType)
-                            }
-                        }
                     }
 
                     klass.superTypes += getPropertyReferenceClass(mutable, parameterCount, false).defaultType

compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/lower/FunctionReferenceLowering.kt

@@ -26,6 +26,7 @@ import org.jetbrains.kotlin.ir.expressions.*
 import org.jetbrains.kotlin.ir.expressions.impl.IrClassReferenceImpl
 import org.jetbrains.kotlin.ir.expressions.impl.IrFunctionReferenceImpl
 import org.jetbrains.kotlin.ir.expressions.impl.IrInstanceInitializerCallImpl
+import org.jetbrains.kotlin.ir.symbols.IrFunctionSymbol
 import org.jetbrains.kotlin.ir.types.*
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.ir.visitors.transformChildrenVoid
@@ -119,10 +120,32 @@ internal class FunctionReferenceLowering(private val context: JvmBackendContext)
                     context.ir.symbols.getJvmFunctionClass(argumentTypes.size)
         private val superMethod =
             functionSuperClass.functions.single { it.owner.modality == Modality.ABSTRACT }
+
         private val useOptimizedSuperClass =
             context.state.generateOptimizedCallableReferenceSuperClasses
+
+        private val adaptedReferenceOriginalTarget = if (callee.origin == IrDeclarationOrigin.ADAPTER_FOR_CALLABLE_REFERENCE) {
+            // The body of a callable reference adapter contains either only a call, or an IMPLICIT_COERCION_TO_UNIT type operator
+            // applied to a call. That call's target is the original function which we need to get owner/name/signature.
+            val call = when (val statement = callee.body!!.statements.single()) {
+                is IrTypeOperatorCall -> {
+                    assert(statement.operator == IrTypeOperator.IMPLICIT_COERCION_TO_UNIT) {
+                        "Unexpected type operator in ADAPTER_FOR_CALLABLE_REFERENCE: ${callee.render()}"
+                    }
+                    statement.argument
+                }
+                is IrReturn -> statement.value
+                else -> statement
+            }
+            if (call !is IrFunctionAccessExpression) {
+                throw UnsupportedOperationException("Unknown structure of ADAPTER_FOR_CALLABLE_REFERENCE: ${callee.render()}")
+            }
+            call.symbol.owner
+        } else null
+
         private val superType =
             samSuperType ?: when {
+                adaptedReferenceOriginalTarget != null -> context.ir.symbols.adaptedFunctionReference
                 isLambda -> context.ir.symbols.lambdaClass
                 useOptimizedSuperClass -> context.ir.symbols.functionReferenceImpl
                 else -> context.ir.symbols.functionReference
@@ -171,7 +194,7 @@ internal class FunctionReferenceLowering(private val context: JvmBackendContext)
 
                 if (!isLambda && samSuperType == null && !useOptimizedSuperClass) {
                     createLegacyMethodOverride(irSymbols.functionReferenceGetSignature.owner) {
-                        generateSignature()
+                        generateSignature(callee.symbol)
                     }
                     createLegacyMethodOverride(irSymbols.functionReferenceGetName.owner) {
                         irString(callee.originalName.asString())
@@ -215,9 +238,11 @@ internal class FunctionReferenceLowering(private val context: JvmBackendContext)
                 val constructor = if (samSuperType != null) {
                     context.irBuiltIns.anyClass.owner.constructors.single()
                 } else {
-                    val expectedArity =
+                    val expectedArity = when {
-                        if (isLambda) 1
+                        adaptedReferenceOriginalTarget != null -> 5 + (if (boundReceiver != null) 1 else 0)
-                        else 1 + (if (boundReceiver != null) 1 else 0) + (if (useOptimizedSuperClass) 4 else 0)
+                        isLambda -> 1
+                        else -> 1 + (if (boundReceiver != null) 1 else 0) + (if (useOptimizedSuperClass) 4 else 0)
+                    }
                     superType.getClass()!!.constructors.single {
                         it.valueParameters.size == expectedArity
                     }
@@ -232,13 +257,20 @@ internal class FunctionReferenceLowering(private val context: JvmBackendContext)
                                 if (boundReceiver != null) {
                                     putValueArgument(index++, irGet(valueParameters.first()))
                                 }
-                                if (!isLambda && useOptimizedSuperClass) {
+                                val callableReferenceTarget = when {
-                                    val owner = calculateOwnerKClass(callee.parent, backendContext)
+                                    adaptedReferenceOriginalTarget != null -> adaptedReferenceOriginalTarget
+                                    !isLambda && useOptimizedSuperClass -> callee
+                                    else -> null
+                                }
+                                if (callableReferenceTarget != null) {
+                                    val owner = calculateOwnerKClass(callableReferenceTarget.parent, backendContext)
                                     putValueArgument(index++, kClassToJavaClass(owner, backendContext))
-                                    putValueArgument(index++, irString(callee.originalName.asString()))
+                                    putValueArgument(index++, irString(callableReferenceTarget.originalName.asString()))
-                                    putValueArgument(index++, generateSignature())
+                                    putValueArgument(index++, generateSignature(callableReferenceTarget.symbol))
-                                    // TODO: use correct parents for adapted function references
+                                    putValueArgument(
-                                    putValueArgument(index, irInt(if (callee.parent.let { it is IrClass && it.isFileClass }) 1 else 0))
+                                        index,
+                                        irInt(if (callableReferenceTarget.parent.let { it is IrClass && it.isFileClass }) 1 else 0)
+                                    )
                                 }
                             }
                         }
@@ -346,14 +378,13 @@ internal class FunctionReferenceLowering(private val context: JvmBackendContext)
         private val IrFunction.originalName: Name
             get() = (metadata as? MetadataSource.Function)?.descriptor?.name ?: name
 
-        private fun JvmIrBuilder.generateSignature(): IrExpression =
+        private fun JvmIrBuilder.generateSignature(target: IrFunctionSymbol): IrExpression =
             irCall(backendContext.ir.symbols.signatureStringIntrinsic).apply {
                 putValueArgument(
                     0,
                     //don't pass receivers otherwise LocalDeclarationLowering will create additional captured parameters
                     IrFunctionReferenceImpl(
-                        UNDEFINED_OFFSET, UNDEFINED_OFFSET, irFunctionReference.type, irFunctionReference.symbol, 0,
+                        UNDEFINED_OFFSET, UNDEFINED_OFFSET, irFunctionReference.type, target, 0, irFunctionReference.reflectionTarget, null
-                        irFunctionReference.reflectionTarget, null
                     )
                 )
             }

compiler/testData/codegen/box/callableReference/equality/capturedDefaults.kt

@@ -1,4 +1,4 @@
-// IGNORE_BACKEND: JVM_IR, JS, JS_IR, NATIVE
+// IGNORE_BACKEND: JS, JS_IR, NATIVE
 // FILE: test.kt
 
 fun checkEqual(x: Any, y: Any) {

compiler/testData/codegen/box/callableReference/equality/capturedVararg.kt

@@ -1,4 +1,4 @@
-// IGNORE_BACKEND: JVM_IR, JS, JS_IR, NATIVE
+// IGNORE_BACKEND: JS, JS_IR, NATIVE
 // IGNORE_BACKEND_FIR: JVM_IR
 // FILE: test.kt
 

compiler/testData/codegen/box/callableReference/equality/coercionToUnit.kt

@@ -1,4 +1,4 @@
-// IGNORE_BACKEND: JVM_IR, JS, JS_IR, NATIVE
+// IGNORE_BACKEND: JS, JS_IR, NATIVE
 // IGNORE_BACKEND_FIR: JVM_IR
 // FILE: test.kt
 

compiler/testData/codegen/box/callableReference/equality/coercionToUnitWithDefaults.kt

@@ -1,4 +1,4 @@
-// IGNORE_BACKEND: JVM_IR, JS, JS_IR, NATIVE
+// IGNORE_BACKEND: JS, JS_IR, NATIVE
 // IGNORE_BACKEND_FIR: JVM_IR
 // FILE: test.kt
 

compiler/testData/codegen/box/callableReference/equality/coercionToUnitWithVararg.kt

@@ -1,4 +1,4 @@
-// IGNORE_BACKEND: JVM_IR, JS, JS_IR, NATIVE
+// IGNORE_BACKEND: JS, JS_IR, NATIVE
 // IGNORE_BACKEND_FIR: JVM_IR
 // FILE: test.kt
 

compiler/testData/codegen/box/callableReference/equality/varargWithDefaults.kt

@@ -1,4 +1,4 @@
-// IGNORE_BACKEND: JVM_IR, JS, JS_IR, NATIVE
+// IGNORE_BACKEND: JS, JS_IR, NATIVE
 // IGNORE_BACKEND_FIR: JVM_IR
 // FILE: test.kt
 

compiler/testData/codegen/box/callableReference/noReflectionForAdaptedCallableReferences.kt

@@ -30,9 +30,8 @@ fun box(): String {
     checkAny("::CWithDefaults", ::CWithDefaults)
     checkAny("::CWithVarargs", ::CWithVarargs)
 
-    // TODO KT-37604
+    checkUnit("::CWithDefaults", ::CWithDefaults)
-    // checkUnit("::CWithDefaults", ::CWithDefaults)
+    checkUnit("::CWithVarargs", ::CWithVarargs)
-    // checkUnit("::CWithVarargs", ::CWithVarargs)
 
     return "OK"
-}
+}

compiler/testData/codegen/box/reflection/methodsFromAny/adaptedCallableReferencesNotEqualToCallablesFromAPI.kt

@@ -1,6 +1,4 @@
 // TARGET_BACKEND: JVM
-// Temporarily ignored for JVM until KT-36024 is fixed.
-// IGNORE_BACKEND: JVM
 // IGNORE_BACKEND_FIR: JVM_IR
 // WITH_REFLECT
 

libraries/stdlib/jvm/runtime/kotlin/jvm/internal/AdaptedFunctionReference.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.jvm.internal;
+
+import kotlin.SinceKotlin;
+import kotlin.reflect.KDeclarationContainer;
+
+import static kotlin.jvm.internal.CallableReference.NO_RECEIVER;
+
+/**
+ * Superclass for instances of adapted function references, i.e. references where expected function type differs
+ * from the target function signature: <pre>
+ *     fun target(s: String? = ""): String = s!!
+ *     fun use(f: () -> Unit) {}
+ *     use(::target)  // adapted function reference (default argument conversion + coercion to Unit)
+ * </pre>
+ *
+ * It doesn't inherit from {@link FunctionReference} because such references don't support reflection yet.
+ * Once this changes in the future, the JVM codegen may simply use {@link FunctionReferenceImpl}
+ * for adapted function references instead of this class.
+ */
+@SuppressWarnings({"rawtypes", "WeakerAccess", "unused"})
+@SinceKotlin(version = "1.4")
+public class AdaptedFunctionReference {
+    protected final Object receiver;
+    private final Class owner;
+    private final String name;
+    private final String signature;
+    private final boolean isTopLevel;
+    private final int arity;
+    private final int flags;
+
+    public AdaptedFunctionReference(int arity, Class owner, String name, String signature, int flags) {
+        this(arity, NO_RECEIVER, owner, name, signature, flags);
+    }
+
+    public AdaptedFunctionReference(int arity, Object receiver, Class owner, String name, String signature, int flags) {
+        this.receiver = receiver;
+        this.owner = owner;
+        this.name = name;
+        this.signature = signature;
+        this.isTopLevel = (flags & 1) == 1;
+        this.arity = arity;
+        this.flags = flags >> 1;
+    }
+
+    public KDeclarationContainer getOwner() {
+        return owner == null ? null :
+               isTopLevel ? Reflection.getOrCreateKotlinPackage(owner) : Reflection.getOrCreateKotlinClass(owner);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof AdaptedFunctionReference)) return false;
+        AdaptedFunctionReference other = (AdaptedFunctionReference) o;
+        return isTopLevel == other.isTopLevel &&
+               arity == other.arity &&
+               flags == other.flags &&
+               Intrinsics.areEqual(receiver, other.receiver) &&
+               Intrinsics.areEqual(owner, other.owner) &&
+               name.equals(other.name) &&
+               signature.equals(other.signature);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = receiver != null ? receiver.hashCode() : 0;
+        result = result * 31 + (owner != null ? owner.hashCode() : 0);
+        result = result * 31 + name.hashCode();
+        result = result * 31 + signature.hashCode();
+        result = result * 31 + (isTopLevel ? 1231 : 1237);
+        result = result * 31 + arity;
+        result = result * 31 + flags;
+        return result;
+    }
+}

libraries/tools/binary-compatibility-validator/reference-public-api/kotlin-stdlib-runtime-merged.txt

@@ -3180,6 +3180,15 @@ public abstract interface class kotlin/jvm/functions/FunctionN : kotlin/Function
 	public abstract fun invoke ([Ljava/lang/Object;)Ljava/lang/Object;
 }
 
+public class kotlin/jvm/internal/AdaptedFunctionReference {
+	protected final field receiver Ljava/lang/Object;
+	public fun <init> (ILjava/lang/Class;Ljava/lang/String;Ljava/lang/String;I)V
+	public fun <init> (ILjava/lang/Object;Ljava/lang/Class;Ljava/lang/String;Ljava/lang/String;I)V
+	public fun equals (Ljava/lang/Object;)Z
+	public fun getOwner ()Lkotlin/reflect/KDeclarationContainer;
+	public fun hashCode ()I
+}
+
 public final class kotlin/jvm/internal/ArrayIteratorKt {
 	public static final fun iterator ([Ljava/lang/Object;)Ljava/util/Iterator;
 }