Refine coroutine/suspend functions ABI

- `invoke` method must always start a suspend functions
- For creation of coroutine that has not been started yet, there are
two special internal interfaces `SuspendFunction0`/`SuspendFunction1`

compiler/backend/src/org/jetbrains/kotlin/codegen/ClosureCodegen.java

@@ -304,7 +304,7 @@ public class ClosureCodegen extends MemberCodegen<KtElement> {
         );
     }
 
-    private void generateBridge(@NotNull Method bridge, @NotNull Method delegate) {
+    protected void generateBridge(@NotNull Method bridge, @NotNull Method delegate) {
         if (bridge.equals(delegate)) return;
 
         MethodVisitor mv =

compiler/backend/src/org/jetbrains/kotlin/codegen/JvmRuntimeTypes.kt

@@ -17,6 +17,7 @@
 package org.jetbrains.kotlin.codegen
 
 import org.jetbrains.kotlin.builtins.createFunctionType
+import org.jetbrains.kotlin.builtins.isExtensionFunctionType
 import org.jetbrains.kotlin.codegen.coroutines.createJvmSuspendFunctionView
 import org.jetbrains.kotlin.coroutines.isSuspendLambda
 import org.jetbrains.kotlin.descriptors.*
@@ -28,6 +29,7 @@ import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.descriptorUtil.builtIns
 import org.jetbrains.kotlin.types.KotlinType
 import org.jetbrains.kotlin.types.expressions.ExpressionTypingUtils
+import org.jetbrains.kotlin.utils.addIfNotNull
 
 class JvmRuntimeTypes(module: ModuleDescriptor) {
     private val kotlinJvmInternalPackage = MutablePackageFragmentDescriptor(module, FqName("kotlin.jvm.internal"))
@@ -48,8 +50,16 @@ class JvmRuntimeTypes(module: ModuleDescriptor) {
         (0..2).map { i -> createClass(kotlinJvmInternalPackage, "MutablePropertyReference$i") }
     }
 
-    private fun createClass(packageFragment: PackageFragmentDescriptor, name: String): ClassDescriptor =
+    private val suspendFunctions: List<ClassDescriptor> by lazy {
-            MutableClassDescriptor(packageFragment, ClassKind.CLASS, /* isInner = */ false, /* isExternal = */ false,
+        (0..1).map { i -> createClass(kotlinJvmInternalPackage, "SuspendFunction$i", ClassKind.INTERFACE) }
+    }
+
+    private fun createClass(
+            packageFragment: PackageFragmentDescriptor,
+            name: String,
+            classKind: ClassKind = ClassKind.CLASS
+    ): ClassDescriptor =
+            MutableClassDescriptor(packageFragment, classKind, /* isInner = */ false, /* isExternal = */ false,
                                    Name.identifier(name), SourceElement.NO_SOURCE).apply {
                 modality = Modality.FINAL
                 visibility = Visibilities.PUBLIC
@@ -75,7 +85,15 @@ class JvmRuntimeTypes(module: ModuleDescriptor) {
         )
 
         if (descriptor.isSuspendLambda) {
-            return listOf(coroutineImplClass.defaultType, functionType)
+            return mutableListOf<KotlinType>().apply {
+                add(coroutineImplClass.defaultType)
+                add(functionType)
+
+                val parametersNumber =
+                        descriptor.valueParameters.size + (if (functionType.isExtensionFunctionType) 1 else 0)
+
+                addIfNotNull(suspendFunctions.getOrNull(parametersNumber)?.defaultType)
+            }
         }
 
         return listOf(lambda.defaultType, functionType)

compiler/backend/src/org/jetbrains/kotlin/codegen/codegenUtil.kt

@@ -300,6 +300,14 @@ fun KtExpression?.asmType(typeMapper: KotlinTypeMapper, bindingContext: BindingC
 
 fun KtExpression?.kotlinType(bindingContext: BindingContext) = this?.let(bindingContext::getType)
 
+fun Collection<Type>.withVariableIndices(): List<Pair<Int, Type>> = mutableListOf<Pair<Int, Type>>().apply {
+    var index = 0
+    for (type in this@withVariableIndices) {
+        add(index to type)
+        index += type.size
+    }
+}
+
 fun FunctionDescriptor.isGenericToArray(): Boolean {
     if (valueParameters.size != 1 || typeParameters.size != 1) return false
 
@@ -330,4 +338,4 @@ fun MemberDescriptor.isToArrayFromCollection(): Boolean {
     if (!isSubclass(containingClassDescriptor, collectionClass)) return false
 
     return isGenericToArray() || isNonGenericToArray()
-}
+}

compiler/backend/src/org/jetbrains/kotlin/codegen/coroutines/CoroutineCodegen.kt

@@ -17,6 +17,7 @@
 package org.jetbrains.kotlin.codegen.coroutines
 
 import com.intellij.util.ArrayUtil
+import org.jetbrains.kotlin.backend.common.CONTINUATION_RESUME_METHOD_NAME
 import org.jetbrains.kotlin.codegen.*
 import org.jetbrains.kotlin.codegen.context.ClosureContext
 import org.jetbrains.kotlin.codegen.state.GenerationState
@@ -36,6 +37,8 @@ import org.jetbrains.kotlin.resolve.jvm.diagnostics.JvmDeclarationOrigin
 import org.jetbrains.kotlin.resolve.jvm.diagnostics.OtherOrigin
 import org.jetbrains.kotlin.resolve.jvm.jvmSignature.JvmMethodSignature
 import org.jetbrains.kotlin.types.KotlinType
+import org.jetbrains.kotlin.types.KotlinTypeFactory
+import org.jetbrains.kotlin.types.typeUtil.asTypeProjection
 import org.jetbrains.kotlin.types.typeUtil.makeNullable
 import org.jetbrains.kotlin.utils.addToStdlib.safeAs
 import org.jetbrains.kotlin.utils.singletonOrEmptyList
@@ -91,6 +94,19 @@ class CoroutineCodegen(
                 )
             }
 
+    private val createCoroutineDescriptor =
+        funDescriptor.createCustomCopy {
+            setName(Name.identifier("create"))
+            setReturnType(
+                    KotlinTypeFactory.simpleNotNullType(
+                            Annotations.EMPTY,
+                            funDescriptor.builtIns.continuationClassDescriptor,
+                            listOf(funDescriptor.builtIns.unitType.asTypeProjection())
+                    )
+            )
+            setVisibility(Visibilities.PUBLIC)
+        }
+
     override fun generateClosureBody() {
         for (parameter in allLambdaParameters()) {
             val fieldInfo = parameter.getFieldInfoForCoroutineLambdaParameter()
@@ -108,14 +124,65 @@ class CoroutineCodegen(
     override fun generateBody() {
         super.generateBody()
 
+        functionCodegen.generateMethod(JvmDeclarationOrigin.NO_ORIGIN, createCoroutineDescriptor,
+                                       object : FunctionGenerationStrategy.CodegenBased(state) {
+                                           override fun doGenerateBody(codegen: ExpressionCodegen, signature: JvmMethodSignature) {
+                                               generateCreateCoroutineMethod(codegen)
+                                           }
+                                       })
+
+        // invoke(..) = create(..).resume(Unit)
         functionCodegen.generateMethod(JvmDeclarationOrigin.NO_ORIGIN, funDescriptor,
                                        object : FunctionGenerationStrategy.CodegenBased(state) {
                                            override fun doGenerateBody(codegen: ExpressionCodegen, signature: JvmMethodSignature) {
-                                               generateInvokeMethod(codegen)
+                                               codegen.v.generateInvokeMethod(signature)
                                            }
                                        })
+
+        if (allLambdaParameters().size <= 1) {
+            val delegate = typeMapper.mapSignatureSkipGeneric(createCoroutineDescriptor).asmMethod
+
+            val bridgeParameters = (1..delegate.argumentTypes.size - 1).map { AsmTypes.OBJECT_TYPE } + delegate.argumentTypes.last()
+            val bridge = Method(delegate.name, delegate.returnType, bridgeParameters.toTypedArray())
+
+            generateBridge(bridge, delegate)
+        }
     }
 
+    private fun InstructionAdapter.generateInvokeMethod(signature: JvmMethodSignature) {
+        // this
+        load(0, AsmTypes.OBJECT_TYPE)
+        val parameterTypes = signature.valueParameters.map { it.asmType }
+        parameterTypes.withVariableIndices().forEach {
+            (index, type) -> load(index + 1, type)
+        }
+
+        // this.create(..)
+        invokevirtual(
+                v.thisName,
+                createCoroutineDescriptor.name.identifier,
+                Type.getMethodDescriptor(
+                        AsmTypes.CONTINUATION,
+                        *parameterTypes.toTypedArray()
+                ),
+                false
+        )
+        checkcast(Type.getObjectType(v.thisName))
+
+        // .resume(Unit)
+        StackValue.putUnitInstance(this)
+        invokevirtual(
+                AsmTypes.RESTRICTED_COROUTINE_IMPL.internalName,
+                CONTINUATION_RESUME_METHOD_NAME.identifier,
+                Type.getMethodDescriptor(Type.VOID_TYPE, AsmTypes.OBJECT_TYPE),
+                false
+        )
+
+        loadSuspendMarker()
+        areturn(AsmTypes.OBJECT_TYPE)
+    }
+
+
     override fun generateConstructor(): Method {
         val args = calculateConstructorParameters(typeMapper, closure, asmType)
         val argTypes = args.map { it.fieldType }.plus(AsmTypes.CONTINUATION).toTypedArray()
@@ -149,7 +216,7 @@ class CoroutineCodegen(
         return constructor
     }
 
-    private fun generateInvokeMethod(codegen: ExpressionCodegen) {
+    private fun generateCreateCoroutineMethod(codegen: ExpressionCodegen) {
         val classDescriptor = closureContext.contextDescriptor
         val owner = typeMapper.mapClass(classDescriptor)
 

compiler/backend/src/org/jetbrains/kotlin/codegen/coroutines/coroutineCodegenUtil.kt

@@ -137,7 +137,7 @@ typealias FunctionDescriptorCopyBuilderToFunctionDescriptorCopyBuilder =
     FunctionDescriptor.CopyBuilder<out FunctionDescriptor>.(FunctionDescriptor)
         -> FunctionDescriptor.CopyBuilder<out FunctionDescriptor>
 
-private fun <D : FunctionDescriptor> D.createCustomCopy(
+fun <D : FunctionDescriptor> D.createCustomCopy(
         copySettings: FunctionDescriptorCopyBuilderToFunctionDescriptorCopyBuilder
 ): D {
 

compiler/testData/codegen/bytecodeListing/coroutineFields.txt

@@ -13,6 +13,8 @@ final class CoroutineFieldsKt$box$1 {
     private field p$: Controller
     inner class CoroutineFieldsKt$box$1
     method <init>(p0: kotlin.jvm.internal.Ref$ObjectRef, p1: kotlin.coroutines.Continuation): void
+    public final @org.jetbrains.annotations.NotNull method create(@org.jetbrains.annotations.NotNull p0: Controller, @org.jetbrains.annotations.NotNull p1: kotlin.coroutines.Continuation): kotlin.coroutines.Continuation
+    public synthetic method create(p0: java.lang.Object, p1: kotlin.coroutines.Continuation): kotlin.coroutines.Continuation
     protected final @org.jetbrains.annotations.Nullable method doResume(@org.jetbrains.annotations.Nullable p0: java.lang.Object, @org.jetbrains.annotations.Nullable p1: java.lang.Throwable): java.lang.Object
     public final @org.jetbrains.annotations.Nullable method invoke(@org.jetbrains.annotations.NotNull p0: Controller, @org.jetbrains.annotations.NotNull p1: kotlin.coroutines.Continuation): java.lang.Object
     public synthetic method invoke(p0: java.lang.Object, p1: java.lang.Object): java.lang.Object

compiler/testData/codegen/bytecodeText/constCoroutine.kt

@@ -19,7 +19,7 @@ fun box(): String {
     return "OK"
 }
 
-// 1 GETSTATIC kotlin/Unit.INSTANCE : Lkotlin/Unit;
+// 2 GETSTATIC kotlin/Unit.INSTANCE : Lkotlin/Unit;
 // 1 GETSTATIC EmptyContinuation.INSTANCE
-// 1 GETSTATIC kotlin/coroutines/CoroutineIntrinsics.INSTANCE
+// 2 GETSTATIC kotlin/coroutines/CoroutineIntrinsics.INSTANCE
-// 3 GETSTATIC
+// 5 GETSTATIC

core/runtime.jvm/src/kotlin/jvm/internal/SuspendFunctions.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kotlin.jvm.internal
+
+import kotlin.coroutines.Continuation
+
+interface SuspendFunction0<out R> {
+    fun create(resultingContinuation: Continuation<R>): Continuation<Unit>
+}
+
+interface SuspendFunction1<in T, out R> {
+    fun create(value: T, resultingContinuation: Continuation<R>): Continuation<Unit>
+}

libraries/stdlib/src/kotlin/coroutines/CoroutinesLibrary.kt

@@ -19,7 +19,7 @@ public fun <R, T> (suspend R.() -> T).createCoroutine(
         receiver: R,
         completion: Continuation<T>,
         dispatcher: ContinuationDispatcher? = null
-): Continuation<Unit> = (this as (R, Continuation<T>) -> Continuation<Unit>).invoke(receiver, withDispatcher(completion, dispatcher))
+): Continuation<Unit> = (this as kotlin.jvm.internal.SuspendFunction1<R, T>).create(receiver, withDispatcher(completion, dispatcher))
 
 /**
  * Starts coroutine with receiver type [R] and result type [T].
@@ -34,7 +34,7 @@ public fun <R, T> (suspend R.() -> T).startCoroutine(
         completion: Continuation<T>,
         dispatcher: ContinuationDispatcher? = null
 ) {
-    createCoroutine(receiver, completion, dispatcher).resume(Unit)
+    (this as Function2<R, Continuation<T>, Any?>).invoke(receiver, withDispatcher(completion, dispatcher))
 }
 
 /**
@@ -49,7 +49,7 @@ public fun <R, T> (suspend R.() -> T).startCoroutine(
 public fun <T> (suspend () -> T).createCoroutine(
         completion: Continuation<T>,
         dispatcher: ContinuationDispatcher? = null
-): Continuation<Unit> = (this as (Continuation<T>) -> Continuation<Unit>).invoke(withDispatcher(completion, dispatcher))
+): Continuation<Unit> = (this as kotlin.jvm.internal.SuspendFunction0<T>).create(withDispatcher(completion, dispatcher))
 
 /**
  * Starts coroutine without receiver and with result type [T].
@@ -63,7 +63,7 @@ public fun <T> (suspend  () -> T).startCoroutine(
         completion: Continuation<T>,
         dispatcher: ContinuationDispatcher? = null
 ) {
-    createCoroutine(completion, dispatcher).resume(Unit)
+    (this as Function1<Continuation<T>, Any?>).invoke(withDispatcher(completion, dispatcher))
 }
 
 /**

libraries/tools/binary-compatibility-validator/reference-public-api/kotlin-runtime.txt

@@ -900,6 +900,14 @@ public final class kotlin/jvm/internal/StringCompanionObject {
 	public static final field INSTANCE Lkotlin/jvm/internal/StringCompanionObject;
 }
 
+public abstract interface class kotlin/jvm/internal/SuspendFunction0 {
+	public abstract fun create (Lkotlin/coroutines/Continuation;)Lkotlin/coroutines/Continuation;
+}
+
+public abstract interface class kotlin/jvm/internal/SuspendFunction1 {
+	public abstract fun create (Ljava/lang/Object;Lkotlin/coroutines/Continuation;)Lkotlin/coroutines/Continuation;
+}
+
 public class kotlin/jvm/internal/TypeIntrinsics {
 	public fun <init> ()V
 	public static fun asMutableCollection (Ljava/lang/Object;)Ljava/util/Collection;