Reincarnated devirtualization

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt

@@ -18,7 +18,6 @@ import org.jetbrains.kotlin.backend.konan.library.KonanLibraryWriter
 import org.jetbrains.kotlin.backend.konan.library.LinkData
 import org.jetbrains.kotlin.backend.konan.llvm.*
 import org.jetbrains.kotlin.backend.konan.lower.DECLARATION_ORIGIN_BRIDGE_METHOD
-import org.jetbrains.kotlin.backend.konan.optimizations.DataFlowIR
 import org.jetbrains.kotlin.backend.konan.optimizations.Devirtualization
 import org.jetbrains.kotlin.backend.konan.optimizations.ExternalModulesDFG
 import org.jetbrains.kotlin.backend.konan.optimizations.ModuleDFG
@@ -321,9 +320,6 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) {
 
     val coverage = CoverageManager(this)
 
-    lateinit var privateFunctions: List<Pair<IrFunction, DataFlowIR.FunctionSymbol.Declared>>
-    lateinit var privateClasses: List<Pair<IrClass, DataFlowIR.Type.Declared>>
-
     // Cache used for source offset->(line,column) mapping.
     val fileEntryCache = mutableMapOf<String, SourceManager.FileEntry>()
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt

@@ -268,6 +268,8 @@ internal val bitcodePhase = namedIrModulePhase(
         lower = contextLLVMSetupPhase then
                 RTTIPhase then
                 generateDebugInfoHeaderPhase then
+                buildDFGPhase then
+                serializeDFGPhase then
                 deserializeDFGPhase then
                 devirtualizationPhase then
                 escapeAnalysisPhase then
@@ -295,8 +297,6 @@ internal val toplevelPhase = namedUnitPhase(
                                 dependenciesLowerPhase then // Then lower all libraries in topological order.
                                                             // With that we guarantee that inline functions are unlowered while being inlined.
                                 moduleIndexForCodegenPhase then
-                                buildDFGPhase then
-                                serializeDFGPhase then
                                 bitcodePhase then
                                 produceOutputPhase then
                                 verifyBitcodePhase then
@@ -309,9 +309,7 @@ internal val toplevelPhase = namedUnitPhase(
 internal fun PhaseConfig.konanPhasesConfig(config: KonanConfig) {
     with(config.configuration) {
         disable(compileTimeEvaluatePhase)
-        disable(buildDFGPhase)
         disable(deserializeDFGPhase)
-        disable(devirtualizationPhase)
         disable(escapeAnalysisPhase)
         disable(serializeDFGPhase)
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt

@@ -384,6 +384,8 @@ internal class KonanSymbols(context: Context, val symbolTable: SymbolTable, val
 
     val returnIfSuspended = internalFunction("returnIfSuspended")
 
+    val coroutineLaunchpad = internalFunction("coroutineLaunchpad")
+
     val konanSuspendCoroutineUninterceptedOrReturn = internalFunction("suspendCoroutineUninterceptedOrReturn")
 
     val konanCoroutineContextGetter = internalFunction("getCoroutineContext")

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/KonanIr.kt

@@ -6,10 +6,8 @@
 package org.jetbrains.kotlin.backend.konan.ir
 
 import org.jetbrains.kotlin.backend.konan.SYNTHETIC_OFFSET
-import org.jetbrains.kotlin.backend.konan.optimizations.DataFlowIR
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptor
-import org.jetbrains.kotlin.ir.IrStatement
 import org.jetbrains.kotlin.ir.SourceManager
 import org.jetbrains.kotlin.ir.SourceManager.FileEntry
 import org.jetbrains.kotlin.ir.SourceRangeInfo
@@ -18,14 +16,7 @@ import org.jetbrains.kotlin.ir.declarations.IrDeclaration
 import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.declarations.MetadataSource
 import org.jetbrains.kotlin.ir.expressions.*
-import org.jetbrains.kotlin.ir.expressions.impl.IrContainerExpressionBase
+import org.jetbrains.kotlin.ir.symbols.*
-import org.jetbrains.kotlin.ir.expressions.impl.IrExpressionBase
-import org.jetbrains.kotlin.ir.expressions.impl.IrTerminalDeclarationReferenceBase
-import org.jetbrains.kotlin.ir.symbols.IrClassifierSymbol
-import org.jetbrains.kotlin.ir.symbols.IrFileSymbol
-import org.jetbrains.kotlin.ir.symbols.IrReturnableBlockSymbol
-import org.jetbrains.kotlin.ir.symbols.impl.IrReturnableBlockSymbolImpl
-import org.jetbrains.kotlin.ir.types.IrType
 import org.jetbrains.kotlin.ir.visitors.IrElementTransformer
 import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
 import org.jetbrains.kotlin.konan.file.File
@@ -131,91 +122,4 @@ class IrFileImpl(entry: SourceManager.FileEntry) : IrFile {
     override fun <R, D> accept(visitor: IrElementVisitor<R, D>, data: D): R {
         TODO("not implemented")
     }
-}
-
-//-----------------------------------------------------------------------------//
-
-internal interface IrPrivateFunctionCall : IrExpression {
-    val valueArgumentsCount: Int
-    fun getValueArgument(index: Int): IrExpression?
-    fun putValueArgument(index: Int, valueArgument: IrExpression?)
-    fun removeValueArgument(index: Int)
-
-    val virtualCallee: IrCall?
-    val dfgSymbol: DataFlowIR.FunctionSymbol.Declared
-    val moduleDescriptor: ModuleDescriptor
-    val totalFunctions: Int
-    val functionIndex: Int
-}
-
-internal class IrPrivateFunctionCallImpl(startOffset: Int,
-                                         endOffset: Int,
-                                         type: IrType,
-                                         override val valueArgumentsCount: Int,
-                                         override val virtualCallee: IrCall?,
-                                         override val dfgSymbol: DataFlowIR.FunctionSymbol.Declared,
-                                         override val moduleDescriptor: ModuleDescriptor,
-                                         override val totalFunctions: Int,
-                                         override val functionIndex: Int
-) : IrPrivateFunctionCall, IrExpressionBase(startOffset, endOffset, type) {
-
-    override fun <R, D> accept(visitor: IrElementVisitor<R, D>, data: D): R {
-        return visitor.visitExpression(this, data)
-    }
-
-    private val argumentsByParameterIndex: Array<IrExpression?> = arrayOfNulls(valueArgumentsCount)
-
-    override fun getValueArgument(index: Int): IrExpression? {
-        if (index >= valueArgumentsCount) {
-            throw AssertionError("$this: No such value argument slot: $index")
-        }
-        return argumentsByParameterIndex[index]
-    }
-
-    override fun putValueArgument(index: Int, valueArgument: IrExpression?) {
-        if (index >= valueArgumentsCount) {
-            throw AssertionError("$this: No such value argument slot: $index")
-        }
-        argumentsByParameterIndex[index] = valueArgument
-    }
-
-    override fun removeValueArgument(index: Int) {
-        argumentsByParameterIndex[index] = null
-    }
-
-    override fun <D> acceptChildren(visitor: IrElementVisitor<Unit, D>, data: D) {
-        argumentsByParameterIndex.forEach { it?.accept(visitor, data) }
-    }
-
-    override fun <D> transformChildren(transformer: IrElementTransformer<D>, data: D) {
-        argumentsByParameterIndex.forEachIndexed { i, irExpression ->
-            argumentsByParameterIndex[i] = irExpression?.transform(transformer, data)
-        }
-    }
-}
-
-internal interface IrPrivateClassReference : IrClassReference {
-    val moduleDescriptor: ModuleDescriptor
-    val totalClasses: Int
-    val classIndex: Int
-    val dfgSymbol: DataFlowIR.Type.Declared
-}
-
-internal class IrPrivateClassReferenceImpl(startOffset: Int,
-                                           endOffset: Int,
-                                           type: IrType,
-                                           symbol: IrClassifierSymbol,
-                                           override val classType: IrType,
-                                           override val moduleDescriptor: ModuleDescriptor,
-                                           override val totalClasses: Int,
-                                           override val classIndex: Int,
-                                           override val dfgSymbol: DataFlowIR.Type.Declared
-) : IrPrivateClassReference,
-        IrTerminalDeclarationReferenceBase<IrClassifierSymbol, ClassifierDescriptor>(
-                startOffset, endOffset, type,
-                symbol, symbol.descriptor
-        )
-{
-    override fun <R, D> accept(visitor: IrElementVisitor<R, D>, data: D): R =
-            visitor.visitClassReference(this, data)
 }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BitcodePhases.kt

@@ -63,38 +63,11 @@ internal val deserializeDFGPhase = makeKonanModuleOpPhase(
 internal val devirtualizationPhase = makeKonanModuleOpPhase(
         name = "Devirtualization",
         description = "Devirtualization",
-        prerequisite = setOf(buildDFGPhase, deserializeDFGPhase),
+        prerequisite = setOf(buildDFGPhase),
         op = { context, irModule ->
-            context.externalModulesDFG?.let { externalModulesDFG ->
+            context.devirtualizationAnalysisResult = Devirtualization.run(
-                context.devirtualizationAnalysisResult = Devirtualization.run(
+                    irModule, context, context.moduleDFG!!, ExternalModulesDFG(emptyList(), emptyMap(), emptyMap(), emptyMap())
-                        irModule, context, context.moduleDFG!!, externalModulesDFG
+            )
-                )
-
-                val privateFunctions = context.moduleDFG!!.symbolTable.getPrivateFunctionsTableForExport()
-                privateFunctions.forEachIndexed { index, it ->
-                    val function = context.codegenVisitor.codegen.llvmFunction(it.first)
-                    LLVMAddAlias(
-                            context.llvmModule,
-                            function.type,
-                            function,
-                            irModule.descriptor.privateFunctionSymbolName(index, it.second.name)
-                    )!!
-                }
-                context.privateFunctions = privateFunctions
-
-                val privateClasses = context.moduleDFG!!.symbolTable.getPrivateClassesTableForExport()
-
-                privateClasses.forEachIndexed { index, it ->
-                    val typeInfoPtr = context.codegenVisitor.codegen.typeInfoValue(it.first)
-                    LLVMAddAlias(
-                            context.llvmModule,
-                            typeInfoPtr.type,
-                            typeInfoPtr,
-                            irModule.descriptor.privateClassSymbolName(index, it.second.name)
-                    )!!
-                }
-                context.privateClasses = privateClasses
-            }
         }
 )
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt

@@ -5,12 +5,15 @@ import llvm.*
 import org.jetbrains.kotlin.backend.konan.descriptors.TypedIntrinsic
 import org.jetbrains.kotlin.backend.konan.descriptors.isTypedIntrinsic
 import org.jetbrains.kotlin.backend.konan.llvm.objc.genObjCSelector
+import org.jetbrains.kotlin.backend.konan.ir.isSuspend
 import org.jetbrains.kotlin.backend.konan.reportCompilationError
 import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.declarations.IrClass
 import org.jetbrains.kotlin.ir.declarations.IrConstructor
 import org.jetbrains.kotlin.ir.declarations.IrFunction
 import org.jetbrains.kotlin.ir.expressions.*
 import org.jetbrains.kotlin.ir.types.getClass
+import org.jetbrains.kotlin.ir.util.dump
 import org.jetbrains.kotlin.name.Name
 
 internal enum class IntrinsicType {
@@ -64,6 +67,7 @@ internal enum class IntrinsicType {
     // Coroutines
     GET_CONTINUATION,
     RETURN_IF_SUSPEND,
+    COROUTINE_LAUNCHPAD,
     // Interop
     INTEROP_READ_BITS,
     INTEROP_WRITE_BITS,
@@ -98,7 +102,7 @@ internal interface IntrinsicGeneratorEnvironment {
 
     fun calculateLifetime(element: IrElement): Lifetime
 
-    fun evaluateCall(function: IrFunction, args: List<LLVMValueRef>, resultLifetime: Lifetime): LLVMValueRef
+    fun evaluateCall(function: IrFunction, args: List<LLVMValueRef>, resultLifetime: Lifetime, superClass: IrClass? = null): LLVMValueRef
 
     fun evaluateExplicitArgs(expression: IrMemberAccessExpression): List<LLVMValueRef>
 
@@ -155,9 +159,8 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
                 environment.functionGenerationContext.genObjCSelector(selector)
             }
             IntrinsicType.INIT_INSTANCE -> {
-                val callee = callSite as IrCall
+                val initializer = callSite.getValueArgument(1) as IrCall
-                val initializer = callee.getValueArgument(1) as IrCall
+                val thiz = environment.evaluateExpression(callSite.getValueArgument(0)!!)
-                val thiz = environment.evaluateExpression(callee.getValueArgument(0)!!)
                 environment.evaluateCall(
                         initializer.symbol.owner,
                         listOf(thiz) + environment.evaluateExplicitArgs(initializer),
@@ -165,6 +168,17 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
                 )
                 codegen.theUnitInstanceRef.llvm
             }
+            IntrinsicType.COROUTINE_LAUNCHPAD -> {
+                val suspendFunctionCall = callSite.getValueArgument(0) as IrCall
+                val continuation = environment.evaluateExpression(callSite.getValueArgument(1)!!)
+                val suspendFunction = suspendFunctionCall.symbol.owner
+                assert(suspendFunction.isSuspend) { "Call to a suspend function expected but was ${suspendFunction.dump()}" }
+                environment.evaluateCall(suspendFunction,
+                        environment.evaluateExplicitArgs(suspendFunctionCall) + listOf(continuation),
+                        environment.calculateLifetime(suspendFunctionCall),
+                        suspendFunction.parent as? IrClass // Call non-virtually.
+                )
+            }
             else -> null
         }
     }
@@ -238,6 +252,7 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
                     reportNonLoweredIntrinsic(intrinsicType)
                 IntrinsicType.INIT_INSTANCE,
                 IntrinsicType.OBJC_INIT_BY,
+                IntrinsicType.COROUTINE_LAUNCHPAD,
                 IntrinsicType.OBJC_GET_SELECTOR,
                 IntrinsicType.IMMUTABLE_BLOB ->
                     reportSpecialIntrinsic(intrinsicType)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -13,7 +13,6 @@ import org.jetbrains.kotlin.backend.konan.*
 import org.jetbrains.kotlin.backend.konan.descriptors.*
 import org.jetbrains.kotlin.backend.konan.ir.*
 import org.jetbrains.kotlin.backend.konan.llvm.coverage.*
-import org.jetbrains.kotlin.backend.konan.objcexport.ObjCExport
 import org.jetbrains.kotlin.backend.konan.optimizations.*
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.builtins.UnsignedType
@@ -220,8 +219,8 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
         override val exceptionHandler: ExceptionHandler
             get() = currentCodeContext.exceptionHandler
 
-        override fun evaluateCall(function: IrFunction, args: List<LLVMValueRef>, resultLifetime: Lifetime): LLVMValueRef =
+        override fun evaluateCall(function: IrFunction, args: List<LLVMValueRef>, resultLifetime: Lifetime, superClass: IrClass?) =
-                evaluateSimpleFunctionCall(function, args, resultLifetime)
+                evaluateSimpleFunctionCall(function, args, resultLifetime, superClass)
 
         override fun evaluateExplicitArgs(expression: IrMemberAccessExpression): List<LLVMValueRef> =
                 this@CodeGeneratorVisitor.evaluateExplicitArgs(expression)
@@ -794,9 +793,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
             is IrSuspendableExpression ->
                                         return evaluateSuspendableExpression  (value)
             is IrSuspensionPoint     -> return evaluateSuspensionPoint        (value)
-            is IrPrivateFunctionCall -> return evaluatePrivateFunctionCall    (value)
+            is IrClassReference ->      return evaluateClassReference         (value)
-            is IrPrivateClassReference ->
-                                        return evaluatePrivateClassReference  (value)
             else                     -> {
                 TODO(ir2string(value))
             }
@@ -2017,42 +2014,8 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
 
     //-------------------------------------------------------------------------//
 
-    private fun evaluatePrivateFunctionCall(callee: IrPrivateFunctionCall): LLVMValueRef {
+    private fun evaluateClassReference(classReference: IrClassReference): LLVMValueRef {
-        val args = (0 until callee.valueArgumentsCount).map { index ->
+        val typeInfoPtr = codegen.typeInfoValue(classReference.symbol.owner as IrClass)
-            callee.getValueArgument(index)?.let { evaluateExpression(it) }
-                    ?: run {
-                        assert(index == callee.valueArgumentsCount - 1) { "Only last argument may be null - for suspend functions" }
-                        getContinuation()
-                    }
-        }
-        val dfgSymbol = callee.dfgSymbol
-        val functionIndex = callee.functionIndex
-        val function = if (callee.moduleDescriptor == context.irModule!!.descriptor) {
-            codegen.llvmFunction(context.privateFunctions[functionIndex].first)
-        } else {
-            context.llvm.externalFunction(
-                    callee.moduleDescriptor.privateFunctionSymbolName(functionIndex, callee.dfgSymbol.name),
-                    codegen.getLlvmFunctionType(dfgSymbol),
-                    callee.moduleDescriptor.llvmSymbolOrigin
-
-            )
-        }
-        return call(dfgSymbol, function, args, resultLifetime = Lifetime.GLOBAL)
-    }
-
-    //-------------------------------------------------------------------------//
-
-    private fun evaluatePrivateClassReference(classReference: IrPrivateClassReference): LLVMValueRef {
-        val classIndex = classReference.classIndex
-        val typeInfoPtr = if (classReference.moduleDescriptor == context.irModule!!.descriptor) {
-            codegen.typeInfoValue(context.privateClasses[classIndex].first)
-        } else {
-            codegen.importGlobal(
-                    classReference.moduleDescriptor.privateClassSymbolName(classIndex, classReference.dfgSymbol.name),
-                    codegen.kTypeInfo,
-                    classReference.moduleDescriptor.llvmSymbolOrigin
-            )
-        }
         return functionGenerationContext.bitcast(int8TypePtr, typeInfoPtr)
     }
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGBuilder.kt

@@ -479,11 +479,32 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag
             )
         }
 
+        private fun IrType.erasure(): IrType {
+            if (this !is IrSimpleType) return this
+
+            val classifier = this.classifier
+            return when (classifier) {
+                is IrClassSymbol -> this
+                is IrTypeParameterSymbol -> {
+                    val upperBound = classifier.owner.superTypes.singleOrNull() ?:
+                    TODO("${classifier.descriptor} : ${classifier.descriptor.upperBounds}")
+
+                    if (this.hasQuestionMark) {
+                        // `T?`
+                        upperBound.erasure().makeNullable()
+                    } else {
+                        upperBound.erasure()
+                    }
+                }
+                else -> TODO(classifier.toString())
+            }
+        }
+
         private fun expressionToEdge(expression: IrExpression) =
                 if (expression is IrTypeOperatorCall && expression.operator.isCast())
                     DataFlowIR.Edge(
                             getNode(expression.argument),
-                            symbolTable.mapClassReferenceType(expression.typeOperand.getClass()!!)
+                            symbolTable.mapClassReferenceType(expression.typeOperand.erasure().getClass()!!)
                     )
                 else DataFlowIR.Edge(getNode(expression), null)
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGSerializer.kt

@@ -986,7 +986,7 @@ internal object DFGSerializer {
                                     ++module.numberOfClasses
                                 DataFlowIR.Type.Public(public.hash, public.intestines.base.isFinal,
                                         public.intestines.base.isAbstract, public.intestines.base.primitiveBinaryType,
-                                        module, symbolTableIndex, public.intestines.base.name).also {
+                                        module, symbolTableIndex, null, public.intestines.base.name).also {
                                     publicTypesMap.put(it.hash, it)
                                     allTypes += it
                                 }
@@ -998,7 +998,7 @@ internal object DFGSerializer {
                                     ++module.numberOfClasses
                                 DataFlowIR.Type.Private(privateTypeIndex++, private.intestines.base.isFinal,
                                         private.intestines.base.isAbstract, private.intestines.base.primitiveBinaryType,
-                                        module, symbolTableIndex, private.intestines.base.name).also {
+                                        module, symbolTableIndex, null, private.intestines.base.name).also {
                                     allTypes += it
                                 }
                             }
@@ -1013,14 +1013,14 @@ internal object DFGSerializer {
                     val private = it.private
                     when {
                         external != null ->
-                            DataFlowIR.FunctionSymbol.External(external.hash, attributes, external.name)
+                            DataFlowIR.FunctionSymbol.External(external.hash, attributes, null, external.name)
 
                         public != null -> {
                             val symbolTableIndex = public.index
                             if (symbolTableIndex >= 0)
                                 ++module.numberOfFunctions
                             DataFlowIR.FunctionSymbol.Public(public.hash,
-                                    module, symbolTableIndex, attributes, null, public.name).also {
+                                    module, symbolTableIndex, attributes, null, null, public.name).also {
                                 publicFunctionsMap.put(it.hash, it)
                             }
                         }
@@ -1030,7 +1030,7 @@ internal object DFGSerializer {
                             if (symbolTableIndex >= 0)
                                 ++module.numberOfFunctions
                             DataFlowIR.FunctionSymbol.Private(privateFunIndex++,
-                                    module, symbolTableIndex, attributes, null, private.name)
+                                    module, symbolTableIndex, attributes, null, null, private.name)
                         }
                     }.apply {
                         escapes = it.base.escapes

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DataFlowIR.kt

@@ -41,7 +41,7 @@ internal object DataFlowIR {
                         val primitiveBinaryType: PrimitiveBinaryType?,
                         val name: String?) {
         // Special marker type forbidding devirtualization on its instances.
-        object Virtual : Declared(false, true, null, null, -1, "\$VIRTUAL")
+        object Virtual : Declared(false, true, null, null, -1, null, "\$VIRTUAL")
 
         class External(val hash: Long, isFinal: Boolean, isAbstract: Boolean,
                        primitiveBinaryType: PrimitiveBinaryType?, name: String? = null)
@@ -63,7 +63,7 @@ internal object DataFlowIR {
         }
 
         abstract class Declared(isFinal: Boolean, isAbstract: Boolean, primitiveBinaryType: PrimitiveBinaryType?,
-                                val module: Module?, val symbolTableIndex: Int, name: String?)
+                                val module: Module?, val symbolTableIndex: Int, val irClass: IrClass?, name: String?)
             : Type(isFinal, isAbstract, primitiveBinaryType, name) {
             val superTypes = mutableListOf<Type>()
             val vtable = mutableListOf<FunctionSymbol>()
@@ -71,8 +71,8 @@ internal object DataFlowIR {
         }
 
         class Public(val hash: Long, isFinal: Boolean, isAbstract: Boolean, primitiveBinaryType: PrimitiveBinaryType?,
-                     module: Module, symbolTableIndex: Int, name: String? = null)
+                     module: Module, symbolTableIndex: Int, irClass: IrClass?, name: String? = null)
-            : Declared(isFinal, isAbstract, primitiveBinaryType, module, symbolTableIndex, name) {
+            : Declared(isFinal, isAbstract, primitiveBinaryType, module, symbolTableIndex, irClass, name) {
             override fun equals(other: Any?): Boolean {
                 if (this === other) return true
                 if (other !is Public) return false
@@ -90,8 +90,8 @@ internal object DataFlowIR {
         }
 
         class Private(val index: Int, isFinal: Boolean, isAbstract: Boolean, primitiveBinaryType: PrimitiveBinaryType?,
-                      module: Module, symbolTableIndex: Int, name: String? = null)
+                      module: Module, symbolTableIndex: Int, irClass: IrClass?, name: String? = null)
-            : Declared(isFinal, isAbstract, primitiveBinaryType, module, symbolTableIndex, name) {
+            : Declared(isFinal, isAbstract, primitiveBinaryType, module, symbolTableIndex, irClass, name) {
             override fun equals(other: Any?): Boolean {
                 if (this === other) return true
                 if (other !is Private) return false
@@ -122,7 +122,7 @@ internal object DataFlowIR {
 
     class FunctionParameter(val type: Type, val boxFunction: FunctionSymbol?, val unboxFunction: FunctionSymbol?)
 
-    abstract class FunctionSymbol(val attributes: Int, val name: String?) {
+    abstract class FunctionSymbol(val attributes: Int, val irFunction: IrFunction?, val name: String?) {
         lateinit var parameters: Array<FunctionParameter>
         lateinit var returnParameter: FunctionParameter
 
@@ -133,8 +133,8 @@ internal object DataFlowIR {
         var escapes: Int? = null
         var pointsTo: IntArray? = null
 
-        class External(val hash: Long, attributes: Int, name: String? = null)
+        class External(val hash: Long, attributes: Int, irFunction: IrFunction?, name: String? = null)
-            : FunctionSymbol(attributes, name) {
+            : FunctionSymbol(attributes, irFunction, name) {
 
             override fun equals(other: Any?): Boolean {
                 if (this === other) return true
@@ -153,14 +153,14 @@ internal object DataFlowIR {
         }
 
         abstract class Declared(val module: Module, val symbolTableIndex: Int,
-                                attributes: Int, var bridgeTarget: FunctionSymbol?, name: String?)
+                                attributes: Int, irFunction: IrFunction?, var bridgeTarget: FunctionSymbol?, name: String?)
-            : FunctionSymbol(attributes, name) {
+            : FunctionSymbol(attributes, irFunction, name) {
 
         }
 
         class Public(val hash: Long, module: Module, symbolTableIndex: Int,
-                     attributes: Int, bridgeTarget: FunctionSymbol?, name: String? = null)
+                     attributes: Int, irFunction: IrFunction?, bridgeTarget: FunctionSymbol?, name: String? = null)
-            : Declared(module, symbolTableIndex, attributes, bridgeTarget, name) {
+            : Declared(module, symbolTableIndex, attributes, irFunction, bridgeTarget, name) {
 
             override fun equals(other: Any?): Boolean {
                 if (this === other) return true
@@ -179,8 +179,8 @@ internal object DataFlowIR {
         }
 
         class Private(val index: Int, module: Module, symbolTableIndex: Int,
-                      attributes: Int, bridgeTarget: FunctionSymbol?, name: String? = null)
+                      attributes: Int, irFunction: IrFunction?, bridgeTarget: FunctionSymbol?, name: String? = null)
-            : Declared(module, symbolTableIndex, attributes, bridgeTarget, name) {
+            : Declared(module, symbolTableIndex, attributes, irFunction, bridgeTarget, name) {
 
             override fun equals(other: Any?): Boolean {
                 if (this === other) return true
@@ -500,10 +500,6 @@ internal object DataFlowIR {
             val isFinal = irClass.isFinal()
             val isAbstract = irClass.isAbstract()
             val name = irClass.fqNameSafe.asString()
-            if (irClass.module != irModule.descriptor)
-                return classMap.getOrPut(irClass) {
-                    Type.External(name.localHash.value, isFinal, isAbstract, null, takeName { name })
-                }
 
             classMap[irClass]?.let { return it }
 
@@ -511,10 +507,10 @@ internal object DataFlowIR {
             val symbolTableIndex = if (placeToClassTable) module.numberOfClasses++ else -1
             val type = if (irClass.isExported())
                            Type.Public(name.localHash.value, isFinal, isAbstract, null,
-                                   module, symbolTableIndex, takeName { name })
+                                   module, symbolTableIndex, irClass, takeName { name })
                        else
                            Type.Private(privateTypeIndex++, isFinal, isAbstract, null,
-                                   module, symbolTableIndex, takeName { name })
+                                   module, symbolTableIndex, irClass, takeName { name })
 
             classMap[irClass] = type
 
@@ -586,14 +582,14 @@ internal object DataFlowIR {
             if (returnsNothing)
                 attributes = attributes or FunctionAttributes.RETURNS_NOTHING
             val symbol = when {
-                it.module != irModule.descriptor || it.isExternal || (it.origin == IrDeclarationOrigin.IR_BUILTINS_STUB) -> {
+                it.isExternal || (it.origin == IrDeclarationOrigin.IR_BUILTINS_STUB) -> {
                     val escapesAnnotation = it.descriptor.annotations.findAnnotation(FQ_NAME_ESCAPES)
                     val pointsToAnnotation = it.descriptor.annotations.findAnnotation(FQ_NAME_POINTS_TO)
                     @Suppress("UNCHECKED_CAST")
                     val escapesBitMask = (escapesAnnotation?.allValueArguments?.get(escapesWhoDescriptor.name) as? ConstantValue<Int>)?.value
                     @Suppress("UNCHECKED_CAST")
                     val pointsToBitMask = (pointsToAnnotation?.allValueArguments?.get(pointsToOnWhomDescriptor.name) as? ConstantValue<List<IntValue>>)?.value
-                    FunctionSymbol.External(name.localHash.value, attributes, takeName { name }).apply {
+                    FunctionSymbol.External(name.localHash.value, attributes, it, takeName { name }).apply {
                         escapes  = escapesBitMask
                         pointsTo = pointsToBitMask?.let { it.map { it.value }.toIntArray() }
                     }
@@ -612,9 +608,9 @@ internal object DataFlowIR {
                             && (it.isOverridableOrOverrides || bridgeTarget != null || function.isSpecial || !irClass.isFinal())
                     val symbolTableIndex = if (placeToFunctionsTable) module.numberOfFunctions++ else -1
                     if (it.isExported())
-                        FunctionSymbol.Public(name.localHash.value, module, symbolTableIndex, attributes, bridgeTargetSymbol, takeName { name })
+                        FunctionSymbol.Public(name.localHash.value, module, symbolTableIndex, attributes, it, bridgeTargetSymbol, takeName { name })
                     else
-                        FunctionSymbol.Private(privateFunIndex++, module, symbolTableIndex, attributes, bridgeTargetSymbol, takeName { name })
+                        FunctionSymbol.Private(privateFunIndex++, module, symbolTableIndex, attributes, it, bridgeTargetSymbol, takeName { name })
                 }
             }
             functionMap[it] = symbol
@@ -639,7 +635,7 @@ internal object DataFlowIR {
 
             assert(irField.parent !is IrClass) { "All local properties initializers should've been lowered" }
             val attributes = FunctionAttributes.IS_GLOBAL_INITIALIZER or FunctionAttributes.RETURNS_UNIT
-            val symbol = FunctionSymbol.Private(privateFunIndex++, module, -1, attributes, null, takeName { "${irField.symbolName}_init" })
+            val symbol = FunctionSymbol.Private(privateFunIndex++, module, -1, attributes, null, null, takeName { "${irField.symbolName}_init" })
 
             functionMap[irField] = symbol
 
@@ -647,32 +643,5 @@ internal object DataFlowIR {
             symbol.returnParameter = mapTypeToFunctionParameter(context.irBuiltIns.unitType)
             return symbol
         }
-
-        fun getPrivateFunctionsTableForExport() =
-                functionMap
-                        .asSequence()
-                        .filter { it.key is IrFunction }
-                        .filter { it.value.let { it is DataFlowIR.FunctionSymbol.Declared && it.symbolTableIndex >= 0 } }
-                        .sortedBy { (it.value as DataFlowIR.FunctionSymbol.Declared).symbolTableIndex }
-                        .apply {
-                            forEachIndexed { index, entry ->
-                                assert((entry.value  as DataFlowIR.FunctionSymbol.Declared).symbolTableIndex == index) { "Inconsistent function table" }
-                            }
-                        }
-                        .map { (it.key as IrFunction) to (it.value as DataFlowIR.FunctionSymbol.Declared) }
-                        .toList()
-
-        fun getPrivateClassesTableForExport() =
-                classMap
-                        .asSequence()
-                        .filter { it.value.let { it is DataFlowIR.Type.Declared && it.symbolTableIndex >= 0 } }
-                        .sortedBy { (it.value as DataFlowIR.Type.Declared).symbolTableIndex }
-                        .apply {
-                            forEachIndexed { index, entry ->
-                                assert((entry.value  as DataFlowIR.Type.Declared).symbolTableIndex == index) { "Inconsistent class table" }
-                            }
-                        }
-                        .map { it.key to (it.value as DataFlowIR.Type.Declared) }
-                        .toList()
     }
 }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/Devirtualization.kt

@@ -15,16 +15,11 @@ import org.jetbrains.kotlin.backend.common.lower.irBlock
 import org.jetbrains.kotlin.backend.konan.*
 import org.jetbrains.kotlin.backend.konan.getInlinedClass
 import org.jetbrains.kotlin.backend.konan.descriptors.isInterface
-import org.jetbrains.kotlin.backend.konan.ir.IrPrivateClassReferenceImpl
+import org.jetbrains.kotlin.backend.konan.ir.isSuspend
-import org.jetbrains.kotlin.backend.konan.ir.IrPrivateFunctionCall
-import org.jetbrains.kotlin.backend.konan.ir.IrPrivateFunctionCallImpl
-import org.jetbrains.kotlin.backend.konan.ir.getErasedTypeClass
 import org.jetbrains.kotlin.descriptors.ClassDescriptor
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.builders.*
-import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
+import org.jetbrains.kotlin.ir.declarations.*
-import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
-import org.jetbrains.kotlin.ir.declarations.IrVariable
 import org.jetbrains.kotlin.ir.declarations.impl.IrVariableImpl
 import org.jetbrains.kotlin.ir.descriptors.IrTemporaryVariableDescriptorImpl
 import org.jetbrains.kotlin.ir.expressions.*
@@ -1016,7 +1011,7 @@ internal object Devirtualization {
                         .filter { it.key.irCallSite != null }
                         .associate { it.key.irCallSite!! to it.value }
         devirtualize(irModule, context, externalModulesDFG, devirtualizedCallSites)
-        removeRedundantCoercions(irModule, context, moduleDFG, externalModulesDFG)
+        removeRedundantCoercions(irModule, context)
         return AnalysisResult(devirtualizationAnalysisResult)
     }
 
@@ -1034,12 +1029,9 @@ internal object Devirtualization {
     private val specialNames = listOf("<box>", "<unbox>")
 
     // TODO: do it more reliably.
-    private fun IrExpression.isBoxOrUnboxCall() =
+    private fun IrExpression.isBoxOrUnboxCall() = (this is IrCall && symbol.owner.name.asString().let { specialNames.contains(it) })
-            (this is IrCall && symbol.owner.name.asString().let { specialNames.contains(it) })
-                    || (this is IrPrivateFunctionCall && specialNames.any { dfgSymbol.name?.contains(it) == true })
 
-    private fun devirtualize(irModule: IrModuleFragment, context: Context,
+    private fun devirtualize(irModule: IrModuleFragment, context: Context, externalModulesDFG: ExternalModulesDFG,
-                             externalModulesDFG: ExternalModulesDFG,
                              devirtualizedCallSites: Map<IrCall, DevirtualizedCallSite>) {
         val symbols = context.ir.symbols
         val nativePtrEqualityOperatorSymbol = symbols.areEqualByValue[PrimitiveBinaryType.POINTER]!!
@@ -1067,17 +1059,7 @@ internal object Devirtualization {
         fun IrBuilderWithScope.irCoerce(value: IrExpression, coercion: DataFlowIR.FunctionSymbol.Declared?) =
                 if (coercion == null)
                     value
-                else IrPrivateFunctionCallImpl(
+                else irCall(coercion.irFunction!!).apply {
-                        startOffset         = startOffset,
-                        endOffset           = endOffset,
-                        type                = value.type, // TODO: What type is actually must be here?
-                        valueArgumentsCount = 1,
-                        virtualCallee       = null,
-                        dfgSymbol           = coercion,
-                        moduleDescriptor    = coercion.module.descriptor,
-                        totalFunctions      = coercion.module.numberOfFunctions,
-                        functionIndex       = coercion.symbolTableIndex
-                ).apply {
                     putValueArgument(0, value)
                 }
 
@@ -1124,45 +1106,80 @@ internal object Devirtualization {
             }
             if (actualType.boxFunction == null)
                 return targetType.unboxFunction!!.resolved() as DataFlowIR.FunctionSymbol.Declared
-            return actualType.boxFunction!!.resolved() as DataFlowIR.FunctionSymbol.Declared
+            return actualType.boxFunction.resolved() as DataFlowIR.FunctionSymbol.Declared
         }
 
-        fun irDevirtualizedCall(callee: IrCall, actualType: IrType, devirtualizedCallee: DataFlowIR.FunctionSymbol.Declared) =
+        fun IrCallImpl.putArgument(index: Int, value: IrExpression) {
-                IrPrivateFunctionCallImpl(
+            var receiversCount = 0
-                        startOffset         = callee.startOffset,
+            val callee = symbol.owner
-                        endOffset           = callee.endOffset,
+            if (callee.dispatchReceiverParameter != null)
-                        type                = actualType,
+                ++receiversCount
-                        valueArgumentsCount = devirtualizedCallee.parameters.size,
+            if (callee.extensionReceiverParameter != null)
-                        virtualCallee       = callee,
+                ++receiversCount
-                        dfgSymbol           = devirtualizedCallee,
+            if (index >= receiversCount)
-                        moduleDescriptor    = devirtualizedCallee.module.descriptor,
+                putValueArgument(index - receiversCount, value)
-                        totalFunctions      = devirtualizedCallee.module.numberOfFunctions,
+            else {
-                        functionIndex       = devirtualizedCallee.symbolTableIndex
+                if (callee.dispatchReceiverParameter != null && index == 0)
-                )
+                    dispatchReceiver = value
+                else
+                    extensionReceiver = value
+            }
+        }
+
+        fun IrBuilderWithScope.irDevirtualizedCall(callSite: IrCall,
+                                                   actualType: IrType,
+                                                   devirtualizedCallee: DevirtualizedCallee,
+                                                   arguments: List<IrExpression>): IrCall {
+            val actualCallee = devirtualizedCallee.callee.irFunction!!
+            val call = IrCallImpl(
+                    callSite.startOffset, callSite.endOffset,
+                    actualType,
+                    actualCallee.symbol,
+                    actualCallee.descriptor,
+                    actualCallee.typeParameters.size,
+                    actualCallee.valueParameters.size,
+                    callSite.origin,
+                    actualCallee.parentAsClass.symbol
+            )
+            if (actualCallee.explicitParameters.size == arguments.size) {
+                arguments.forEachIndexed { index, argument -> call.putArgument(index, argument) }
+                return call
+            }
+            assert(actualCallee.isSuspend && actualCallee.explicitParameters.size == arguments.size - 1) {
+                "Incorrect number of arguments: expected [${actualCallee.explicitParameters.size}] but was [${arguments.size - 1}]\n" +
+                        actualCallee.dump()
+            }
+            val continuation = arguments.last()
+            for (index in 0..arguments.size - 2)
+                call.putArgument(index, arguments[index])
+            return irCall(context.ir.symbols.coroutineLaunchpad, actualType).apply {
+                putValueArgument(0, call)
+                putValueArgument(1, continuation)
+            }
+        }
 
         fun IrBuilderWithScope.irDevirtualizedCall(callee: IrCall, actualType: IrType,
-                                                   actualCallee: DataFlowIR.FunctionSymbol.Declared,
+                                                   devirtualizedCallee: DevirtualizedCallee,
-                                                   parameters: List<PossiblyCoercedValue>) =
+                                                   arguments: List<PossiblyCoercedValue>): IrExpression {
-                actualCallee.bridgeTarget.let {
+            val actualCallee = devirtualizedCallee.callee as DataFlowIR.FunctionSymbol.Declared
-                    if (it == null)
+            return actualCallee.bridgeTarget.let { bridgeTarget ->
-                        irDevirtualizedCall(callee, actualType, actualCallee).apply {
+                if (bridgeTarget == null)
-                            parameters.forEachIndexed { index, value ->
+                    irDevirtualizedCall(callee, actualType,
-                                putValueArgument(index, value.getFullValue(this@irDevirtualizedCall))
+                            devirtualizedCallee,
-                            }
+                            arguments.map { it.getFullValue(this@irDevirtualizedCall) })
-                        }
+                else {
-                    else {
+                    val callResult = irDevirtualizedCall(callee, actualType,
-                        val bridgeTarget = it.resolved() as DataFlowIR.FunctionSymbol.Declared
+                            DevirtualizedCallee(devirtualizedCallee.receiverType, bridgeTarget),
-                        val callResult = irDevirtualizedCall(callee, actualType, bridgeTarget).apply {
+                            arguments.mapIndexed { index, value ->
-                            parameters.forEachIndexed { index, value ->
                                 val coercion = getTypeConversion(actualCallee.parameters[index], bridgeTarget.parameters[index])
                                 val fullValue = value.getFullValue(this@irDevirtualizedCall)
-                                putValueArgument(index, coercion?.let { irCoerce(fullValue, coercion) } ?: fullValue)
+                                coercion?.let { irCoerce(fullValue, coercion) } ?: fullValue
-                            }
+                            })
-                        }
+                    val returnCoercion = getTypeConversion(bridgeTarget.returnParameter, actualCallee.returnParameter)
-                        val returnCoercion = getTypeConversion(bridgeTarget.returnParameter, actualCallee.returnParameter)
+                    irCoerce(callResult, returnCoercion)
-                        irCoerce(callResult, returnCoercion)
-                    }
                 }
+            }
+        }
 
         irModule.transformChildrenVoid(object: IrElementTransformerVoidWithContext() {
             override fun visitCall(expression: IrCall): IrExpression {
@@ -1208,37 +1225,31 @@ internal object Devirtualization {
                         }
 
                         optimize && possibleCallees.size == 1 -> { // Monomorphic callsite.
-                            val actualCallee = possibleCallees[0].callee as DataFlowIR.FunctionSymbol.Declared
                             irBlock(expression) {
                                 val parameters = expression.getArgumentsWithSymbols().mapIndexed { index, arg ->
                                     irSplitCoercion(arg.second, "arg$index", arg.first.owner.type)
                                 }
-                                +irDevirtualizedCall(expression, type, actualCallee, parameters)
+                                +irDevirtualizedCall(expression, type, possibleCallees[0], parameters)
                             }
                         }
 
                         else -> irBlock(expression) {
-                            val parameters = expression.getArgumentsWithSymbols().mapIndexed { index, arg ->
+                            val arguments = expression.getArgumentsWithSymbols().mapIndexed { index, arg ->
                                 irSplitCoercion(arg.second, "arg$index", arg.first.owner.type)
                             }
                             val typeInfo = irTemporary(irCall(symbols.getObjectTypeInfo).apply {
-                                putValueArgument(0, parameters[0].getFullValue(this@irBlock))
+                                putValueArgument(0, arguments[0].getFullValue(this@irBlock))
                             })
 
                             val branches = mutableListOf<IrBranchImpl>()
                             possibleCallees.mapIndexedTo(branches) { index, devirtualizedCallee ->
-                                val actualCallee = devirtualizedCallee.callee as DataFlowIR.FunctionSymbol.Declared
                                 val actualReceiverType = devirtualizedCallee.receiverType as DataFlowIR.Type.Declared
-                                val expectedTypeInfo = IrPrivateClassReferenceImpl(
+                                val expectedTypeInfo = IrClassReferenceImpl(
-                                        startOffset      = startOffset,
+                                        startOffset, endOffset,
-                                        endOffset        = endOffset,
+                                        symbols.nativePtrType,
-                                        type             = symbols.nativePtrType,
+                                        actualReceiverType.irClass!!.symbol,
-                                        symbol           = dispatchReceiver.type.getErasedTypeClass(),
+                                        actualReceiverType.irClass.defaultType
-                                        classType        = dispatchReceiver.type,
+                                )
-                                        moduleDescriptor = actualReceiverType.module!!.descriptor,
-                                        totalClasses     = actualReceiverType.module.numberOfClasses,
-                                        classIndex       = actualReceiverType.symbolTableIndex,
-                                        dfgSymbol        = actualReceiverType)
                                 val condition =
                                         if (optimize && index == possibleCallees.size - 1)
                                             irTrue() // Don't check last type in optimize mode.
@@ -1251,7 +1262,7 @@ internal object Devirtualization {
                                         startOffset = startOffset,
                                         endOffset   = endOffset,
                                         condition   = condition,
-                                        result      = irDevirtualizedCall(expression, type, actualCallee, parameters)
+                                        result      = irDevirtualizedCall(expression, type, devirtualizedCallee, arguments)
                                 )
                             }
                             if (!optimize) { // Add else branch throwing exception for debug purposes.
@@ -1285,14 +1296,7 @@ internal object Devirtualization {
         })
     }
 
-    private fun removeRedundantCoercions(irModule: IrModuleFragment, context: Context,
+    private fun removeRedundantCoercions(irModule: IrModuleFragment, context: Context) {
-                                         moduleDFG: ModuleDFG, externalModulesDFG: ExternalModulesDFG) {
-
-        fun DataFlowIR.FunctionSymbol.resolved(): DataFlowIR.FunctionSymbol {
-            if (this is DataFlowIR.FunctionSymbol.External)
-                return externalModulesDFG.publicFunctions[this.hash] ?: this
-            return this
-        }
 
         class PossiblyFoldedExpression(val expression: IrExpression, val folded: Boolean) {
             fun getFullExpression(coercion: IrCall, cast: IrTypeOperatorCall?): IrExpression {
@@ -1355,21 +1359,12 @@ internal object Devirtualization {
 
                 val coercionDeclaringClass = coercion.symbol.owner.parentAsClass
                 if (expression.isBoxOrUnboxCall()) {
+                    expression as IrCall
                     val result =
-                            (expression as? IrCall)?.let {
+                            if (coercionDeclaringClass == expression.symbol.owner.parentAsClass)
-                                if (coercionDeclaringClass == it.symbol.owner.parentAsClass)
+                                expression.getArguments().single().second
-                                    it.getArguments().single().second
+                            else expression
-                                else expression
+
-                            } ?: (expression as IrPrivateFunctionCall).let {
-                                val argarg = it.getValueArgument(0)!!
-                                val boxFunction = context.getBoxFunction(coercionDeclaringClass)
-                                val unboxFunction = context.getUnboxFunction(coercionDeclaringClass)
-                                val boxFunctionSymbol = moduleDFG.symbolTable.mapFunction(boxFunction).resolved()
-                                val unboxFunctionSymbol = moduleDFG.symbolTable.mapFunction(unboxFunction).resolved()
-                                if (it.dfgSymbol == boxFunctionSymbol || it.dfgSymbol == unboxFunctionSymbol)
-                                    argarg
-                                else it
-                            }
                     return PossiblyFoldedExpression(result.transformIfAsked(), result != expression)
                 }
                 return when (expression) {

runtime/src/main/kotlin/kotlin/native/internal/Coroutines.kt

@@ -25,3 +25,6 @@ internal inline suspend fun getCoroutineContext(): CoroutineContext =
 @TypedIntrinsic(IntrinsicType.RETURN_IF_SUSPEND)
 @PublishedApi
 internal external suspend fun <T> returnIfSuspended(@Suppress("UNUSED_PARAMETER") argument: Any?): T
+
+@TypedIntrinsic(IntrinsicType.COROUTINE_LAUNCHPAD)
+internal external fun coroutineLaunchpad(suspendFunctionCall: Any?, continuation: Continuation<*>): Any?

runtime/src/main/kotlin/kotlin/native/internal/IntrinsicType.kt

@@ -57,6 +57,7 @@ class IntrinsicType {
 
         const val GET_CONTINUATION              = "GET_CONTINUATION"
         const val RETURN_IF_SUSPEND             = "RETURN_IF_SUSPEND"
+        const val COROUTINE_LAUNCHPAD           = "COROUTINE_LAUNCHPAD"
 
         // Interop
         const val INTEROP_READ_PRIMITIVE        = "INTEROP_READ_PRIMITIVE"