Implemented devirtualization of multimorphic callsites

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/DescriptorUtils.kt

@@ -78,8 +78,6 @@ private val intrinsicAnnotation = FqName("konan.internal.Intrinsic")
 internal val CallableDescriptor.isIntrinsic: Boolean
     get() = this.annotations.findAnnotation(intrinsicAnnotation) != null
 
-internal fun FunctionDescriptor.externalOrIntrinsic() = isExternal || isIntrinsic || (this is IrBuiltinOperatorDescriptorBase)
-
 private val intrinsicTypes = setOf(
         "kotlin.Boolean", "kotlin.Char",
         "kotlin.Byte", "kotlin.Short",

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt

@@ -132,6 +132,9 @@ internal class KonanSymbols(context: Context, val symbolTable: SymbolTable): Sym
     override val ThrowTypeCastException = symbolTable.referenceSimpleFunction(
             context.getInternalFunctions("ThrowTypeCastException").single())
 
+    val throwInvalidReceiverTypeException = symbolTable.referenceSimpleFunction(
+            context.getInternalFunctions("ThrowInvalidReceiverTypeException").single())
+
     override val ThrowUninitializedPropertyAccessException = symbolTable.referenceSimpleFunction(
             context.getInternalFunctions("ThrowUninitializedPropertyAccessException").single()
     )

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/KonanIr.kt

@@ -24,10 +24,13 @@ import org.jetbrains.kotlin.ir.declarations.IrDeclaration
 import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.declarations.IrVariable
 import org.jetbrains.kotlin.ir.expressions.IrCall
+import org.jetbrains.kotlin.ir.expressions.IrClassReference
 import org.jetbrains.kotlin.ir.expressions.IrExpression
 import org.jetbrains.kotlin.ir.expressions.impl.IrCallWithIndexedArgumentsBase
 import org.jetbrains.kotlin.ir.expressions.impl.IrExpressionBase
+import org.jetbrains.kotlin.ir.expressions.impl.IrTerminalDeclarationReferenceBase
 import org.jetbrains.kotlin.ir.symbols.IrClassSymbol
+import org.jetbrains.kotlin.ir.symbols.IrClassifierSymbol
 import org.jetbrains.kotlin.ir.symbols.IrFileSymbol
 import org.jetbrains.kotlin.ir.symbols.IrFunctionSymbol
 import org.jetbrains.kotlin.ir.visitors.IrElementTransformer
@@ -209,4 +212,28 @@ internal class IrPrivateFunctionCallImpl(startOffset: Int,
     override fun <R, D> accept(visitor: IrElementVisitor<R, D>, data: D): R =
             visitor.visitCall(this, data)
 
+}
+
+internal interface IrPrivateClassReference : IrClassReference {
+    val moduleDescriptor: ModuleDescriptor
+    val totalClasses: Int
+    val classIndex: Int
+}
+
+internal class IrPrivateClassReferenceImpl(startOffset: Int,
+                                           endOffset: Int,
+                                           type: KotlinType,
+                                           symbol: IrClassifierSymbol,
+                                           override val classType: KotlinType,
+                                           override val moduleDescriptor: ModuleDescriptor,
+                                           override val totalClasses: Int,
+                                           override val classIndex: Int
+) : IrPrivateClassReference,
+        IrTerminalDeclarationReferenceBase<IrClassifierSymbol, ClassifierDescriptor>(
+                startOffset, endOffset, type,
+                symbol, symbol.descriptor
+        )
+{
+    override fun <R, D> accept(visitor: IrElementVisitor<R, D>, data: D): R =
+            visitor.visitClassReference(this, data)
 }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt

@@ -274,3 +274,5 @@ internal val ClassDescriptor.typeInfoHasVtableAttached: Boolean
     get() = !this.isAbstract() && !this.isExternalObjCClass()
 
 internal val ModuleDescriptor.privateFunctionsTableSymbolName get() = "private_functions_${name.asString()}"
+
+internal val ModuleDescriptor.privateClassesTableSymbolName get() = "private_classes_${name.asString()}"

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -48,7 +48,6 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
 
     private fun countParams(fn: FunctionDescriptor) = LLVMCountParams(fn.llvmFunction)
 
-    fun functionLlvmValue(descriptor: FunctionDescriptor) = descriptor.llvmFunction
     fun functionEntryPointAddress(descriptor: FunctionDescriptor) = descriptor.entryPointAddress.llvm
     fun functionHash(descriptor: FunctionDescriptor): LLVMValueRef = descriptor.functionName.localHash.llvm
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -96,7 +96,14 @@ internal fun emitLLVM(context: Context) {
         val privateFunctions = moduleDFG!!.symbolTable.getPrivateFunctionsTableForExport()
 
         codegenVisitor.codegen.staticData.placeGlobalConstArray(irModule.descriptor.privateFunctionsTableSymbolName, int8TypePtr,
-                privateFunctions.map { constPointer(codegenVisitor.codegen.functionLlvmValue(it)).bitcast(int8TypePtr) },
+                privateFunctions.map { constPointer(codegenVisitor.codegen.llvmFunction(it)).bitcast(int8TypePtr) },
+                isExported = true
+        )
+
+        val privateClasses = moduleDFG!!.symbolTable.getPrivateClassesTableForExport()
+
+        codegenVisitor.codegen.staticData.placeGlobalConstArray(irModule.descriptor.privateClassesTableSymbolName, int8TypePtr,
+                privateClasses.map { constPointer(codegenVisitor.codegen.typeInfoValue(it)).bitcast(int8TypePtr) },
                 isExported = true
         )
     }
@@ -766,6 +773,8 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
             is IrSuspendableExpression ->
                                         return evaluateSuspendableExpression  (value)
             is IrSuspensionPoint     -> return evaluateSuspensionPoint        (value)
+            is IrPrivateClassReference ->
+                                        return evaluatePrivateClassReference  (value)
             else                     -> {
                 TODO(ir2string(value))
             }
@@ -1942,6 +1951,21 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
 
     //-------------------------------------------------------------------------//
 
+    private fun evaluatePrivateClassReference(classReference: IrPrivateClassReference): LLVMValueRef {
+        val classesListName = classReference.moduleDescriptor.privateClassesTableSymbolName
+        // LLVM inlines access to this global array (with -opt option on).
+        val functionsList =
+                if (classReference.moduleDescriptor == context.irModule!!.descriptor)
+                    LLVMGetNamedGlobal(context.llvmModule, classesListName)
+                else
+                    codegen.importGlobal(classesListName, LLVMArrayType(int8TypePtr, classReference.totalClasses)!!, classReference.moduleDescriptor.llvmSymbolOrigin)
+        val classIndex    = Int32(classReference.classIndex).llvm
+        val classPlacePtr = LLVMBuildGEP(functionGenerationContext.builder, functionsList, cValuesOf(kImmZero, classIndex), 2, "")!!
+        return functionGenerationContext.load(classPlacePtr)
+    }
+
+    //-------------------------------------------------------------------------//
+
     private fun evaluateSimpleFunctionCall(
             descriptor: FunctionDescriptor, args: List<LLVMValueRef>,
             resultLifetime: Lifetime, superClass: ClassDescriptor? = null): LLVMValueRef {
@@ -2362,7 +2386,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
     fun callDirect(descriptor: FunctionDescriptor, args: List<LLVMValueRef>,
                    resultLifetime: Lifetime): LLVMValueRef {
         val realDescriptor = descriptor.target
-        val llvmFunction = codegen.functionLlvmValue(realDescriptor)
+        val llvmFunction = codegen.llvmFunction(realDescriptor)
         return call(descriptor, llvmFunction, args, resultLifetime)
     }
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/DefaultArgumentsStubGenerator.kt

@@ -207,18 +207,16 @@ private fun maskParameterSymbol(function: IrFunction, number: Int) =
 
 private fun markerParameterDescriptor(descriptor: FunctionDescriptor) = descriptor.valueParameters.single { it.name == kConstructorMarkerName }
 
-private fun nullConst(expression: IrElement, type: KotlinType): IrExpression? {
+fun nullConst(expression: IrElement, type: KotlinType) = when {
-    when {
+    KotlinBuiltIns.isFloat(type)   -> IrConstImpl.float     (expression.startOffset, expression.endOffset, type, 0.0F)
-        KotlinBuiltIns.isFloat(type)   -> return IrConstImpl.float     (expression.startOffset, expression.endOffset, type, 0.0F)
+    KotlinBuiltIns.isDouble(type)  -> IrConstImpl.double    (expression.startOffset, expression.endOffset, type, 0.0)
-        KotlinBuiltIns.isDouble(type)  -> return IrConstImpl.double    (expression.startOffset, expression.endOffset, type, 0.0)
+    KotlinBuiltIns.isBoolean(type) -> IrConstImpl.boolean   (expression.startOffset, expression.endOffset, type, false)
-        KotlinBuiltIns.isBoolean(type) -> return IrConstImpl.boolean   (expression.startOffset, expression.endOffset, type, false)
+    KotlinBuiltIns.isByte(type)    -> IrConstImpl.byte      (expression.startOffset, expression.endOffset, type, 0)
-        KotlinBuiltIns.isByte(type)    -> return IrConstImpl.byte      (expression.startOffset, expression.endOffset, type, 0)
+    KotlinBuiltIns.isChar(type)    -> IrConstImpl.char      (expression.startOffset, expression.endOffset, type, 0.toChar())
-        KotlinBuiltIns.isChar(type)    -> return IrConstImpl.char      (expression.startOffset, expression.endOffset, type, 0.toChar())
+    KotlinBuiltIns.isShort(type)   -> IrConstImpl.short     (expression.startOffset, expression.endOffset, type, 0)
-        KotlinBuiltIns.isShort(type)   -> return IrConstImpl.short     (expression.startOffset, expression.endOffset, type, 0)
+    KotlinBuiltIns.isInt(type)     -> IrConstImpl.int       (expression.startOffset, expression.endOffset, type, 0)
-        KotlinBuiltIns.isInt(type)     -> return IrConstImpl.int       (expression.startOffset, expression.endOffset, type, 0)
+    KotlinBuiltIns.isLong(type)    -> IrConstImpl.long      (expression.startOffset, expression.endOffset, type, 0)
-        KotlinBuiltIns.isLong(type)    -> return IrConstImpl.long      (expression.startOffset, expression.endOffset, type, 0)
+    else                           -> IrConstImpl.constNull (expression.startOffset, expression.endOffset, type.builtIns.nullableNothingType)
-        else                           -> return IrConstImpl.constNull (expression.startOffset, expression.endOffset, type.builtIns.nullableNothingType)
-    }
 }
 
 class DefaultParameterInjector constructor(val context: CommonBackendContext): BodyLoweringPass {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGSerializer.kt

@@ -204,15 +204,16 @@ internal object DFGSerializer {
         }
     }
 
-    class DeclaredType(val isFinal: Boolean, val isAbstract: Boolean, val superTypes: IntArray,
+    class DeclaredType(val isFinal: Boolean, val isAbstract: Boolean, val index: Int, val superTypes: IntArray,
                        val vtable: IntArray, val itable: Array<ItableSlot>) {
 
-        constructor(data: ArraySlice) : this(data.readBoolean(), data.readBoolean(), data.readIntArray(),
+        constructor(data: ArraySlice) : this(data.readBoolean(), data.readBoolean(), data.readInt(), data.readIntArray(),
                 data.readIntArray(), data.readArray { ItableSlot(this) })
 
         fun write(result: ArraySlice) {
             result.writeBoolean(isFinal)
             result.writeBoolean(isAbstract)
+            result.writeInt(index)
             result.writeIntArray(superTypes)
             result.writeIntArray(vtable)
             result.writeArray(itable) { it.write(this) }
@@ -699,7 +700,7 @@ internal object DFGSerializer {
             println("TYPES: ${typeMap.size}, " +
                     "FUNCTIONS: ${functionSymbolMap.size}, " +
                     "PRIVATE FUNCTIONS: ${functionSymbolMap.keys.count { it is DataFlowIR.FunctionSymbol.Private }}, " +
-                    "FUNCTION TABLE SIZE: ${symbolTable.couldBeCalledVirtuallyIndex}"
+                    "FUNCTION TABLE SIZE: ${symbolTable.privateFunIndex}"
             )
         }
         val types = typeMap.entries
@@ -709,6 +710,7 @@ internal object DFGSerializer {
                             DeclaredType(
                                     type.isFinal,
                                     type.isAbstract,
+                                    type.symbolTableIndex,
                                     type.superTypes.map { typeMap[it]!! }.toIntArray(),
                                     type.vtable.map { functionSymbolMap[it]!! }.toIntArray(),
                                     type.itable.map { (hash, symbol) -> ItableSlot(hash, functionSymbolMap[symbol]!!) }.toTypedArray()
@@ -865,18 +867,26 @@ internal object DFGSerializer {
                         when {
                             external != null -> DataFlowIR.Type.External(external.hash, external.name)
 
-                            public != null ->
+                            public != null -> {
+                                val symbolTableIndex = public.intestines.index
+                                if (symbolTableIndex >= 0)
+                                    ++module.numberOfClasses
                                 DataFlowIR.Type.Public(public.hash, public.intestines.isFinal,
-                                        public.intestines.isAbstract, public.name).also {
+                                        public.intestines.isAbstract, module, symbolTableIndex, public.name).also {
                                     publicTypesMap.put(it.hash, it)
                                     allTypes += it
                                 }
+                            }
 
-                            else ->
+                            else -> {
-                                DataFlowIR.Type.Private(privateTypeIndex++, private!!.intestines.isFinal,
+                                val symbolTableIndex = private!!.intestines.index
-                                        private.intestines.isAbstract, private.name).also {
+                                if (symbolTableIndex >= 0)
+                                    ++module.numberOfClasses
+                                DataFlowIR.Type.Private(privateTypeIndex++, private.intestines.isFinal,
+                                        private.intestines.isAbstract, module, symbolTableIndex, private.name).also {
                                     allTypes += it
                                 }
+                            }
                         }
                     }
                 }
@@ -1120,6 +1130,10 @@ internal object DFGSerializer {
                     println("        $it")
                 }
             }
+
+            functions.forEach {
+                println(it.key)
+            }
         }
 
         return ExternalModulesDFG(allTypes, publicTypesMap, publicFunctionsMap, functions)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DataFlowIR.kt

@@ -18,10 +18,10 @@ package org.jetbrains.kotlin.backend.konan.optimizations
 
 import org.jetbrains.kotlin.backend.common.descriptors.allParameters
 import org.jetbrains.kotlin.backend.konan.Context
-import org.jetbrains.kotlin.backend.konan.descriptors.externalOrIntrinsic
 import org.jetbrains.kotlin.backend.konan.descriptors.isAbstract
 import org.jetbrains.kotlin.backend.konan.descriptors.isInterface
 import org.jetbrains.kotlin.backend.konan.isObjCClass
+import org.jetbrains.kotlin.backend.konan.isValueType
 import org.jetbrains.kotlin.backend.konan.llvm.functionName
 import org.jetbrains.kotlin.backend.konan.llvm.isExported
 import org.jetbrains.kotlin.backend.konan.llvm.localHash
@@ -33,6 +33,7 @@ import org.jetbrains.kotlin.ir.declarations.IrClass
 import org.jetbrains.kotlin.ir.declarations.IrField
 import org.jetbrains.kotlin.ir.declarations.IrFunction
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.ir.descriptors.IrBuiltinOperatorDescriptorBase
 import org.jetbrains.kotlin.ir.expressions.IrCall
 import org.jetbrains.kotlin.ir.expressions.IrFunctionAccessExpression
 import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
@@ -50,7 +51,7 @@ internal object DataFlowIR {
 
     abstract class Type {
         // Special marker type forbidding devirtualization on its instances.
-        object Virtual : Declared(false, true)
+        object Virtual : Declared(false, true, null, -1)
 
         class External(val hash: Long, val name: String? = null) : Type() {
             override fun equals(other: Any?): Boolean {
@@ -69,13 +70,14 @@ internal object DataFlowIR {
             }
         }
 
-        abstract class Declared(val isFinal: Boolean, val isAbstract: Boolean) : Type() {
+        abstract class Declared(val isFinal: Boolean, val isAbstract: Boolean, val module: Module?, val symbolTableIndex: Int) : Type() {
             val superTypes = mutableListOf<Type>()
             val vtable = mutableListOf<FunctionSymbol>()
             val itable = mutableMapOf<Long, FunctionSymbol>()
         }
 
-        class Public(val hash: Long, isFinal: Boolean, isAbstract: Boolean, val name: String? = null) : Declared(isFinal, isAbstract) {
+        class Public(val hash: Long, isFinal: Boolean, isAbstract: Boolean, module: Module, symbolTableIndex: Int, val name: String? = null)
+            : Declared(isFinal, isAbstract, module, symbolTableIndex) {
             override fun equals(other: Any?): Boolean {
                 if (this === other) return true
                 if (other !is Public) return false
@@ -88,11 +90,12 @@ internal object DataFlowIR {
             }
 
             override fun toString(): String {
-                return "PublicType(hash='$hash', name='$name')"
+                return "PublicType(hash='$hash', symbolTableIndex='$symbolTableIndex', name='$name')"
             }
         }
 
-        class Private(val index: Int, isFinal: Boolean, isAbstract: Boolean, val name: String? = null) : Declared(isFinal, isAbstract) {
+        class Private(val index: Int, isFinal: Boolean, isAbstract: Boolean, module: Module, symbolTableIndex: Int, val name: String? = null)
+            : Declared(isFinal, isAbstract, module, symbolTableIndex) {
             override fun equals(other: Any?): Boolean {
                 if (this === other) return true
                 if (other !is Private) return false
@@ -105,13 +108,14 @@ internal object DataFlowIR {
             }
 
             override fun toString(): String {
-                return "PrivateType(index=$index, name='$name')"
+                return "PrivateType(index=$index, symbolTableIndex='$symbolTableIndex', name='$name')"
             }
         }
     }
 
     class Module(val descriptor: ModuleDescriptor) {
         var numberOfFunctions = 0
+        var numberOfClasses = 0
     }
 
     abstract class FunctionSymbol(val numberOfParameters: Int, val isGlobalInitializer: Boolean, val name: String?) {
@@ -154,7 +158,7 @@ internal object DataFlowIR {
             }
 
             override fun toString(): String {
-                return "PublicFunction(hash='$hash', name='$name')"
+                return "PublicFunction(hash='$hash', symbolTableIndex='$symbolTableIndex', name='$name')"
             }
         }
 
@@ -173,7 +177,7 @@ internal object DataFlowIR {
             }
 
             override fun toString(): String {
-                return "PrivateFunction(index=$index, name='$name')"
+                return "PrivateFunction(index=$index, symbolTableIndex='$symbolTableIndex', name='$name')"
             }
         }
     }
@@ -435,7 +439,6 @@ internal object DataFlowIR {
 
         var privateTypeIndex = 0
         var privateFunIndex = 0
-        var couldBeCalledVirtuallyIndex = 0
 
         init {
             irModule.accept(object : IrElementVisitorVoid {
@@ -474,10 +477,12 @@ internal object DataFlowIR {
 
             val isFinal = descriptor.isFinal()
             val isAbstract = descriptor.isAbstract()
+            val placeToClassTable = !descriptor.defaultType.isValueType()
+            val symbolTableIndex = if (placeToClassTable) module.numberOfClasses++ else -1
             val type = if (descriptor.isExported())
-                Type.Public(name.localHash.value, isFinal, isAbstract, takeName { name })
+                           Type.Public(name.localHash.value, isFinal, isAbstract, module, symbolTableIndex, takeName { name })
-            else
+                       else
-                Type.Private(privateTypeIndex++, isFinal, isAbstract, takeName { name })
+                           Type.Private(privateTypeIndex++, isFinal, isAbstract, module, symbolTableIndex, takeName { name })
             if (!descriptor.isInterface) {
                 val vtableBuilder = context.getVtableBuilder(descriptor)
                 type.vtable += vtableBuilder.vtableEntries.map { mapFunction(it.getImplementation(context)) }
@@ -528,7 +533,7 @@ internal object DataFlowIR {
                     is FunctionDescriptor -> {
                         val name = if (it.isExported()) it.symbolName else it.internalName
                         val numberOfParameters = it.allParameters.size + if (it.isSuspend) 1 else 0
-                        if (it.module != irModule.descriptor || it.externalOrIntrinsic()) {
+                        if (it.module != irModule.descriptor || it.isExternal || (it is IrBuiltinOperatorDescriptorBase)) {
                             val escapesAnnotation = it.annotations.findAnnotation(FQ_NAME_ESCAPES)
                             val pointsToAnnotation = it.annotations.findAnnotation(FQ_NAME_POINTS_TO)
                             @Suppress("UNCHECKED_CAST")
@@ -543,9 +548,7 @@ internal object DataFlowIR {
                             val placeToFunctionsTable = !isAbstract && it !is ConstructorDescriptor && classDescriptor != null
                                     && classDescriptor.kind != ClassKind.ANNOTATION_CLASS
                                     && (it.isOverridableOrOverrides || it.name.asString().contains("<bridge-") || !classDescriptor.isFinal())
-                            if (placeToFunctionsTable)
+                            val symbolTableIndex = if (placeToFunctionsTable) module.numberOfFunctions++ else -1
-                                ++module.numberOfFunctions
-                            val symbolTableIndex = if (!placeToFunctionsTable) -1 else couldBeCalledVirtuallyIndex++
                             if (it.isExported())
                                 FunctionSymbol.Public(name.localHash.value, numberOfParameters, module, symbolTableIndex, false, takeName { name })
                             else
@@ -571,5 +574,18 @@ internal object DataFlowIR {
                         }
                         .map { it.key as FunctionDescriptor }
                         .toList()
+
+        fun getPrivateClassesTableForExport() =
+                classMap
+                        .asSequence()
+                        .filter { it.value.let { it is DataFlowIR.Type.Declared && it.symbolTableIndex >= 0 } }
+                        .sortedBy { (it.value as DataFlowIR.Type.Declared).symbolTableIndex }
+                        .apply {
+                            forEachIndexed { index, entry ->
+                                assert((entry.value  as DataFlowIR.Type.Declared).symbolTableIndex == index) { "Inconsistent class table" }
+                            }
+                        }
+                        .map { it.key }
+                        .toList()
     }
 }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/Devirtualization.kt

@@ -21,15 +21,26 @@ import org.jetbrains.kotlin.backend.common.ir.ir2stringWhole
 import org.jetbrains.kotlin.backend.common.pop
 import org.jetbrains.kotlin.backend.common.push
 import org.jetbrains.kotlin.backend.common.lower.createIrBuilder
+import org.jetbrains.kotlin.backend.common.lower.irBlock
 import org.jetbrains.kotlin.backend.konan.*
+import org.jetbrains.kotlin.backend.konan.descriptors.isInterface
+import org.jetbrains.kotlin.backend.konan.ir.IrPrivateClassReferenceImpl
 import org.jetbrains.kotlin.backend.konan.ir.IrPrivateFunctionCallImpl
 import org.jetbrains.kotlin.backend.konan.llvm.*
+import org.jetbrains.kotlin.backend.konan.lower.nullConst
+import org.jetbrains.kotlin.backend.konan.objcexport.getErasedTypeClass
+import org.jetbrains.kotlin.descriptors.ClassDescriptor
+import org.jetbrains.kotlin.ir.builders.*
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
 import org.jetbrains.kotlin.ir.expressions.IrCall
 import org.jetbrains.kotlin.ir.expressions.IrExpression
 import org.jetbrains.kotlin.ir.expressions.getValueArgument
+import org.jetbrains.kotlin.ir.expressions.impl.IrBranchImpl
 import org.jetbrains.kotlin.ir.expressions.impl.IrCallImpl
+import org.jetbrains.kotlin.ir.expressions.impl.IrWhenImpl
+import org.jetbrains.kotlin.ir.symbols.impl.IrClassSymbolImpl
 import org.jetbrains.kotlin.ir.visitors.transformChildrenVoid
+import org.jetbrains.kotlin.types.KotlinType
 import java.util.*
 
 // Devirtualization analysis is performed using Variable Type Analysis algorithm.
@@ -873,37 +884,127 @@ internal object Devirtualization {
 
     private fun devirtualize(irModule: IrModuleFragment, context: Context,
                              devirtualizedCallSites: Map<IrCall, DevirtualizedCallSite>) {
+        val nativePtrType = context.builtIns.nativePtr.defaultType
+        val nativePtrEqualityOperatorSymbol = context.ir.symbols.areEqualByValue.single { it.descriptor.valueParameters[0].type == nativePtrType }
+
         irModule.transformChildrenVoid(object: IrElementTransformerVoidWithContext() {
             override fun visitCall(expression: IrCall): IrExpression {
                 expression.transformChildrenVoid(this)
 
                 val devirtualizedCallSite = devirtualizedCallSites[expression]
-                val actualCallee = devirtualizedCallSite?.possibleCallees?.singleOrNull()?.callee as? DataFlowIR.FunctionSymbol.Declared
+                val possibleCallees = devirtualizedCallSite?.possibleCallees
-                        ?: return expression
+                if (possibleCallees == null
+                        || possibleCallees.any { it.callee is DataFlowIR.FunctionSymbol.External }
+                        || possibleCallees.any { it.receiverType is DataFlowIR.Type.External })
+                    return expression
+
+                val descriptor = expression.descriptor
+                val owner = (descriptor.containingDeclaration as ClassDescriptor)
+                val maxUnfoldFactor = if (owner.isInterface) 3 else 1
+                if (possibleCallees.size > maxUnfoldFactor) {
+                    // Callsite too complicated to devirtualize.
+                    return expression
+                }
+
                 val startOffset = expression.startOffset
                 val endOffset = expression.endOffset
+                val type = if (descriptor.isSuspend)
+                               context.builtIns.nullableAnyType
+                           else descriptor.original.returnType!!
                 val irBuilder = context.createIrBuilder(currentScope!!.scope.scopeOwnerSymbol, startOffset, endOffset)
                 irBuilder.run {
                     val dispatchReceiver = expression.dispatchReceiver!!
-                    return IrPrivateFunctionCallImpl(
+                    return when {
-                            startOffset,
+                        possibleCallees.isEmpty() -> irBlock(expression) {
-                            endOffset,
+                            +irCall(context.ir.symbols.throwInvalidReceiverTypeException).apply {
-                            expression.type,
+                                putValueArgument(0, irCall(context.ir.symbols.kClassImplConstructor, listOf(dispatchReceiver.type)).apply {
-                            expression.symbol,
+                                    putValueArgument(0, irCall(context.ir.symbols.getObjectTypeInfo).apply {
-                            expression.descriptor,
+                                        putValueArgument(0, dispatchReceiver)
-                            (expression as? IrCallImpl)?.typeArguments,
+                                    })
-                            actualCallee.module.descriptor,
+                                })
-                            actualCallee.module.numberOfFunctions,
+                            }
-                            actualCallee.symbolTableIndex
+                            +nullConst(expression, type)
-                    ).apply {
+                        }
-                        this.dispatchReceiver    = dispatchReceiver
+
-                        this.extensionReceiver   = expression.extensionReceiver
+                        possibleCallees.size == 1 -> { // Monomorphic callsite.
-                        expression.descriptor.valueParameters.forEach {
+                            val actualCallee = possibleCallees[0].callee as DataFlowIR.FunctionSymbol.Declared
-                            this.putValueArgument(it.index, expression.getValueArgument(it))
+                            irDevirtualizedCall(expression, type, actualCallee).apply {
+                                this.dispatchReceiver = dispatchReceiver
+                                this.extensionReceiver = expression.extensionReceiver
+                                expression.descriptor.valueParameters.forEach {
+                                    this.putValueArgument(it.index, expression.getValueArgument(it))
+                                }
+                            }
+                        }
+
+                        else -> irBlock(expression) {
+                            val receiver = irTemporary(dispatchReceiver, "receiver")
+                            val extensionReceiver = expression.extensionReceiver?.let { irTemporary(it, "extensionReceiver") }
+                            val parameters = expression.descriptor.valueParameters.associate { it to irTemporary(expression.getValueArgument(it)!!) }
+                            val typeInfo = irTemporary(irCall(context.ir.symbols.getObjectTypeInfo).apply {
+                                putValueArgument(0, irGet(receiver.symbol))
+                            })
+
+                            val branches = possibleCallees.mapIndexed { index, devirtualizedCallee ->
+                                val actualCallee = devirtualizedCallee.callee as DataFlowIR.FunctionSymbol.Declared
+                                val actualReceiverType = devirtualizedCallee.receiverType as DataFlowIR.Type.Declared
+                                val expectedTypeInfo = IrPrivateClassReferenceImpl(
+                                        startOffset,
+                                        endOffset,
+                                        nativePtrType,
+                                        IrClassSymbolImpl(dispatchReceiver.type.getErasedTypeClass()),
+                                        receiver.type,
+                                        actualReceiverType.module!!.descriptor,
+                                        actualReceiverType.module.numberOfClasses,
+                                        actualReceiverType.symbolTableIndex)
+                                val condition =
+                                        if (index == possibleCallees.size - 1)
+                                            irTrue()
+                                        else
+                                            irCall(nativePtrEqualityOperatorSymbol).apply {
+                                                putValueArgument(0, irGet(typeInfo.symbol))
+                                                putValueArgument(1, expectedTypeInfo)
+                                            }
+                                IrBranchImpl(
+                                        startOffset,
+                                        endOffset,
+                                        condition,
+                                        irDevirtualizedCall(expression, type, actualCallee).apply {
+                                            this.dispatchReceiver = irGet(receiver.symbol)
+                                            this.extensionReceiver = extensionReceiver?.let { irGet(it.symbol) }
+                                            expression.descriptor.valueParameters.forEach {
+                                                this.putValueArgument(it.index, irGet(parameters[it]!!.symbol))
+                                            }
+                                        }
+                                )
+                            }
+
+                            +IrWhenImpl(
+                                    startOffset,
+                                    endOffset,
+                                    type,
+                                    expression.origin,
+                                    branches
+                            )
                         }
                     }
                 }
             }
+
+            fun IrBuilderWithScope.irDevirtualizedCall(callee: IrCall, actualType: KotlinType,
+                                                       devirtualizedCallee: DataFlowIR.FunctionSymbol.Declared) =
+                    IrPrivateFunctionCallImpl(
+                            startOffset,
+                            endOffset,
+                            actualType,
+                            callee.symbol,
+                            callee.descriptor,
+                            (callee as? IrCallImpl)?.typeArguments,
+                            devirtualizedCallee.module.descriptor,
+                            devirtualizedCallee.module.numberOfFunctions,
+                            devirtualizedCallee.symbolTableIndex
+                    )
+
         })
     }
 }

runtime/src/main/kotlin/konan/internal/RuntimeUtils.kt

@@ -17,6 +17,7 @@
 package konan.internal
 
 import kotlin.internal.getProgressionLastElement
+import kotlin.reflect.KClass
 
 @ExportForCppRuntime
 fun ThrowNullPointerException(): Nothing {
@@ -37,6 +38,10 @@ fun ThrowTypeCastException(): Nothing {
     throw TypeCastException()
 }
 
+fun ThrowInvalidReceiverTypeException(klass: KClass<*>): Nothing {
+    throw RuntimeException("Unexpected receiver type: " + (klass.qualifiedName ?: "noname"))
+}
+
 @ExportForCppRuntime
 internal fun ThrowArithmeticException() : Nothing {
     throw ArithmeticException()