Implemented devirtualization of monomorphic callsites

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt

@@ -31,8 +31,8 @@ import org.jetbrains.kotlin.descriptors.PropertyDescriptor
 import org.jetbrains.kotlin.descriptors.findClassAcrossModuleDependencies
 import org.jetbrains.kotlin.incremental.components.NoLookupLocation
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
-import org.jetbrains.kotlin.ir.symbols.IrEnumEntrySymbol
 import org.jetbrains.kotlin.ir.symbols.IrClassSymbol
+import org.jetbrains.kotlin.ir.symbols.IrEnumEntrySymbol
 import org.jetbrains.kotlin.ir.symbols.IrSimpleFunctionSymbol
 import org.jetbrains.kotlin.ir.util.SymbolTable
 import org.jetbrains.kotlin.ir.util.constructors
@@ -258,4 +258,4 @@ internal class KonanSymbols(context: Context, val symbolTable: SymbolTable): Sym
                 kind.runtimeKindName, NoLookupLocation.FROM_BACKEND
         ) as ClassDescriptor)
     }
-}
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/KonanIr.kt

@@ -16,16 +16,20 @@
 
 package org.jetbrains.kotlin.backend.konan.ir
 
-import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
+import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptor
 import org.jetbrains.kotlin.ir.SourceManager
 import org.jetbrains.kotlin.ir.SourceRangeInfo
 import org.jetbrains.kotlin.ir.declarations.IrDeclaration
 import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.declarations.IrVariable
+import org.jetbrains.kotlin.ir.expressions.IrCall
 import org.jetbrains.kotlin.ir.expressions.IrExpression
+import org.jetbrains.kotlin.ir.expressions.impl.IrCallWithIndexedArgumentsBase
 import org.jetbrains.kotlin.ir.expressions.impl.IrExpressionBase
+import org.jetbrains.kotlin.ir.symbols.IrClassSymbol
 import org.jetbrains.kotlin.ir.symbols.IrFileSymbol
+import org.jetbrains.kotlin.ir.symbols.IrFunctionSymbol
 import org.jetbrains.kotlin.ir.visitors.IrElementTransformer
 import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
 import org.jetbrains.kotlin.konan.file.File
@@ -177,3 +181,32 @@ class IrSuspendableExpressionImpl(startOffset: Int, endOffset: Int, type: Kotlin
         result = result.transform(transformer, data)
     }
 }
+
+internal interface IrPrivateFunctionCall : IrCall {
+    val moduleDescriptor: ModuleDescriptor
+    val totalFunctions: Int
+    val functionIndex: Int
+}
+
+internal class IrPrivateFunctionCallImpl(startOffset: Int,
+                                         endOffset: Int,
+                                         type: KotlinType,
+                                         override val symbol: IrFunctionSymbol,
+                                         override val descriptor: FunctionDescriptor,
+                                         typeArguments: Map<TypeParameterDescriptor, KotlinType>?,
+                                         override val moduleDescriptor: ModuleDescriptor,
+                                         override val totalFunctions: Int,
+                                         override val functionIndex: Int
+) : IrPrivateFunctionCall,
+        IrCallWithIndexedArgumentsBase(startOffset, endOffset, type, symbol.descriptor.valueParameters.size, typeArguments) {
+
+    override val superQualifierSymbol: IrClassSymbol?
+        get() = null
+
+    override val superQualifier: ClassDescriptor?
+        get() = null
+
+    override fun <R, D> accept(visitor: IrElementVisitor<R, D>, data: D): R =
+            visitor.visitCall(this, data)
+
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BinaryInterface.kt

@@ -272,3 +272,5 @@ internal val ClassDescriptor.objectInstanceFieldSymbolName: String
 
 internal val ClassDescriptor.typeInfoHasVtableAttached: Boolean
     get() = !this.isAbstract() && !this.isExternalObjCClass()
+
+internal val ModuleDescriptor.privateFunctionsTableSymbolName get() = "private_functions_${name.asString()}"

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -79,9 +79,8 @@ internal fun emitLLVM(context: Context) {
         moduleDFG = ModuleDFGBuilder(context, irModule).build()
     }
 
-    phaser.phase(KonanPhase.SERIALIZE_DFG) {
+    val lifetimes = mutableMapOf<IrElement, Lifetime>()
-        DFGSerializer.serialize(context, moduleDFG!!)
+    val codegenVisitor = CodeGeneratorVisitor(context, lifetimes)
-    }
 
     @Suppress("ASSIGNED_BUT_NEVER_ACCESSED_VARIABLE")
     var externalModulesDFG: ExternalModulesDFG? = null
@@ -89,12 +88,17 @@ internal fun emitLLVM(context: Context) {
         externalModulesDFG = DFGSerializer.deserialize(context, moduleDFG!!.symbolTable.privateTypeIndex, moduleDFG!!.symbolTable.privateFunIndex)
     }
 
-    val lifetimes = mutableMapOf<IrElement, Lifetime>()
-    val codegenVisitor = CodeGeneratorVisitor(context, lifetimes)
     @Suppress("ASSIGNED_BUT_NEVER_ACCESSED_VARIABLE")
     var devirtualizationAnalysisResult: Map<DataFlowIR.Node.VirtualCall, Devirtualization.DevirtualizedCallSite>? = null
     phaser.phase(KonanPhase.DEVIRTUALIZATION) {
-        devirtualizationAnalysisResult = Devirtualization.analyze(context, moduleDFG!!, externalModulesDFG!!)
+        devirtualizationAnalysisResult = Devirtualization.run(irModule, context, moduleDFG!!, externalModulesDFG!!)
+
+        val privateFunctions = moduleDFG!!.symbolTable.getPrivateFunctionsTableForExport()
+
+        codegenVisitor.codegen.staticData.placeGlobalConstArray(irModule.descriptor.privateFunctionsTableSymbolName, int8TypePtr,
+                privateFunctions.map { constPointer(codegenVisitor.codegen.functionLlvmValue(it)).bitcast(int8TypePtr) },
+                isExported = true
+        )
     }
 
     phaser.phase(KonanPhase.ESCAPE_ANALYSIS) {
@@ -102,6 +106,10 @@ internal fun emitLLVM(context: Context) {
         EscapeAnalysis.computeLifetimes(moduleDFG!!, externalModulesDFG!!, callGraph, lifetimes)
     }
 
+    phaser.phase(KonanPhase.SERIALIZE_DFG) {
+        DFGSerializer.serialize(context, moduleDFG!!)
+    }
+
     phaser.phase(KonanPhase.CODEGEN) {
         irModule.acceptVoid(codegenVisitor)
     }
@@ -1945,6 +1953,9 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
             return evaluateIntrinsicCall(callee, argsWithContinuationIfNeeded)
         }
 
+        if (callee is IrPrivateFunctionCall)
+            return evaluatePrivateFunctionCall(callee, argsWithContinuationIfNeeded, resultLifetime)
+
         when (descriptor) {
             is IrBuiltinOperatorDescriptorBase -> return evaluateOperatorCall      (callee, argsWithContinuationIfNeeded)
             is ConstructorDescriptor           -> return evaluateConstructorCall   (callee, argsWithContinuationIfNeeded)
@@ -1955,6 +1966,25 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
 
     //-------------------------------------------------------------------------//
 
+    private fun evaluatePrivateFunctionCall(callee: IrPrivateFunctionCall, args: List<LLVMValueRef>, resultLifetime: Lifetime): LLVMValueRef {
+        val functionsListName = callee.moduleDescriptor.privateFunctionsTableSymbolName
+        // LLVM inlines access to this global array (with -opt option on).
+        val functionsList =
+                if (callee.moduleDescriptor == context.irModule!!.descriptor)
+                    LLVMGetNamedGlobal(context.llvmModule, functionsListName)
+                else
+                    codegen.importGlobal(functionsListName, LLVMArrayType(int8TypePtr, callee.totalFunctions)!!, callee.moduleDescriptor.llvmSymbolOrigin)
+        val functionIndex    = Int32(callee.functionIndex).llvm
+        val functionPlacePtr = LLVMBuildGEP(functionGenerationContext.builder, functionsList, cValuesOf(kImmZero, functionIndex), 2, "")!!
+        val functionPtr      = functionGenerationContext.load(functionPlacePtr)
+
+        val functionPtrType  = pointerType(codegen.getLlvmFunctionType(callee.descriptor))
+        val function         = functionGenerationContext.bitcast(functionPtrType, functionPtr)
+        return call(callee.descriptor, function, args, resultLifetime)
+    }
+
+    //-------------------------------------------------------------------------//
+
     private fun evaluateSimpleFunctionCall(
             descriptor: FunctionDescriptor, args: List<LLVMValueRef>,
             resultLifetime: Lifetime, superClass: ClassDescriptor? = null): LLVMValueRef {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticData.kt

@@ -162,9 +162,9 @@ internal class StaticData(override val context: Context): ContextUtils {
     /**
      * Creates array-typed global with given name and value.
      */
-    fun placeGlobalArray(name: String, elemType: LLVMTypeRef?, elements: List<ConstValue>): Global {
+    fun placeGlobalArray(name: String, elemType: LLVMTypeRef?, elements: List<ConstValue>, isExported: Boolean = false): Global {
         val initializer = ConstArray(elemType, elements)
-        val global = placeGlobal(name, initializer)
+        val global = placeGlobal(name, initializer, isExported)
 
         return global
     }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticDataUtils.kt

@@ -27,9 +27,10 @@ import llvm.*
  */
 internal fun StaticData.placeGlobalConstArray(name: String,
                                               elemType: LLVMTypeRef,
-                                              elements: List<ConstValue>): ConstPointer {
+                                              elements: List<ConstValue>,
-    if (elements.isNotEmpty()) {
+                                              isExported: Boolean = false): ConstPointer {
-        val global = this.placeGlobalArray(name, elemType, elements)
+    if (elements.isNotEmpty() || isExported) {
+        val global = this.placeGlobalArray(name, elemType, elements, isExported)
         global.setConstant(true)
         return global.pointer.getElementPtr(0)
     } else {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/Devirtualization.kt

@@ -16,11 +16,20 @@
 
 package org.jetbrains.kotlin.backend.konan.optimizations
 
+import org.jetbrains.kotlin.backend.common.IrElementTransformerVoidWithContext
 import org.jetbrains.kotlin.backend.common.ir.ir2stringWhole
 import org.jetbrains.kotlin.backend.common.pop
 import org.jetbrains.kotlin.backend.common.push
+import org.jetbrains.kotlin.backend.common.lower.createIrBuilder
 import org.jetbrains.kotlin.backend.konan.*
+import org.jetbrains.kotlin.backend.konan.ir.IrPrivateFunctionCallImpl
 import org.jetbrains.kotlin.backend.konan.llvm.*
+import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.ir.expressions.IrCall
+import org.jetbrains.kotlin.ir.expressions.IrExpression
+import org.jetbrains.kotlin.ir.expressions.getValueArgument
+import org.jetbrains.kotlin.ir.expressions.impl.IrCallImpl
+import org.jetbrains.kotlin.ir.visitors.transformChildrenVoid
 import java.util.*
 
 // TODO: Exceptions.
@@ -845,8 +854,51 @@ internal object Devirtualization {
 
     class DevirtualizedCallSite(val possibleCallees: List<DevirtualizedCallee>)
 
-    fun analyze(context: Context, moduleDFG: ModuleDFG, externalModulesDFG: ExternalModulesDFG)
+    fun run(irModule: IrModuleFragment, context: Context, moduleDFG: ModuleDFG, externalModulesDFG: ExternalModulesDFG)
             : Map<DataFlowIR.Node.VirtualCall, DevirtualizedCallSite> {
-        return DevirtualizationAnalysis(context, moduleDFG, externalModulesDFG).analyze()
+        val devirtualizationAnalysisResult = DevirtualizationAnalysis(context, moduleDFG, externalModulesDFG).analyze()
+        val devirtualizedCallSites =
+                devirtualizationAnalysisResult
+                        .asSequence()
+                        .filter { it.key.callSite != null }
+                        .associate { it.key.callSite!! to it.value }
+        Devirtualization.devirtualize(irModule, context, devirtualizedCallSites)
+        return devirtualizationAnalysisResult
+    }
+
+    private fun devirtualize(irModule: IrModuleFragment, context: Context,
+                             devirtualizedCallSites: Map<IrCall, DevirtualizedCallSite>) {
+        irModule.transformChildrenVoid(object: IrElementTransformerVoidWithContext() {
+            override fun visitCall(expression: IrCall): IrExpression {
+                expression.transformChildrenVoid(this)
+
+                val devirtualizedCallSite = devirtualizedCallSites[expression]
+                val actualCallee = devirtualizedCallSite?.possibleCallees?.singleOrNull()?.callee as? DataFlowIR.FunctionSymbol.Declared
+                        ?: return expression
+                val startOffset = expression.startOffset
+                val endOffset = expression.endOffset
+                val irBuilder = context.createIrBuilder(currentScope!!.scope.scopeOwnerSymbol, startOffset, endOffset)
+                irBuilder.run {
+                    val dispatchReceiver = expression.dispatchReceiver!!
+                    return IrPrivateFunctionCallImpl(
+                            startOffset,
+                            endOffset,
+                            expression.type,
+                            expression.symbol,
+                            expression.descriptor,
+                            (expression as? IrCallImpl)?.typeArguments,
+                            actualCallee.module.descriptor,
+                            actualCallee.module.numberOfFunctions,
+                            actualCallee.symbolTableIndex
+                    ).apply {
+                        this.dispatchReceiver    = dispatchReceiver
+                        this.extensionReceiver   = expression.extensionReceiver
+                        expression.descriptor.valueParameters.forEach {
+                            this.putValueArgument(it.index, expression.getValueArgument(it))
+                        }
+                    }
+                }
+            }
+        })
     }
 }