diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt index f86eb4ab7a2..f15ea7e8e60 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt @@ -18,7 +18,6 @@ import org.jetbrains.kotlin.backend.konan.library.KonanLibraryWriter import org.jetbrains.kotlin.backend.konan.library.LinkData import org.jetbrains.kotlin.backend.konan.llvm.* import org.jetbrains.kotlin.backend.konan.lower.DECLARATION_ORIGIN_BRIDGE_METHOD -import org.jetbrains.kotlin.backend.konan.optimizations.DataFlowIR import org.jetbrains.kotlin.backend.konan.optimizations.Devirtualization import org.jetbrains.kotlin.backend.konan.optimizations.ExternalModulesDFG import org.jetbrains.kotlin.backend.konan.optimizations.ModuleDFG @@ -321,9 +320,6 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) { val coverage = CoverageManager(this) - lateinit var privateFunctions: List> - lateinit var privateClasses: List> - // Cache used for source offset->(line,column) mapping. val fileEntryCache = mutableMapOf() diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt index c418add18b8..846ee36db68 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt @@ -268,6 +268,8 @@ internal val bitcodePhase = namedIrModulePhase( lower = contextLLVMSetupPhase then RTTIPhase then generateDebugInfoHeaderPhase then + buildDFGPhase then + serializeDFGPhase then deserializeDFGPhase then devirtualizationPhase then escapeAnalysisPhase then @@ -295,8 +297,6 @@ internal val toplevelPhase = namedUnitPhase( dependenciesLowerPhase then // Then lower all libraries in topological order. // With that we guarantee that inline functions are unlowered while being inlined. moduleIndexForCodegenPhase then - buildDFGPhase then - serializeDFGPhase then bitcodePhase then produceOutputPhase then verifyBitcodePhase then @@ -309,9 +309,7 @@ internal val toplevelPhase = namedUnitPhase( internal fun PhaseConfig.konanPhasesConfig(config: KonanConfig) { with(config.configuration) { disable(compileTimeEvaluatePhase) - disable(buildDFGPhase) disable(deserializeDFGPhase) - disable(devirtualizationPhase) disable(escapeAnalysisPhase) disable(serializeDFGPhase) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt index 6ce9c60c4d5..bf3c229e029 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt @@ -384,6 +384,8 @@ internal class KonanSymbols(context: Context, val symbolTable: SymbolTable, val val returnIfSuspended = internalFunction("returnIfSuspended") + val coroutineLaunchpad = internalFunction("coroutineLaunchpad") + val konanSuspendCoroutineUninterceptedOrReturn = internalFunction("suspendCoroutineUninterceptedOrReturn") val konanCoroutineContextGetter = internalFunction("getCoroutineContext") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/KonanIr.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/KonanIr.kt index 262ebbc519b..597421c0f5d 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/KonanIr.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/KonanIr.kt @@ -6,10 +6,8 @@ package org.jetbrains.kotlin.backend.konan.ir import org.jetbrains.kotlin.backend.konan.SYNTHETIC_OFFSET -import org.jetbrains.kotlin.backend.konan.optimizations.DataFlowIR import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptor -import org.jetbrains.kotlin.ir.IrStatement import org.jetbrains.kotlin.ir.SourceManager import org.jetbrains.kotlin.ir.SourceManager.FileEntry import org.jetbrains.kotlin.ir.SourceRangeInfo @@ -18,14 +16,7 @@ import org.jetbrains.kotlin.ir.declarations.IrDeclaration import org.jetbrains.kotlin.ir.declarations.IrFile import org.jetbrains.kotlin.ir.declarations.MetadataSource import org.jetbrains.kotlin.ir.expressions.* -import org.jetbrains.kotlin.ir.expressions.impl.IrContainerExpressionBase -import org.jetbrains.kotlin.ir.expressions.impl.IrExpressionBase -import org.jetbrains.kotlin.ir.expressions.impl.IrTerminalDeclarationReferenceBase -import org.jetbrains.kotlin.ir.symbols.IrClassifierSymbol -import org.jetbrains.kotlin.ir.symbols.IrFileSymbol -import org.jetbrains.kotlin.ir.symbols.IrReturnableBlockSymbol -import org.jetbrains.kotlin.ir.symbols.impl.IrReturnableBlockSymbolImpl -import org.jetbrains.kotlin.ir.types.IrType +import org.jetbrains.kotlin.ir.symbols.* import org.jetbrains.kotlin.ir.visitors.IrElementTransformer import org.jetbrains.kotlin.ir.visitors.IrElementVisitor import org.jetbrains.kotlin.konan.file.File @@ -131,91 +122,4 @@ class IrFileImpl(entry: SourceManager.FileEntry) : IrFile { override fun accept(visitor: IrElementVisitor, data: D): R { TODO("not implemented") } -} - -//-----------------------------------------------------------------------------// - -internal interface IrPrivateFunctionCall : IrExpression { - val valueArgumentsCount: Int - fun getValueArgument(index: Int): IrExpression? - fun putValueArgument(index: Int, valueArgument: IrExpression?) - fun removeValueArgument(index: Int) - - val virtualCallee: IrCall? - val dfgSymbol: DataFlowIR.FunctionSymbol.Declared - val moduleDescriptor: ModuleDescriptor - val totalFunctions: Int - val functionIndex: Int -} - -internal class IrPrivateFunctionCallImpl(startOffset: Int, - endOffset: Int, - type: IrType, - override val valueArgumentsCount: Int, - override val virtualCallee: IrCall?, - override val dfgSymbol: DataFlowIR.FunctionSymbol.Declared, - override val moduleDescriptor: ModuleDescriptor, - override val totalFunctions: Int, - override val functionIndex: Int -) : IrPrivateFunctionCall, IrExpressionBase(startOffset, endOffset, type) { - - override fun accept(visitor: IrElementVisitor, data: D): R { - return visitor.visitExpression(this, data) - } - - private val argumentsByParameterIndex: Array = arrayOfNulls(valueArgumentsCount) - - override fun getValueArgument(index: Int): IrExpression? { - if (index >= valueArgumentsCount) { - throw AssertionError("$this: No such value argument slot: $index") - } - return argumentsByParameterIndex[index] - } - - override fun putValueArgument(index: Int, valueArgument: IrExpression?) { - if (index >= valueArgumentsCount) { - throw AssertionError("$this: No such value argument slot: $index") - } - argumentsByParameterIndex[index] = valueArgument - } - - override fun removeValueArgument(index: Int) { - argumentsByParameterIndex[index] = null - } - - override fun acceptChildren(visitor: IrElementVisitor, data: D) { - argumentsByParameterIndex.forEach { it?.accept(visitor, data) } - } - - override fun transformChildren(transformer: IrElementTransformer, data: D) { - argumentsByParameterIndex.forEachIndexed { i, irExpression -> - argumentsByParameterIndex[i] = irExpression?.transform(transformer, data) - } - } -} - -internal interface IrPrivateClassReference : IrClassReference { - val moduleDescriptor: ModuleDescriptor - val totalClasses: Int - val classIndex: Int - val dfgSymbol: DataFlowIR.Type.Declared -} - -internal class IrPrivateClassReferenceImpl(startOffset: Int, - endOffset: Int, - type: IrType, - symbol: IrClassifierSymbol, - override val classType: IrType, - override val moduleDescriptor: ModuleDescriptor, - override val totalClasses: Int, - override val classIndex: Int, - override val dfgSymbol: DataFlowIR.Type.Declared -) : IrPrivateClassReference, - IrTerminalDeclarationReferenceBase( - startOffset, endOffset, type, - symbol, symbol.descriptor - ) -{ - override fun accept(visitor: IrElementVisitor, data: D): R = - visitor.visitClassReference(this, data) } \ No newline at end of file diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BitcodePhases.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BitcodePhases.kt index d9554ad903d..0149adfc7f8 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BitcodePhases.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/BitcodePhases.kt @@ -63,38 +63,11 @@ internal val deserializeDFGPhase = makeKonanModuleOpPhase( internal val devirtualizationPhase = makeKonanModuleOpPhase( name = "Devirtualization", description = "Devirtualization", - prerequisite = setOf(buildDFGPhase, deserializeDFGPhase), + prerequisite = setOf(buildDFGPhase), op = { context, irModule -> - context.externalModulesDFG?.let { externalModulesDFG -> - context.devirtualizationAnalysisResult = Devirtualization.run( - irModule, context, context.moduleDFG!!, externalModulesDFG - ) - - val privateFunctions = context.moduleDFG!!.symbolTable.getPrivateFunctionsTableForExport() - privateFunctions.forEachIndexed { index, it -> - val function = context.codegenVisitor.codegen.llvmFunction(it.first) - LLVMAddAlias( - context.llvmModule, - function.type, - function, - irModule.descriptor.privateFunctionSymbolName(index, it.second.name) - )!! - } - context.privateFunctions = privateFunctions - - val privateClasses = context.moduleDFG!!.symbolTable.getPrivateClassesTableForExport() - - privateClasses.forEachIndexed { index, it -> - val typeInfoPtr = context.codegenVisitor.codegen.typeInfoValue(it.first) - LLVMAddAlias( - context.llvmModule, - typeInfoPtr.type, - typeInfoPtr, - irModule.descriptor.privateClassSymbolName(index, it.second.name) - )!! - } - context.privateClasses = privateClasses - } + context.devirtualizationAnalysisResult = Devirtualization.run( + irModule, context, context.moduleDFG!!, ExternalModulesDFG(emptyList(), emptyMap(), emptyMap(), emptyMap()) + ) } ) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt index 6c70a324677..823025138b0 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt @@ -5,12 +5,15 @@ import llvm.* import org.jetbrains.kotlin.backend.konan.descriptors.TypedIntrinsic import org.jetbrains.kotlin.backend.konan.descriptors.isTypedIntrinsic import org.jetbrains.kotlin.backend.konan.llvm.objc.genObjCSelector +import org.jetbrains.kotlin.backend.konan.ir.isSuspend import org.jetbrains.kotlin.backend.konan.reportCompilationError import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.IrClass import org.jetbrains.kotlin.ir.declarations.IrConstructor import org.jetbrains.kotlin.ir.declarations.IrFunction import org.jetbrains.kotlin.ir.expressions.* import org.jetbrains.kotlin.ir.types.getClass +import org.jetbrains.kotlin.ir.util.dump import org.jetbrains.kotlin.name.Name internal enum class IntrinsicType { @@ -64,6 +67,7 @@ internal enum class IntrinsicType { // Coroutines GET_CONTINUATION, RETURN_IF_SUSPEND, + COROUTINE_LAUNCHPAD, // Interop INTEROP_READ_BITS, INTEROP_WRITE_BITS, @@ -98,7 +102,7 @@ internal interface IntrinsicGeneratorEnvironment { fun calculateLifetime(element: IrElement): Lifetime - fun evaluateCall(function: IrFunction, args: List, resultLifetime: Lifetime): LLVMValueRef + fun evaluateCall(function: IrFunction, args: List, resultLifetime: Lifetime, superClass: IrClass? = null): LLVMValueRef fun evaluateExplicitArgs(expression: IrMemberAccessExpression): List @@ -155,9 +159,8 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv environment.functionGenerationContext.genObjCSelector(selector) } IntrinsicType.INIT_INSTANCE -> { - val callee = callSite as IrCall - val initializer = callee.getValueArgument(1) as IrCall - val thiz = environment.evaluateExpression(callee.getValueArgument(0)!!) + val initializer = callSite.getValueArgument(1) as IrCall + val thiz = environment.evaluateExpression(callSite.getValueArgument(0)!!) environment.evaluateCall( initializer.symbol.owner, listOf(thiz) + environment.evaluateExplicitArgs(initializer), @@ -165,6 +168,17 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv ) codegen.theUnitInstanceRef.llvm } + IntrinsicType.COROUTINE_LAUNCHPAD -> { + val suspendFunctionCall = callSite.getValueArgument(0) as IrCall + val continuation = environment.evaluateExpression(callSite.getValueArgument(1)!!) + val suspendFunction = suspendFunctionCall.symbol.owner + assert(suspendFunction.isSuspend) { "Call to a suspend function expected but was ${suspendFunction.dump()}" } + environment.evaluateCall(suspendFunction, + environment.evaluateExplicitArgs(suspendFunctionCall) + listOf(continuation), + environment.calculateLifetime(suspendFunctionCall), + suspendFunction.parent as? IrClass // Call non-virtually. + ) + } else -> null } } @@ -238,6 +252,7 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv reportNonLoweredIntrinsic(intrinsicType) IntrinsicType.INIT_INSTANCE, IntrinsicType.OBJC_INIT_BY, + IntrinsicType.COROUTINE_LAUNCHPAD, IntrinsicType.OBJC_GET_SELECTOR, IntrinsicType.IMMUTABLE_BLOB -> reportSpecialIntrinsic(intrinsicType) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 0a190346bb0..b2503565411 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -13,7 +13,6 @@ import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.descriptors.* import org.jetbrains.kotlin.backend.konan.ir.* import org.jetbrains.kotlin.backend.konan.llvm.coverage.* -import org.jetbrains.kotlin.backend.konan.objcexport.ObjCExport import org.jetbrains.kotlin.backend.konan.optimizations.* import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.builtins.UnsignedType @@ -220,8 +219,8 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map, resultLifetime: Lifetime): LLVMValueRef = - evaluateSimpleFunctionCall(function, args, resultLifetime) + override fun evaluateCall(function: IrFunction, args: List, resultLifetime: Lifetime, superClass: IrClass?) = + evaluateSimpleFunctionCall(function, args, resultLifetime, superClass) override fun evaluateExplicitArgs(expression: IrMemberAccessExpression): List = this@CodeGeneratorVisitor.evaluateExplicitArgs(expression) @@ -794,9 +793,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map return evaluateSuspendableExpression (value) is IrSuspensionPoint -> return evaluateSuspensionPoint (value) - is IrPrivateFunctionCall -> return evaluatePrivateFunctionCall (value) - is IrPrivateClassReference -> - return evaluatePrivateClassReference (value) + is IrClassReference -> return evaluateClassReference (value) else -> { TODO(ir2string(value)) } @@ -2017,42 +2014,8 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map - callee.getValueArgument(index)?.let { evaluateExpression(it) } - ?: run { - assert(index == callee.valueArgumentsCount - 1) { "Only last argument may be null - for suspend functions" } - getContinuation() - } - } - val dfgSymbol = callee.dfgSymbol - val functionIndex = callee.functionIndex - val function = if (callee.moduleDescriptor == context.irModule!!.descriptor) { - codegen.llvmFunction(context.privateFunctions[functionIndex].first) - } else { - context.llvm.externalFunction( - callee.moduleDescriptor.privateFunctionSymbolName(functionIndex, callee.dfgSymbol.name), - codegen.getLlvmFunctionType(dfgSymbol), - callee.moduleDescriptor.llvmSymbolOrigin - - ) - } - return call(dfgSymbol, function, args, resultLifetime = Lifetime.GLOBAL) - } - - //-------------------------------------------------------------------------// - - private fun evaluatePrivateClassReference(classReference: IrPrivateClassReference): LLVMValueRef { - val classIndex = classReference.classIndex - val typeInfoPtr = if (classReference.moduleDescriptor == context.irModule!!.descriptor) { - codegen.typeInfoValue(context.privateClasses[classIndex].first) - } else { - codegen.importGlobal( - classReference.moduleDescriptor.privateClassSymbolName(classIndex, classReference.dfgSymbol.name), - codegen.kTypeInfo, - classReference.moduleDescriptor.llvmSymbolOrigin - ) - } + private fun evaluateClassReference(classReference: IrClassReference): LLVMValueRef { + val typeInfoPtr = codegen.typeInfoValue(classReference.symbol.owner as IrClass) return functionGenerationContext.bitcast(int8TypePtr, typeInfoPtr) } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGBuilder.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGBuilder.kt index 58ec53f2081..298f8427067 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGBuilder.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGBuilder.kt @@ -479,11 +479,32 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag ) } + private fun IrType.erasure(): IrType { + if (this !is IrSimpleType) return this + + val classifier = this.classifier + return when (classifier) { + is IrClassSymbol -> this + is IrTypeParameterSymbol -> { + val upperBound = classifier.owner.superTypes.singleOrNull() ?: + TODO("${classifier.descriptor} : ${classifier.descriptor.upperBounds}") + + if (this.hasQuestionMark) { + // `T?` + upperBound.erasure().makeNullable() + } else { + upperBound.erasure() + } + } + else -> TODO(classifier.toString()) + } + } + private fun expressionToEdge(expression: IrExpression) = if (expression is IrTypeOperatorCall && expression.operator.isCast()) DataFlowIR.Edge( getNode(expression.argument), - symbolTable.mapClassReferenceType(expression.typeOperand.getClass()!!) + symbolTable.mapClassReferenceType(expression.typeOperand.erasure().getClass()!!) ) else DataFlowIR.Edge(getNode(expression), null) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGSerializer.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGSerializer.kt index 2821e766353..02429789bd1 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGSerializer.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGSerializer.kt @@ -986,7 +986,7 @@ internal object DFGSerializer { ++module.numberOfClasses DataFlowIR.Type.Public(public.hash, public.intestines.base.isFinal, public.intestines.base.isAbstract, public.intestines.base.primitiveBinaryType, - module, symbolTableIndex, public.intestines.base.name).also { + module, symbolTableIndex, null, public.intestines.base.name).also { publicTypesMap.put(it.hash, it) allTypes += it } @@ -998,7 +998,7 @@ internal object DFGSerializer { ++module.numberOfClasses DataFlowIR.Type.Private(privateTypeIndex++, private.intestines.base.isFinal, private.intestines.base.isAbstract, private.intestines.base.primitiveBinaryType, - module, symbolTableIndex, private.intestines.base.name).also { + module, symbolTableIndex, null, private.intestines.base.name).also { allTypes += it } } @@ -1013,14 +1013,14 @@ internal object DFGSerializer { val private = it.private when { external != null -> - DataFlowIR.FunctionSymbol.External(external.hash, attributes, external.name) + DataFlowIR.FunctionSymbol.External(external.hash, attributes, null, external.name) public != null -> { val symbolTableIndex = public.index if (symbolTableIndex >= 0) ++module.numberOfFunctions DataFlowIR.FunctionSymbol.Public(public.hash, - module, symbolTableIndex, attributes, null, public.name).also { + module, symbolTableIndex, attributes, null, null, public.name).also { publicFunctionsMap.put(it.hash, it) } } @@ -1030,7 +1030,7 @@ internal object DFGSerializer { if (symbolTableIndex >= 0) ++module.numberOfFunctions DataFlowIR.FunctionSymbol.Private(privateFunIndex++, - module, symbolTableIndex, attributes, null, private.name) + module, symbolTableIndex, attributes, null, null, private.name) } }.apply { escapes = it.base.escapes diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DataFlowIR.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DataFlowIR.kt index ac96682f0cc..e3da55b5e0e 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DataFlowIR.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DataFlowIR.kt @@ -41,7 +41,7 @@ internal object DataFlowIR { val primitiveBinaryType: PrimitiveBinaryType?, val name: String?) { // Special marker type forbidding devirtualization on its instances. - object Virtual : Declared(false, true, null, null, -1, "\$VIRTUAL") + object Virtual : Declared(false, true, null, null, -1, null, "\$VIRTUAL") class External(val hash: Long, isFinal: Boolean, isAbstract: Boolean, primitiveBinaryType: PrimitiveBinaryType?, name: String? = null) @@ -63,7 +63,7 @@ internal object DataFlowIR { } abstract class Declared(isFinal: Boolean, isAbstract: Boolean, primitiveBinaryType: PrimitiveBinaryType?, - val module: Module?, val symbolTableIndex: Int, name: String?) + val module: Module?, val symbolTableIndex: Int, val irClass: IrClass?, name: String?) : Type(isFinal, isAbstract, primitiveBinaryType, name) { val superTypes = mutableListOf() val vtable = mutableListOf() @@ -71,8 +71,8 @@ internal object DataFlowIR { } class Public(val hash: Long, isFinal: Boolean, isAbstract: Boolean, primitiveBinaryType: PrimitiveBinaryType?, - module: Module, symbolTableIndex: Int, name: String? = null) - : Declared(isFinal, isAbstract, primitiveBinaryType, module, symbolTableIndex, name) { + module: Module, symbolTableIndex: Int, irClass: IrClass?, name: String? = null) + : Declared(isFinal, isAbstract, primitiveBinaryType, module, symbolTableIndex, irClass, name) { override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is Public) return false @@ -90,8 +90,8 @@ internal object DataFlowIR { } class Private(val index: Int, isFinal: Boolean, isAbstract: Boolean, primitiveBinaryType: PrimitiveBinaryType?, - module: Module, symbolTableIndex: Int, name: String? = null) - : Declared(isFinal, isAbstract, primitiveBinaryType, module, symbolTableIndex, name) { + module: Module, symbolTableIndex: Int, irClass: IrClass?, name: String? = null) + : Declared(isFinal, isAbstract, primitiveBinaryType, module, symbolTableIndex, irClass, name) { override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is Private) return false @@ -122,7 +122,7 @@ internal object DataFlowIR { class FunctionParameter(val type: Type, val boxFunction: FunctionSymbol?, val unboxFunction: FunctionSymbol?) - abstract class FunctionSymbol(val attributes: Int, val name: String?) { + abstract class FunctionSymbol(val attributes: Int, val irFunction: IrFunction?, val name: String?) { lateinit var parameters: Array lateinit var returnParameter: FunctionParameter @@ -133,8 +133,8 @@ internal object DataFlowIR { var escapes: Int? = null var pointsTo: IntArray? = null - class External(val hash: Long, attributes: Int, name: String? = null) - : FunctionSymbol(attributes, name) { + class External(val hash: Long, attributes: Int, irFunction: IrFunction?, name: String? = null) + : FunctionSymbol(attributes, irFunction, name) { override fun equals(other: Any?): Boolean { if (this === other) return true @@ -153,14 +153,14 @@ internal object DataFlowIR { } abstract class Declared(val module: Module, val symbolTableIndex: Int, - attributes: Int, var bridgeTarget: FunctionSymbol?, name: String?) - : FunctionSymbol(attributes, name) { + attributes: Int, irFunction: IrFunction?, var bridgeTarget: FunctionSymbol?, name: String?) + : FunctionSymbol(attributes, irFunction, name) { } class Public(val hash: Long, module: Module, symbolTableIndex: Int, - attributes: Int, bridgeTarget: FunctionSymbol?, name: String? = null) - : Declared(module, symbolTableIndex, attributes, bridgeTarget, name) { + attributes: Int, irFunction: IrFunction?, bridgeTarget: FunctionSymbol?, name: String? = null) + : Declared(module, symbolTableIndex, attributes, irFunction, bridgeTarget, name) { override fun equals(other: Any?): Boolean { if (this === other) return true @@ -179,8 +179,8 @@ internal object DataFlowIR { } class Private(val index: Int, module: Module, symbolTableIndex: Int, - attributes: Int, bridgeTarget: FunctionSymbol?, name: String? = null) - : Declared(module, symbolTableIndex, attributes, bridgeTarget, name) { + attributes: Int, irFunction: IrFunction?, bridgeTarget: FunctionSymbol?, name: String? = null) + : Declared(module, symbolTableIndex, attributes, irFunction, bridgeTarget, name) { override fun equals(other: Any?): Boolean { if (this === other) return true @@ -500,10 +500,6 @@ internal object DataFlowIR { val isFinal = irClass.isFinal() val isAbstract = irClass.isAbstract() val name = irClass.fqNameSafe.asString() - if (irClass.module != irModule.descriptor) - return classMap.getOrPut(irClass) { - Type.External(name.localHash.value, isFinal, isAbstract, null, takeName { name }) - } classMap[irClass]?.let { return it } @@ -511,10 +507,10 @@ internal object DataFlowIR { val symbolTableIndex = if (placeToClassTable) module.numberOfClasses++ else -1 val type = if (irClass.isExported()) Type.Public(name.localHash.value, isFinal, isAbstract, null, - module, symbolTableIndex, takeName { name }) + module, symbolTableIndex, irClass, takeName { name }) else Type.Private(privateTypeIndex++, isFinal, isAbstract, null, - module, symbolTableIndex, takeName { name }) + module, symbolTableIndex, irClass, takeName { name }) classMap[irClass] = type @@ -586,14 +582,14 @@ internal object DataFlowIR { if (returnsNothing) attributes = attributes or FunctionAttributes.RETURNS_NOTHING val symbol = when { - it.module != irModule.descriptor || it.isExternal || (it.origin == IrDeclarationOrigin.IR_BUILTINS_STUB) -> { + it.isExternal || (it.origin == IrDeclarationOrigin.IR_BUILTINS_STUB) -> { val escapesAnnotation = it.descriptor.annotations.findAnnotation(FQ_NAME_ESCAPES) val pointsToAnnotation = it.descriptor.annotations.findAnnotation(FQ_NAME_POINTS_TO) @Suppress("UNCHECKED_CAST") val escapesBitMask = (escapesAnnotation?.allValueArguments?.get(escapesWhoDescriptor.name) as? ConstantValue)?.value @Suppress("UNCHECKED_CAST") val pointsToBitMask = (pointsToAnnotation?.allValueArguments?.get(pointsToOnWhomDescriptor.name) as? ConstantValue>)?.value - FunctionSymbol.External(name.localHash.value, attributes, takeName { name }).apply { + FunctionSymbol.External(name.localHash.value, attributes, it, takeName { name }).apply { escapes = escapesBitMask pointsTo = pointsToBitMask?.let { it.map { it.value }.toIntArray() } } @@ -612,9 +608,9 @@ internal object DataFlowIR { && (it.isOverridableOrOverrides || bridgeTarget != null || function.isSpecial || !irClass.isFinal()) val symbolTableIndex = if (placeToFunctionsTable) module.numberOfFunctions++ else -1 if (it.isExported()) - FunctionSymbol.Public(name.localHash.value, module, symbolTableIndex, attributes, bridgeTargetSymbol, takeName { name }) + FunctionSymbol.Public(name.localHash.value, module, symbolTableIndex, attributes, it, bridgeTargetSymbol, takeName { name }) else - FunctionSymbol.Private(privateFunIndex++, module, symbolTableIndex, attributes, bridgeTargetSymbol, takeName { name }) + FunctionSymbol.Private(privateFunIndex++, module, symbolTableIndex, attributes, it, bridgeTargetSymbol, takeName { name }) } } functionMap[it] = symbol @@ -639,7 +635,7 @@ internal object DataFlowIR { assert(irField.parent !is IrClass) { "All local properties initializers should've been lowered" } val attributes = FunctionAttributes.IS_GLOBAL_INITIALIZER or FunctionAttributes.RETURNS_UNIT - val symbol = FunctionSymbol.Private(privateFunIndex++, module, -1, attributes, null, takeName { "${irField.symbolName}_init" }) + val symbol = FunctionSymbol.Private(privateFunIndex++, module, -1, attributes, null, null, takeName { "${irField.symbolName}_init" }) functionMap[irField] = symbol @@ -647,32 +643,5 @@ internal object DataFlowIR { symbol.returnParameter = mapTypeToFunctionParameter(context.irBuiltIns.unitType) return symbol } - - fun getPrivateFunctionsTableForExport() = - functionMap - .asSequence() - .filter { it.key is IrFunction } - .filter { it.value.let { it is DataFlowIR.FunctionSymbol.Declared && it.symbolTableIndex >= 0 } } - .sortedBy { (it.value as DataFlowIR.FunctionSymbol.Declared).symbolTableIndex } - .apply { - forEachIndexed { index, entry -> - assert((entry.value as DataFlowIR.FunctionSymbol.Declared).symbolTableIndex == index) { "Inconsistent function table" } - } - } - .map { (it.key as IrFunction) to (it.value as DataFlowIR.FunctionSymbol.Declared) } - .toList() - - fun getPrivateClassesTableForExport() = - classMap - .asSequence() - .filter { it.value.let { it is DataFlowIR.Type.Declared && it.symbolTableIndex >= 0 } } - .sortedBy { (it.value as DataFlowIR.Type.Declared).symbolTableIndex } - .apply { - forEachIndexed { index, entry -> - assert((entry.value as DataFlowIR.Type.Declared).symbolTableIndex == index) { "Inconsistent class table" } - } - } - .map { it.key to (it.value as DataFlowIR.Type.Declared) } - .toList() } } \ No newline at end of file diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/Devirtualization.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/Devirtualization.kt index 88eb2625a8f..4fd7b4491b8 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/Devirtualization.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/Devirtualization.kt @@ -15,16 +15,11 @@ import org.jetbrains.kotlin.backend.common.lower.irBlock import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.getInlinedClass import org.jetbrains.kotlin.backend.konan.descriptors.isInterface -import org.jetbrains.kotlin.backend.konan.ir.IrPrivateClassReferenceImpl -import org.jetbrains.kotlin.backend.konan.ir.IrPrivateFunctionCall -import org.jetbrains.kotlin.backend.konan.ir.IrPrivateFunctionCallImpl -import org.jetbrains.kotlin.backend.konan.ir.getErasedTypeClass +import org.jetbrains.kotlin.backend.konan.ir.isSuspend import org.jetbrains.kotlin.descriptors.ClassDescriptor import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.builders.* -import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin -import org.jetbrains.kotlin.ir.declarations.IrModuleFragment -import org.jetbrains.kotlin.ir.declarations.IrVariable +import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.declarations.impl.IrVariableImpl import org.jetbrains.kotlin.ir.descriptors.IrTemporaryVariableDescriptorImpl import org.jetbrains.kotlin.ir.expressions.* @@ -1016,7 +1011,7 @@ internal object Devirtualization { .filter { it.key.irCallSite != null } .associate { it.key.irCallSite!! to it.value } devirtualize(irModule, context, externalModulesDFG, devirtualizedCallSites) - removeRedundantCoercions(irModule, context, moduleDFG, externalModulesDFG) + removeRedundantCoercions(irModule, context) return AnalysisResult(devirtualizationAnalysisResult) } @@ -1034,12 +1029,9 @@ internal object Devirtualization { private val specialNames = listOf("", "") // TODO: do it more reliably. - private fun IrExpression.isBoxOrUnboxCall() = - (this is IrCall && symbol.owner.name.asString().let { specialNames.contains(it) }) - || (this is IrPrivateFunctionCall && specialNames.any { dfgSymbol.name?.contains(it) == true }) + private fun IrExpression.isBoxOrUnboxCall() = (this is IrCall && symbol.owner.name.asString().let { specialNames.contains(it) }) - private fun devirtualize(irModule: IrModuleFragment, context: Context, - externalModulesDFG: ExternalModulesDFG, + private fun devirtualize(irModule: IrModuleFragment, context: Context, externalModulesDFG: ExternalModulesDFG, devirtualizedCallSites: Map) { val symbols = context.ir.symbols val nativePtrEqualityOperatorSymbol = symbols.areEqualByValue[PrimitiveBinaryType.POINTER]!! @@ -1067,17 +1059,7 @@ internal object Devirtualization { fun IrBuilderWithScope.irCoerce(value: IrExpression, coercion: DataFlowIR.FunctionSymbol.Declared?) = if (coercion == null) value - else IrPrivateFunctionCallImpl( - startOffset = startOffset, - endOffset = endOffset, - type = value.type, // TODO: What type is actually must be here? - valueArgumentsCount = 1, - virtualCallee = null, - dfgSymbol = coercion, - moduleDescriptor = coercion.module.descriptor, - totalFunctions = coercion.module.numberOfFunctions, - functionIndex = coercion.symbolTableIndex - ).apply { + else irCall(coercion.irFunction!!).apply { putValueArgument(0, value) } @@ -1124,45 +1106,80 @@ internal object Devirtualization { } if (actualType.boxFunction == null) return targetType.unboxFunction!!.resolved() as DataFlowIR.FunctionSymbol.Declared - return actualType.boxFunction!!.resolved() as DataFlowIR.FunctionSymbol.Declared + return actualType.boxFunction.resolved() as DataFlowIR.FunctionSymbol.Declared } - fun irDevirtualizedCall(callee: IrCall, actualType: IrType, devirtualizedCallee: DataFlowIR.FunctionSymbol.Declared) = - IrPrivateFunctionCallImpl( - startOffset = callee.startOffset, - endOffset = callee.endOffset, - type = actualType, - valueArgumentsCount = devirtualizedCallee.parameters.size, - virtualCallee = callee, - dfgSymbol = devirtualizedCallee, - moduleDescriptor = devirtualizedCallee.module.descriptor, - totalFunctions = devirtualizedCallee.module.numberOfFunctions, - functionIndex = devirtualizedCallee.symbolTableIndex - ) + fun IrCallImpl.putArgument(index: Int, value: IrExpression) { + var receiversCount = 0 + val callee = symbol.owner + if (callee.dispatchReceiverParameter != null) + ++receiversCount + if (callee.extensionReceiverParameter != null) + ++receiversCount + if (index >= receiversCount) + putValueArgument(index - receiversCount, value) + else { + if (callee.dispatchReceiverParameter != null && index == 0) + dispatchReceiver = value + else + extensionReceiver = value + } + } + + fun IrBuilderWithScope.irDevirtualizedCall(callSite: IrCall, + actualType: IrType, + devirtualizedCallee: DevirtualizedCallee, + arguments: List): IrCall { + val actualCallee = devirtualizedCallee.callee.irFunction!! + val call = IrCallImpl( + callSite.startOffset, callSite.endOffset, + actualType, + actualCallee.symbol, + actualCallee.descriptor, + actualCallee.typeParameters.size, + actualCallee.valueParameters.size, + callSite.origin, + actualCallee.parentAsClass.symbol + ) + if (actualCallee.explicitParameters.size == arguments.size) { + arguments.forEachIndexed { index, argument -> call.putArgument(index, argument) } + return call + } + assert(actualCallee.isSuspend && actualCallee.explicitParameters.size == arguments.size - 1) { + "Incorrect number of arguments: expected [${actualCallee.explicitParameters.size}] but was [${arguments.size - 1}]\n" + + actualCallee.dump() + } + val continuation = arguments.last() + for (index in 0..arguments.size - 2) + call.putArgument(index, arguments[index]) + return irCall(context.ir.symbols.coroutineLaunchpad, actualType).apply { + putValueArgument(0, call) + putValueArgument(1, continuation) + } + } fun IrBuilderWithScope.irDevirtualizedCall(callee: IrCall, actualType: IrType, - actualCallee: DataFlowIR.FunctionSymbol.Declared, - parameters: List) = - actualCallee.bridgeTarget.let { - if (it == null) - irDevirtualizedCall(callee, actualType, actualCallee).apply { - parameters.forEachIndexed { index, value -> - putValueArgument(index, value.getFullValue(this@irDevirtualizedCall)) - } - } - else { - val bridgeTarget = it.resolved() as DataFlowIR.FunctionSymbol.Declared - val callResult = irDevirtualizedCall(callee, actualType, bridgeTarget).apply { - parameters.forEachIndexed { index, value -> + devirtualizedCallee: DevirtualizedCallee, + arguments: List): IrExpression { + val actualCallee = devirtualizedCallee.callee as DataFlowIR.FunctionSymbol.Declared + return actualCallee.bridgeTarget.let { bridgeTarget -> + if (bridgeTarget == null) + irDevirtualizedCall(callee, actualType, + devirtualizedCallee, + arguments.map { it.getFullValue(this@irDevirtualizedCall) }) + else { + val callResult = irDevirtualizedCall(callee, actualType, + DevirtualizedCallee(devirtualizedCallee.receiverType, bridgeTarget), + arguments.mapIndexed { index, value -> val coercion = getTypeConversion(actualCallee.parameters[index], bridgeTarget.parameters[index]) val fullValue = value.getFullValue(this@irDevirtualizedCall) - putValueArgument(index, coercion?.let { irCoerce(fullValue, coercion) } ?: fullValue) - } - } - val returnCoercion = getTypeConversion(bridgeTarget.returnParameter, actualCallee.returnParameter) - irCoerce(callResult, returnCoercion) - } + coercion?.let { irCoerce(fullValue, coercion) } ?: fullValue + }) + val returnCoercion = getTypeConversion(bridgeTarget.returnParameter, actualCallee.returnParameter) + irCoerce(callResult, returnCoercion) } + } + } irModule.transformChildrenVoid(object: IrElementTransformerVoidWithContext() { override fun visitCall(expression: IrCall): IrExpression { @@ -1208,37 +1225,31 @@ internal object Devirtualization { } optimize && possibleCallees.size == 1 -> { // Monomorphic callsite. - val actualCallee = possibleCallees[0].callee as DataFlowIR.FunctionSymbol.Declared irBlock(expression) { val parameters = expression.getArgumentsWithSymbols().mapIndexed { index, arg -> irSplitCoercion(arg.second, "arg$index", arg.first.owner.type) } - +irDevirtualizedCall(expression, type, actualCallee, parameters) + +irDevirtualizedCall(expression, type, possibleCallees[0], parameters) } } else -> irBlock(expression) { - val parameters = expression.getArgumentsWithSymbols().mapIndexed { index, arg -> + val arguments = expression.getArgumentsWithSymbols().mapIndexed { index, arg -> irSplitCoercion(arg.second, "arg$index", arg.first.owner.type) } val typeInfo = irTemporary(irCall(symbols.getObjectTypeInfo).apply { - putValueArgument(0, parameters[0].getFullValue(this@irBlock)) + putValueArgument(0, arguments[0].getFullValue(this@irBlock)) }) val branches = mutableListOf() possibleCallees.mapIndexedTo(branches) { index, devirtualizedCallee -> - val actualCallee = devirtualizedCallee.callee as DataFlowIR.FunctionSymbol.Declared val actualReceiverType = devirtualizedCallee.receiverType as DataFlowIR.Type.Declared - val expectedTypeInfo = IrPrivateClassReferenceImpl( - startOffset = startOffset, - endOffset = endOffset, - type = symbols.nativePtrType, - symbol = dispatchReceiver.type.getErasedTypeClass(), - classType = dispatchReceiver.type, - moduleDescriptor = actualReceiverType.module!!.descriptor, - totalClasses = actualReceiverType.module.numberOfClasses, - classIndex = actualReceiverType.symbolTableIndex, - dfgSymbol = actualReceiverType) + val expectedTypeInfo = IrClassReferenceImpl( + startOffset, endOffset, + symbols.nativePtrType, + actualReceiverType.irClass!!.symbol, + actualReceiverType.irClass.defaultType + ) val condition = if (optimize && index == possibleCallees.size - 1) irTrue() // Don't check last type in optimize mode. @@ -1251,7 +1262,7 @@ internal object Devirtualization { startOffset = startOffset, endOffset = endOffset, condition = condition, - result = irDevirtualizedCall(expression, type, actualCallee, parameters) + result = irDevirtualizedCall(expression, type, devirtualizedCallee, arguments) ) } if (!optimize) { // Add else branch throwing exception for debug purposes. @@ -1285,14 +1296,7 @@ internal object Devirtualization { }) } - private fun removeRedundantCoercions(irModule: IrModuleFragment, context: Context, - moduleDFG: ModuleDFG, externalModulesDFG: ExternalModulesDFG) { - - fun DataFlowIR.FunctionSymbol.resolved(): DataFlowIR.FunctionSymbol { - if (this is DataFlowIR.FunctionSymbol.External) - return externalModulesDFG.publicFunctions[this.hash] ?: this - return this - } + private fun removeRedundantCoercions(irModule: IrModuleFragment, context: Context) { class PossiblyFoldedExpression(val expression: IrExpression, val folded: Boolean) { fun getFullExpression(coercion: IrCall, cast: IrTypeOperatorCall?): IrExpression { @@ -1355,21 +1359,12 @@ internal object Devirtualization { val coercionDeclaringClass = coercion.symbol.owner.parentAsClass if (expression.isBoxOrUnboxCall()) { + expression as IrCall val result = - (expression as? IrCall)?.let { - if (coercionDeclaringClass == it.symbol.owner.parentAsClass) - it.getArguments().single().second - else expression - } ?: (expression as IrPrivateFunctionCall).let { - val argarg = it.getValueArgument(0)!! - val boxFunction = context.getBoxFunction(coercionDeclaringClass) - val unboxFunction = context.getUnboxFunction(coercionDeclaringClass) - val boxFunctionSymbol = moduleDFG.symbolTable.mapFunction(boxFunction).resolved() - val unboxFunctionSymbol = moduleDFG.symbolTable.mapFunction(unboxFunction).resolved() - if (it.dfgSymbol == boxFunctionSymbol || it.dfgSymbol == unboxFunctionSymbol) - argarg - else it - } + if (coercionDeclaringClass == expression.symbol.owner.parentAsClass) + expression.getArguments().single().second + else expression + return PossiblyFoldedExpression(result.transformIfAsked(), result != expression) } return when (expression) { diff --git a/runtime/src/main/kotlin/kotlin/native/internal/Coroutines.kt b/runtime/src/main/kotlin/kotlin/native/internal/Coroutines.kt index e15e9196bc4..2101e5bc12f 100644 --- a/runtime/src/main/kotlin/kotlin/native/internal/Coroutines.kt +++ b/runtime/src/main/kotlin/kotlin/native/internal/Coroutines.kt @@ -25,3 +25,6 @@ internal inline suspend fun getCoroutineContext(): CoroutineContext = @TypedIntrinsic(IntrinsicType.RETURN_IF_SUSPEND) @PublishedApi internal external suspend fun returnIfSuspended(@Suppress("UNUSED_PARAMETER") argument: Any?): T + +@TypedIntrinsic(IntrinsicType.COROUTINE_LAUNCHPAD) +internal external fun coroutineLaunchpad(suspendFunctionCall: Any?, continuation: Continuation<*>): Any? \ No newline at end of file diff --git a/runtime/src/main/kotlin/kotlin/native/internal/IntrinsicType.kt b/runtime/src/main/kotlin/kotlin/native/internal/IntrinsicType.kt index 27ac4342c6d..20fc3589359 100644 --- a/runtime/src/main/kotlin/kotlin/native/internal/IntrinsicType.kt +++ b/runtime/src/main/kotlin/kotlin/native/internal/IntrinsicType.kt @@ -57,6 +57,7 @@ class IntrinsicType { const val GET_CONTINUATION = "GET_CONTINUATION" const val RETURN_IF_SUSPEND = "RETURN_IF_SUSPEND" + const val COROUTINE_LAUNCHPAD = "COROUTINE_LAUNCHPAD" // Interop const val INTEROP_READ_PRIMITIVE = "INTEROP_READ_PRIMITIVE"