[K/N] Remove isSuspend hacks from codeGenerator and global optimizations

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -180,7 +180,8 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
         configuration.get(BinaryOptions.appStateTracking) ?: AppStateTracking.DISABLED
     }
 
-    val mimallocUseDefaultOptions by lazy {
+
+    val mimallocUseDefaultOptions: Boolean by lazy {
         configuration.get(BinaryOptions.mimallocUseDefaultOptions) ?: false
     }
 

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/ClassLayoutBuilder.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.backend.konan.descriptors
 
 import llvm.LLVMStoreSizeOfType
+import org.jetbrains.kotlin.backend.common.lower.coroutines.getOrCreateFunctionWithContinuationStub
 import org.jetbrains.kotlin.backend.konan.*
 import org.jetbrains.kotlin.backend.konan.ir.*
 import org.jetbrains.kotlin.backend.konan.llvm.computeFunctionName
@@ -507,8 +508,10 @@ internal class ClassLayoutBuilder(val irClass: IrClass, val context: Context) {
     /**
      * Normally, function should be already replaced. But if the function come from LazyIr, it can be not replaced.
      */
-    fun IrSimpleFunction.getLoweredVersion() = context.mapping.suspendFunctionsToFunctionWithContinuations[this] ?: this
+    fun IrSimpleFunction.getLoweredVersion() = when {
-
+        isSuspend -> this.getOrCreateFunctionWithContinuationStub(context)
+        else -> this
+    }
     private val overridableOrOverridingMethods: List<IrSimpleFunction>
         get() = irClass.simpleFunctions()
                 .map {it.getLoweredVersion() }

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IntrinsicGenerator.kt

@@ -105,8 +105,6 @@ internal interface IntrinsicGeneratorEnvironment {
 
     val functionGenerationContext: FunctionGenerationContext
 
-    val continuation: LLVMValueRef
-
     val exceptionHandler: ExceptionHandler
 
     val stackLocalsManager: StackLocalsManager
@@ -156,10 +154,10 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
     /**
      * Some intrinsics have to be processed before evaluation of their arguments.
      * So this method looks at [callSite] and if it is call to "special" intrinsic
-     * processes it. Otherwise it returns null.
+     * processes it. Otherwise, it returns null.
      */
+    @Suppress("UNUSED_PARAMETER")
     fun tryEvaluateSpecialCall(callSite: IrFunctionAccessExpression, resultSlot: LLVMValueRef?): LLVMValueRef? {
-        resultSlot.let{}
         val function = callSite.symbol.owner
         if (!function.isTypedIntrinsic) {
             return null
@@ -241,9 +239,9 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
                 IntrinsicType.INTEROP_NATIVE_PTR_PLUS_LONG -> emitNativePtrPlusLong(args)
                 IntrinsicType.INTEROP_GET_NATIVE_NULL_PTR -> emitGetNativeNullPtr()
                 IntrinsicType.IDENTITY -> emitIdentity(args)
-                IntrinsicType.GET_CONTINUATION -> emitGetContinuation()
                 IntrinsicType.INTEROP_MEMORY_COPY -> emitMemoryCopy(callSite, args)
                 IntrinsicType.IS_EXPERIMENTAL_MM -> emitIsExperimentalMM()
+                IntrinsicType.GET_CONTINUATION,
                 IntrinsicType.RETURN_IF_SUSPENDED,
                 IntrinsicType.INTEROP_BITS_TO_FLOAT,
                 IntrinsicType.INTEROP_BITS_TO_DOUBLE,
@@ -286,8 +284,6 @@ internal class IntrinsicGenerator(private val environment: IntrinsicGeneratorEnv
     private fun reportNonLoweredIntrinsic(intrinsicType: ConstantConstructorIntrinsicType): Nothing =
             context.reportCompilationError("Constant constructor intrinsic of type $intrinsicType should be handled by previous lowering phase")
 
-    private fun FunctionGenerationContext.emitGetContinuation(): LLVMValueRef =
-            environment.continuation
 
     private fun FunctionGenerationContext.emitIdentity(args: List<LLVMValueRef>): LLVMValueRef =
             args.single()

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.backend.konan.llvm
 
 import kotlinx.cinterop.*
 import llvm.*
+import org.jetbrains.kotlin.backend.common.lower.coroutines.getOrCreateFunctionWithContinuationStub
 import org.jetbrains.kotlin.backend.common.lower.inline.InlinerExpressionLocationHint
 import org.jetbrains.kotlin.backend.konan.*
 import org.jetbrains.kotlin.backend.konan.cgen.CBridgeOrigin
@@ -224,9 +225,6 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
         override fun calculateLifetime(element: IrElement): Lifetime =
                 resultLifetime(element)
 
-        override val continuation: LLVMValueRef
-            get() = getContinuation()
-
         override val exceptionHandler: ExceptionHandler
             get() = currentCodeContext.exceptionHandler
 
@@ -2238,20 +2236,21 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
     private fun IrFunction.scope(startLine:Int): DIScopeOpaqueRef? {
         if (!context.shouldContainLocationDebugInfo())
             return null
-        val functionLlvmValue =
+
-                // TODO: May be tie up inline lambdas to their outer function?
+        val functionLlvmValue = when {
-                if (codegen.isExternal(this) && !KonanBinaryInterface.isExported(this))
+            isReifiedInline -> null
-                    null
+            // TODO: May be tie up inline lambdas to their outer function?
-                else
+            codegen.isExternal(this) && !KonanBinaryInterface.isExported(this) -> null
-                    codegen.llvmFunctionOrNull(this)?.llvmValue
+            this is IrSimpleFunction && isSuspend -> this.getOrCreateFunctionWithContinuationStub(context).let { codegen.llvmFunctionOrNull(it)?.llvmValue }
-        return if (!isReifiedInline && functionLlvmValue != null) {
+            else -> codegen.llvmFunctionOrNull(this)?.llvmValue
+        }
+        return if (functionLlvmValue != null) {
             debugInfo.subprograms.getOrPut(functionLlvmValue) {
                 memScoped {
                     val subroutineType = subroutineType(context, codegen.llvmTargetData)
-                    val llvmFunction = codegen.llvmFunction(this@scope).llvmValue
+                    diFunctionScope(name.asString(), functionLlvmValue.name!!, startLine, subroutineType).also {
-                    diFunctionScope(name.asString(), llvmFunction.name!!, startLine, subroutineType).also {
                         if (!this@scope.isInline)
-                            DIFunctionAddSubprogram(llvmFunction, it)
+                            DIFunctionAddSubprogram(functionLlvmValue, it)
                     }
                 }
             } as DIScopeOpaqueRef
@@ -2291,19 +2290,10 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
 
     //-------------------------------------------------------------------------//
 
-    private fun getContinuation(): LLVMValueRef {
-        val caller = functionGenerationContext.irFunction!!
-        return if (caller.isSuspend)
-            codegen.param(caller, caller.allParametersCount)    // The last argument.
-        else {
-            // Suspend call from non-suspend function - must be [invokeSuspend].
-            assert ((caller as IrSimpleFunction).overrides(context.ir.symbols.invokeSuspendFunction.owner),
-                    { "Expected 'BaseContinuationImpl.invokeSuspend' but was '$caller'" })
-            currentCodeContext.genGetValue(caller.dispatchReceiverParameter!!, null)
-        }
-    }
 
-    private fun IrFunction.returnsUnit() = returnType.isUnit() && !isSuspend
+    private fun IrFunction.returnsUnit() = returnType.isUnit().also {
+        require(!isSuspend) { "Suspend functions should be lowered out at this point"}
+    }
 
     /**
      * Evaluates all arguments of [expression] that are explicitly represented in the IR.
@@ -2421,17 +2411,15 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
     private fun evaluateFunctionCall(callee: IrCall, args: List<LLVMValueRef>,
                                      resultLifetime: Lifetime, resultSlot: LLVMValueRef?): LLVMValueRef {
         val function = callee.symbol.owner
+        require(!function.isSuspend) { "Suspend functions should be lowered out at this point"}
 
-        val argsWithContinuationIfNeeded = if (function.isSuspend)
-                                               args + getContinuation()
-                                           else args
         return when {
             function.isTypedIntrinsic -> intrinsicGenerator.evaluateCall(callee, args, resultSlot)
-            function.isBuiltInOperator -> evaluateOperatorCall(callee, argsWithContinuationIfNeeded)
+            function.isBuiltInOperator -> evaluateOperatorCall(callee, args)
             function.origin == DECLARATION_ORIGIN_FILE_GLOBAL_INITIALIZER -> evaluateFileGlobalInitializerCall(function)
             function.origin == DECLARATION_ORIGIN_FILE_THREAD_LOCAL_INITIALIZER -> evaluateFileThreadLocalInitializerCall(function)
             function.origin == DECLARATION_ORIGIN_FILE_STANDALONE_THREAD_LOCAL_INITIALIZER -> evaluateFileStandaloneThreadLocalInitializerCall(function)
-            else -> evaluateSimpleFunctionCall(function, argsWithContinuationIfNeeded, resultLifetime, callee.superQualifierSymbol?.owner, resultSlot)
+            else -> evaluateSimpleFunctionCall(function, args, resultLifetime, callee.superQualifierSymbol?.owner, resultSlot)
         }
     }
 
@@ -2681,11 +2669,9 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
 
         val result = call(llvmCallable, args, resultLifetime, exceptionHandler, resultSlot)
 
-        when {
+        when  {
-            !function.isSuspend && function.returnType.isNothing() ->
+            function.returnType.isNothing() -> functionGenerationContext.unreachable()
-                functionGenerationContext.unreachable()
+            needsNativeThreadState -> functionGenerationContext.switchThreadState(ThreadState.Runnable)
-            needsNativeThreadState ->
-                functionGenerationContext.switchThreadState(ThreadState.Runnable)
         }
 
         if (llvmCallable.returnType == voidType) {

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmAttributes.kt

@@ -7,11 +7,6 @@ package org.jetbrains.kotlin.backend.konan.llvm
 
 import llvm.*
 import org.jetbrains.kotlin.backend.konan.Context
-import org.jetbrains.kotlin.ir.declarations.IrConstructor
-import org.jetbrains.kotlin.ir.declarations.IrFunction
-import org.jetbrains.kotlin.ir.types.isNothing
-import org.jetbrains.kotlin.ir.util.isSuspend
-import org.jetbrains.kotlin.ir.util.isThrowable
 import org.jetbrains.kotlin.konan.target.Architecture
 import org.jetbrains.kotlin.konan.target.Family
 import org.jetbrains.kotlin.konan.target.KonanTarget

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmFunctionPrototype.kt

@@ -193,8 +193,8 @@ private fun mustNotInline(context: Context, irFunction: IrFunction): Boolean {
 
 private fun inferFunctionAttributes(contextUtils: ContextUtils, irFunction: IrFunction): List<LlvmFunctionAttribute> =
         mutableListOf<LlvmFunctionAttribute>().apply {
-            // suspend function can return value in case of COROUTINE_SUSPENDED.
+            if (irFunction.returnType.isNothing()) {
-            if (irFunction.returnType.isNothing() && !irFunction.isSuspend) {
+                require(!irFunction.isSuspend) { "Suspend functions should be lowered out at this point"}
                 add(LlvmFunctionAttribute.NoReturn)
             }
             if (mustNotInline(contextUtils.context, irFunction)) {

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmParamType.kt

@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.ir.declarations.IrConstructor
 import org.jetbrains.kotlin.ir.declarations.IrFunction
 import org.jetbrains.kotlin.ir.util.allParameters
 import org.jetbrains.kotlin.ir.util.isSuspend
+import org.jetbrains.kotlin.ir.util.render
 
 /**
  * LLVM function's parameter type with its attributes.
@@ -26,8 +27,7 @@ internal fun ContextUtils.getLlvmFunctionParameterTypes(function: IrFunction): L
     val paramTypes = ArrayList(function.allParameters.map {
         LlvmParamType(getLLVMType(it.type), argumentAbiInfo.defaultParameterAttributesForIrType(it.type))
     })
-    if (function.isSuspend)
+    require(!function.isSuspend) { "Suspend functions should be lowered out at this point"}
-        paramTypes.add(LlvmParamType(kObjHeaderPtr))                       // Suspend functions have implicit parameter of type Continuation<>.
     if (isObjectType(returnType))
         paramTypes.add(LlvmParamType(kObjHeaderPtrPtr))
 
@@ -37,7 +37,7 @@ internal fun ContextUtils.getLlvmFunctionParameterTypes(function: IrFunction): L
 internal fun ContextUtils.getLlvmFunctionReturnType(function: IrFunction): LlvmRetType {
     val returnType = when {
         function is IrConstructor -> LlvmParamType(voidType)
-        function.isSuspend -> LlvmParamType(kObjHeaderPtr)                // Suspend functions return Any?.
+        function.isSuspend ->  error("Suspend functions should be lowered out at this point, but ${function.render()} is still here")
         else -> LlvmParamType(
                 getLLVMReturnType(function.returnType),
                 argumentAbiInfo.defaultParameterAttributesForIrType(function.returnType)

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/Autoboxing.kt

@@ -76,11 +76,7 @@ private class AutoboxingTransformer(val context: Context) : AbstractValueUsageTr
     }
 
     override fun IrExpression.useAsReturnValue(returnTarget: IrReturnTargetSymbol): IrExpression = when (returnTarget) {
-        is IrSimpleFunctionSymbol -> if (returnTarget.owner.isSuspend && returnTarget == currentFunction?.symbol) {
+        is IrSimpleFunctionSymbol -> this.useAs(returnTarget.owner.returnType)
-            this.useAs(irBuiltIns.anyNType)
-        } else {
-            this.useAs(returnTarget.owner.returnType)
-        }
         is IrConstructorSymbol -> this.useAs(irBuiltIns.unitType)
         is IrReturnableBlockSymbol -> this.useAs(returnTarget.owner.type)
         else -> error(returnTarget)
@@ -89,8 +85,7 @@ private class AutoboxingTransformer(val context: Context) : AbstractValueUsageTr
     override fun IrExpression.useAs(type: IrType): IrExpression {
         val actualType = when (this) {
             is IrCall -> {
-                if (this.symbol.owner.isSuspend) irBuiltIns.anyNType
+                if (this.symbol == symbols.reinterpret) this.getTypeArgument(1)!!
-                else if (this.symbol == symbols.reinterpret) this.getTypeArgument(1)!!
                 else this.callTarget.returnType
             }
             is IrGetField -> this.symbol.owner.type

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/FunctionReferenceLowering.kt

@@ -481,9 +481,6 @@ internal class FunctionReferenceLowering(val context: Context) : FileLoweringPas
                                                 if (parameter == referencedFunction.extensionReceiverParameter
                                                         && extensionReceiverParameter != null)
                                                     irGet(extensionReceiverParameter!!)
-                                                else if (function.isSuspend && unboundIndex == valueParameters.size)
-                                                // For suspend functions the last argument is continuation and it is implicit.
-                                                    irCall(getContinuationSymbol.owner, listOf(returnType))
                                                 else
                                                     irGet(valueParameters[unboundIndex++])
                                             }

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGBuilder.kt

@@ -445,8 +445,6 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag
             symbols.baseContinuationImpl.owner.declarations
                     .filterIsInstance<IrSimpleFunction>().single { it.name.asString() == "invokeSuspend" }.symbol
 
-    private val getContinuationSymbol = symbols.getContinuation
-
     private val arrayGetSymbols = symbols.arrayGet.values
     private val arraySetSymbols = symbols.arraySet.values
     private val createUninitializedInstanceSymbol = symbols.createUninitializedInstance
@@ -485,27 +483,12 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag
                 { Scoped(DataFlowIR.Node.Parameter(it.index), rootScope) }
         )
 
-        private val continuationParameter = when {
-            declaration !is IrSimpleFunction -> null
-
-            declaration.isSuspend -> Scoped(DataFlowIR.Node.Parameter(allParameters.size), rootScope)
-
-            declaration.overrides(invokeSuspendFunctionSymbol.owner) ->           // <this> is a ContinuationImpl inheritor.
-                templateParameters[declaration.dispatchReceiverParameter!!]       // It is its own continuation.
-
-            else -> null
-        }
-
-        private fun getContinuation() = continuationParameter
-                ?: error("Function ${declaration.descriptor} has no continuation parameter")
 
         private val nodes = mutableMapOf<IrExpression, Scoped<DataFlowIR.Node>>()
         private val variables = mutableMapOf<IrValueDeclaration, Scoped<DataFlowIR.Node.Variable>>()
         private val expressionsScopes = mutableMapOf<IrExpression, DataFlowIR.Node.Scope>()
 
         fun build(): DataFlowIR.Function {
-            val isSuspend = declaration is IrSimpleFunction && declaration.isSuspend
-
             val scopes = mutableMapOf<IrLoop, DataFlowIR.Node.Scope>()
             fun transformLoop(loop: IrLoop, parentLoop: IrLoop?): DataFlowIR.Node.Scope {
                 scopes[loop]?.let { return it }
@@ -566,8 +549,6 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag
             rootScope.nodes += templateParameters.values.map { it.value }
             rootScope.nodes += returnsNode
             rootScope.nodes += throwsNode
-            if (isSuspend)
-                rootScope.nodes += continuationParameter!!.value
 
             return DataFlowIR.Function(
                     symbol = symbolTable.mapFunction(declaration),
@@ -610,7 +591,7 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag
         private fun mapReturnType(actualType: IrType, returnType: IrType) = mapWrappedType(actualType, returnType)
 
 
-        private fun getNode(expression: IrExpression, continuationOverride: DataFlowIR.Node? = null): Scoped<DataFlowIR.Node> {
+        private fun getNode(expression: IrExpression): Scoped<DataFlowIR.Node> {
             if (expression is IrGetValue) {
                 val valueDeclaration = expression.symbol.owner
                 if (valueDeclaration is IrValueParameter)
@@ -712,8 +693,6 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag
                             }
 
                             is IrCall -> when (value.symbol) {
-                                getContinuationSymbol -> continuationOverride ?: getContinuation().value
-
                                 in arrayGetSymbols -> {
                                     val actualCallee = value.actualCallee
 
@@ -760,12 +739,6 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag
                                     val callee = value.symbol.owner
                                     val arguments = value.getArguments()
                                             .map { expressionToEdge(it.second) }
-                                            .let {
-                                                if (callee.isSuspend)
-                                                    it + DataFlowIR.Edge(continuationOverride ?: getContinuation().value, null)
-                                                else
-                                                    it
-                                            }
 
                                     if (value.isVirtualCall) {
                                         val owner = callee.parentAsClass

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DataFlowIR.kt

@@ -595,8 +595,8 @@ internal object DataFlowIR {
             }
             val name = "kfun:$containingDeclarationPart${it.computeFunctionName()}"
 
-            val returnsUnit = it is IrConstructor || (!it.isSuspend && it.returnType.isUnit())
+            val returnsUnit = it is IrConstructor || it.returnType.isUnit()
-            val returnsNothing = !it.isSuspend && it.returnType.isNothing()
+            val returnsNothing = it.returnType.isNothing()
             var attributes = 0
             if (returnsUnit)
                 attributes = attributes or FunctionAttributes.RETURNS_UNIT
@@ -648,14 +648,10 @@ internal object DataFlowIR {
             }
             functionMap[it] = symbol
 
-            symbol.parameters =
+            symbol.parameters = function.allParameters.map { it.type }
-                    (function.allParameters.map { it.type } + (if (function.isSuspend) listOf(continuationType) else emptyList()))
                             .map { mapTypeToFunctionParameter(it) }
                             .toTypedArray()
-            symbol.returnParameter = mapTypeToFunctionParameter(if (function.isSuspend)
+            symbol.returnParameter = mapTypeToFunctionParameter(function.returnType)
-                                                               context.irBuiltIns.anyType
-                                                           else
-                                                               function.returnType)
 
             return symbol
         }

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DevirtualizationAnalysis.kt

@@ -1446,9 +1446,7 @@ internal object DevirtualizationAnalysis {
                 val startOffset = expression.startOffset
                 val endOffset = expression.endOffset
                 val function = expression.symbol.owner
-                val type = if (callee.isSuspend)
+                val type = function.returnType
-                               context.irBuiltIns.anyNType
-                           else function.returnType
                 val irBuilder = context.createIrBuilder(currentScope!!.scope.scopeOwnerSymbol, startOffset, endOffset)
                 irBuilder.run {
                     val dispatchReceiver = expression.dispatchReceiver!!