Support release coroutines (#1909)

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt

@@ -198,7 +198,7 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
     }
 
     override fun createArguments(): K2NativeCompilerArguments {
-        return K2NativeCompilerArguments().apply { coroutinesState = "enable" }
+        return K2NativeCompilerArguments()
     }
 
     override fun executableScriptFileName() = "kotlinc-native"

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt

@@ -18,8 +18,6 @@ package org.jetbrains.kotlin.cli.bc
 
 import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
 import org.jetbrains.kotlin.cli.common.arguments.Argument
-import org.jetbrains.kotlin.cli.common.messages.MessageCollector
-import org.jetbrains.kotlin.config.LanguageFeature
 
 class K2NativeCompilerArguments : CommonCompilerArguments() {
     // First go the options interesting to the general public.
@@ -149,10 +147,5 @@ class K2NativeCompilerArguments : CommonCompilerArguments() {
             description = "Paths to friend modules"
     )
     var friendModules: String? = null
-
-    override fun configureLanguageFeatures(collector: MessageCollector) = super.configureLanguageFeatures(collector).also {
-        it[LanguageFeature.InlineClasses] = LanguageFeature.State.ENABLED // TODO: remove after updating to 1.3.
-        it[LanguageFeature.ReleaseCoroutines] = LanguageFeature.State.DISABLED
-    }
 }
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt

@@ -327,9 +327,6 @@ internal class KonanSymbols(context: Context, val symbolTable: SymbolTable, val
     val getContinuation = symbolTable.referenceSimpleFunction(
             context.getInternalFunctions("getContinuation").single())
 
-    val konanIntercepted = symbolTable.referenceSimpleFunction(
-            context.getInternalFunctions("intercepted").single())
-
     val konanSuspendCoroutineUninterceptedOrReturn = symbolTable.referenceSimpleFunction(
             context.getInternalFunctions("suspendCoroutineUninterceptedOrReturn").single())
 
@@ -347,7 +344,22 @@ internal class KonanSymbols(context: Context, val symbolTable: SymbolTable, val
             .single()
             .getter!!
 
-    override val coroutineImpl = symbolTable.referenceClass(context.getInternalClass("CoroutineImpl"))
+    override val coroutineImpl get() = TODO()
+
+    val baseContinuationImpl = symbolTable.referenceClass(
+            builtIns.builtInsModule.findClassAcrossModuleDependencies(
+                    ClassId.topLevel(FqName("kotlin.coroutines.native.internal.BaseContinuationImpl")))!!
+    )
+
+    val restrictedContinuationImpl = symbolTable.referenceClass(
+            builtIns.builtInsModule.findClassAcrossModuleDependencies(
+                    ClassId.topLevel(FqName("kotlin.coroutines.native.internal.RestrictedContinuationImpl")))!!
+    )
+
+    val continuationImpl = symbolTable.referenceClass(
+            builtIns.builtInsModule.findClassAcrossModuleDependencies(
+                    ClassId.topLevel(FqName("kotlin.coroutines.native.internal.ContinuationImpl")))!!
+    )
 
     override val coroutineSuspendedGetter = symbolTable.referenceSimpleFunction(
             coroutinesIntrinsicsPackage
@@ -355,6 +367,11 @@ internal class KonanSymbols(context: Context, val symbolTable: SymbolTable, val
                     .filterNot { it.isExpect }.single().getter!!
     )
 
+    val successOrFailure = symbolTable.referenceClass(
+            builtIns.builtInsModule.findClassAcrossModuleDependencies(
+                    ClassId.topLevel(FqName("kotlin.SuccessOrFailure")))!!
+    )
+
     val refClass = symbolTable.referenceClass(context.getInternalClass("Ref"))
 
     val kLocalDelegatedPropertyImpl = symbolTable.referenceClass(context.reflectionTypes.kLocalDelegatedPropertyImpl)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -1920,19 +1920,18 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
 
     //-------------------------------------------------------------------------//
 
-    private val coroutineImplDescriptor = context.ir.symbols.coroutineImpl.owner
+    private val invokeSuspendFunction = context.ir.symbols.baseContinuationImpl.owner.declarations
-    private val doResumeFunctionDescriptor = coroutineImplDescriptor.declarations
+            .filterIsInstance<IrSimpleFunction>().single { it.name.asString() == "invokeSuspend" }
-            .filterIsInstance<IrSimpleFunction>().single { it.name.asString() == "doResume" }
 
     private fun getContinuation(): LLVMValueRef {
         val caller = functionGenerationContext.functionDescriptor!!
         return if (caller.isSuspend)
             codegen.param(caller, caller.allParameters.size)    // The last argument.
         else {
-            // Suspend call from non-suspend function - must be [CoroutineImpl].
+            // Suspend call from non-suspend function - must be [BaseContinuationImpl].
-            assert (doResumeFunctionDescriptor.symbol in (caller as IrSimpleFunction).overriddenSymbols,
+            assert ((caller as IrSimpleFunction).overrides(invokeSuspendFunction),
-                    { "Expected 'CoroutineImpl.doResume' but was '$caller'" })
+                    { "Expected 'BaseContinuationImpl.invokeSuspend' but was '$caller'" })
-            currentCodeContext.genGetValue(caller.dispatchReceiverParameter!!)   // Coroutine itself is a continuation.
+            currentCodeContext.genGetValue(caller.dispatchReceiverParameter!!)
         }
     }
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/FunctionInlining.kt

@@ -99,7 +99,7 @@ internal class FunctionInlining(val context: Context): IrElementTransformerVoidW
             descriptor.resolveFakeOverride().original.let {
                 when  {
                     it.isBuiltInIntercepted(context.config.configuration.languageVersionSettings) ->
-                        getFunctionDeclaration(context.ir.symbols.konanIntercepted.descriptor)
+                        error("Continuation.intercepted is not available with release coroutines")
 
                     it.isBuiltInSuspendCoroutineUninterceptedOrReturn(context.config.configuration.languageVersionSettings) ->
                         getFunctionDeclaration(context.ir.symbols.konanSuspendCoroutineUninterceptedOrReturn.descriptor)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/SuspendFunctionsLowering.kt

@@ -18,11 +18,11 @@ package org.jetbrains.kotlin.backend.konan.lower
 
 import org.jetbrains.kotlin.backend.common.*
 import org.jetbrains.kotlin.backend.common.descriptors.getFunction
-import org.jetbrains.kotlin.backend.common.ir.createOverriddenDescriptor
 import org.jetbrains.kotlin.backend.common.lower.*
 import org.jetbrains.kotlin.backend.konan.Context
 import org.jetbrains.kotlin.backend.konan.descriptors.synthesizedName
 import org.jetbrains.kotlin.backend.konan.irasdescriptors.typeWith
+import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.descriptors.annotations.Annotations
 import org.jetbrains.kotlin.descriptors.impl.ClassConstructorDescriptorImpl
@@ -242,7 +242,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
             val irBuilder = context.createIrBuilder(irFunction.symbol, irFunction.startOffset, irFunction.endOffset)
             irFunction.body = irBuilder.irBlockBody(irFunction) {
                 +irReturn(
-                        irCall(coroutine.doResumeFunction.symbol).apply {
+                        irCall(coroutine.invokeSuspendFunction.symbol).apply {
                             dispatchReceiver = irCall(coroutine.coroutineConstructor.symbol).apply {
                                 val functionParameters = irFunction.explicitParameters
                                 functionParameters.forEachIndexed { index, argument ->
@@ -251,8 +251,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                                 putValueArgument(functionParameters.size,
                                         irCall(getContinuation, listOf(irFunction.returnType)))
                             }
-                            putValueArgument(0, irGetObject(symbols.unit)) // value
+                            putValueArgument(0, irSuccess(irGetObject(symbols.unit)))
-                            putValueArgument(1, irNull()) // exception
                         })
             }
         }
@@ -262,7 +261,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
 
     private class BuiltCoroutine(val coroutineClass: IrClass,
                                  val coroutineConstructor: IrConstructor,
-                                 val doResumeFunction: IrFunction)
+                                 val invokeSuspendFunction: IrFunction)
 
     private var coroutineId = 0
 
@@ -280,26 +279,28 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
 
         private var tempIndex = 0
         private var suspensionPointIdIndex = 0
+        private lateinit var labelField: IrField
         private lateinit var suspendResult: IrVariable
-        private lateinit var dataArgument: IrValueParameter
+        private lateinit var resultArgument: IrValueParameter
-        private lateinit var exceptionArgument: IrValueParameter
         private lateinit var coroutineClassDescriptor: ClassDescriptorImpl
         private lateinit var coroutineClass: IrClassImpl
         private lateinit var coroutineClassThis: IrValueParameter
         private lateinit var argumentToPropertiesMap: Map<ParameterDescriptor, IrField>
 
-        private val coroutineImplSymbol = symbols.coroutineImpl
+        private val baseClass =
-        private val coroutineImplConstructorSymbol = coroutineImplSymbol.constructors.single()
+                (if (irFunction.isRestrictedSuspendFunction(context.config.configuration.languageVersionSettings)) {
-        private val coroutineImplClassDescriptor = coroutineImplSymbol.descriptor
+                    symbols.restrictedContinuationImpl
-        private val create1Function = coroutineImplSymbol.owner.simpleFunctions()
+                } else {
+                    symbols.continuationImpl
+                }).owner
+
+        private val baseClassConstructor = baseClass.constructors.single { it.valueParameters.size == 1 }
+        private val create1Function = baseClass.simpleFunctions()
                 .single { it.name.asString() == "create" && it.valueParameters.size == 1 }
         private val create1CompletionParameter = create1Function.valueParameters[0]
 
-        private val coroutineImplLabelGetterSymbol = coroutineImplSymbol.getPropertyGetter("label")!!
-        private val coroutineImplLabelSetterSymbol = coroutineImplSymbol.getPropertySetter("label")!!
-
         fun build(): BuiltCoroutine {
-            val superTypes = mutableListOf<IrType>(coroutineImplSymbol.owner.defaultType)
+            val superTypes = mutableListOf<IrType>(baseClass.defaultType)
             var suspendFunctionClass: IrClass? = null
             var functionClass: IrClass? = null
             var suspendFunctionClassTypeArguments: List<IrType>? = null
@@ -340,16 +341,27 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
             coroutineClass.createParameterDeclarations()
             coroutineClassThis = coroutineClass.thisReceiver!!
 
+            labelField = createField(
+                    irFunction.startOffset,
+                    irFunction.endOffset,
+                    symbols.nativePtrType,
+                    Name.identifier("label"),
+                    true,
+                    IrDeclarationOrigin.DEFINED,
+                    coroutineClass.descriptor
+            )
+            coroutineClass.addChild(labelField)
+
 
             val overriddenMap = mutableMapOf<CallableMemberDescriptor, CallableMemberDescriptor>()
             val coroutineConstructorBuilder = createConstructorBuilder()
             coroutineConstructorBuilder.initialize()
 
-            val doResumeFunction = coroutineImplSymbol.owner.simpleFunctions()
+            val invokeSuspendFunction = baseClass.simpleFunctions()
-                    .single { it.name.asString() == "doResume" }
+                    .single { it.name.asString() == "invokeSuspend" }
-            val doResumeMethodBuilder = createDoResumeMethodBuilder(doResumeFunction, coroutineClass)
+            val invokeSuspendMethodBuilder = createInvokeSuspendMethodBuilder(invokeSuspendFunction, coroutineClass)
-            doResumeMethodBuilder.initialize()
+            invokeSuspendMethodBuilder.initialize()
-            overriddenMap += doResumeFunction.descriptor to doResumeMethodBuilder.symbol.descriptor
+            overriddenMap += invokeSuspendFunction.descriptor to invokeSuspendMethodBuilder.symbol.descriptor
 
             var coroutineFactoryConstructorBuilder: SymbolWithIrBuilder<IrConstructorSymbol, IrConstructor>? = null
             var createMethodBuilder: SymbolWithIrBuilder<IrSimpleFunctionSymbol, IrSimpleFunction>? = null
@@ -358,7 +370,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                 // Suspend lambda - create factory methods.
                 coroutineFactoryConstructorBuilder = createFactoryConstructorBuilder(boundFunctionParameters!!)
 
-                val createFunctionDescriptor = coroutineImplClassDescriptor.unsubstitutedMemberScope
+                val createFunctionDescriptor = baseClass.descriptor.unsubstitutedMemberScope
                         .getContributedFunctions(Name.identifier("create"), NoLookupLocation.FROM_BACKEND)
                         .atMostOne { it.valueParameters.size == unboundFunctionParameters!!.size + 1 }
                 createMethodBuilder = createCreateMethodBuilder(
@@ -378,7 +390,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                         suspendFunctionInvokeFunctionDescriptor = suspendInvokeFunctionDescriptor,
                         functionInvokeFunctionDescriptor        = invokeFunctionDescriptor,
                         createFunction                          = createMethodBuilder.ir,
-                        doResumeFunction                        = doResumeMethodBuilder.ir,
+                        invokeSuspendFunction                   = invokeSuspendMethodBuilder.ir,
                         coroutineClass                          = coroutineClass)
             }
 
@@ -398,7 +410,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                 coroutineClass.addChild(it.ir)
             }
 
-            coroutineClass.addChild(doResumeMethodBuilder.ir)
+            coroutineClass.addChild(invokeSuspendMethodBuilder.ir)
 
             coroutineClass.setSuperSymbolsAndAddFakeOverrides(superTypes)
 
@@ -406,7 +418,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                     coroutineClass       = coroutineClass,
                     coroutineConstructor = coroutineFactoryConstructorBuilder?.ir
                             ?: coroutineConstructorBuilder.ir,
-                    doResumeFunction     = doResumeMethodBuilder.ir)
+                    invokeSuspendFunction = invokeSuspendMethodBuilder.ir)
         }
 
         private fun createConstructorBuilder()
@@ -427,7 +439,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                 val descriptor = symbol.descriptor as ClassConstructorDescriptorImpl
                 constructorParameters = (
                         functionParameters
-                        + coroutineImplConstructorSymbol.owner.valueParameters[0] // completion.
+                        + baseClassConstructor.valueParameters[0] // completion.
                         ).mapIndexed { index, parameter ->
 
                     val parameterDescriptor = parameter.descriptor.copyAsValueParameter(descriptor, index)
@@ -464,10 +476,13 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                         val completionParameter = valueParameters.last()
                         +IrDelegatingConstructorCallImpl(startOffset, endOffset,
                                 context.irBuiltIns.unitType,
-                                coroutineImplConstructorSymbol, coroutineImplConstructorSymbol.descriptor).apply {
+                                baseClassConstructor.symbol, baseClassConstructor.descriptor).apply {
                             putValueArgument(0, irGet(completionParameter))
                         }
                         +IrInstanceInitializerCallImpl(startOffset, endOffset, coroutineClass.symbol, context.irBuiltIns.unitType)
+
+                        +irSetField(irGet(coroutineClassThis), labelField, irCall(symbols.getNativeNullPtr.owner))
+
                         functionParameters.forEachIndexed { index, parameter ->
                             +irSetField(
                                     irGet(coroutineClassThis),
@@ -523,7 +538,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                     val irBuilder = context.createIrBuilder(symbol, startOffset, endOffset)
                     body = irBuilder.irBlockBody {
                         +IrDelegatingConstructorCallImpl(startOffset, endOffset, context.irBuiltIns.unitType,
-                                coroutineImplConstructorSymbol, coroutineImplConstructorSymbol.descriptor).apply {
+                                baseClassConstructor.symbol, baseClassConstructor.descriptor).apply {
                             putValueArgument(0, irNull()) // Completion.
                         }
                         +IrInstanceInitializerCallImpl(startOffset, endOffset, coroutineClass.symbol,
@@ -573,8 +588,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                         /* modality                     = */ Modality.FINAL,
                         /* visibility                   = */ Visibilities.PRIVATE).apply {
                     if (superFunctionDescriptor != null) {
-                        overriddenDescriptors           +=   superFunctionDescriptor.overriddenDescriptors
+                        overriddenDescriptors           =    listOf(superFunctionDescriptor)
-                        overriddenDescriptors           +=   superFunctionDescriptor
                     }
                 }
             }
@@ -622,7 +636,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
         private fun createInvokeMethodBuilder(suspendFunctionInvokeFunctionDescriptor: FunctionDescriptor,
                                               functionInvokeFunctionDescriptor: FunctionDescriptor,
                                               createFunction: IrFunction,
-                                              doResumeFunction: IrFunction,
+                                              invokeSuspendFunction: IrFunction,
                                               coroutineClass: IrClass)
                 = object: SymbolWithIrBuilder<IrSimpleFunctionSymbol, IrSimpleFunction>() {
 
@@ -679,7 +693,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                     val irBuilder = context.createIrBuilder(symbol, startOffset, endOffset)
                     body = irBuilder.irBlockBody(startOffset, endOffset) {
                         +irReturn(
-                                irCall(doResumeFunction).apply {
+                                irCall(invokeSuspendFunction).apply {
                                     dispatchReceiver = irCall(createFunction).apply {
                                         dispatchReceiver = irGet(thisReceiver)
                                         valueParameters.forEachIndexed { index, parameter ->
@@ -688,8 +702,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                                         putValueArgument(valueParameters.size,
                                                 irCall(getContinuation, listOf(returnType)))
                                     }
-                                    putValueArgument(0, irGetObject(symbols.unit))       // value
+                                    putValueArgument(0, irSuccess(irGetObject(symbols.unit)))
-                                    putValueArgument(1, irNull())       // exception
                                 }
                         )
                     }
@@ -709,11 +722,11 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
             coroutineClass.addChild(it)
         }
 
-        private fun createDoResumeMethodBuilder(doResumeFunction: IrFunction, coroutineClass: IrClass)
+        private fun createInvokeSuspendMethodBuilder(invokeSuspendFunction: IrFunction, coroutineClass: IrClass)
                 = object: SymbolWithIrBuilder<IrSimpleFunctionSymbol, IrSimpleFunction>() {
 
             override fun buildSymbol() = IrSimpleFunctionSymbolImpl(
-                    doResumeFunction.descriptor.createOverriddenDescriptor(coroutineClassDescriptor)
+                    invokeSuspendFunction.descriptor.createOverriddenDescriptor(coroutineClassDescriptor)
             )
 
             override fun doInitialize() { }
@@ -733,17 +746,13 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
 
                     this.createDispatchReceiverParameter()
 
-                    doResumeFunction.valueParameters.mapIndexedTo(this.valueParameters) { index, it ->
+                    invokeSuspendFunction.valueParameters.mapIndexedTo(this.valueParameters) { index, it ->
                         it.copy(descriptor.valueParameters[index])
                     }
 
                 }
 
-                dataArgument = function.valueParameters[0]
+                resultArgument = function.valueParameters.single()
-                exceptionArgument = function.valueParameters[1]
-
-                val label = coroutineImplSymbol.owner.declarations.filterIsInstance<IrProperty>()
-                        .single { it.name.asString() == "label" }
 
                 val irBuilder = context.createIrBuilder(function.symbol, startOffset, endOffset)
                 function.body = irBuilder.irBlockBody(startOffset, endOffset) {
@@ -756,7 +765,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                     , context.irBuiltIns.anyNType)
 
                     // Extract all suspend calls to temporaries in order to make correct jumps to them.
-                    originalBody.transformChildrenVoid(ExpressionSlicer(label.getter!!.returnType))
+                    originalBody.transformChildrenVoid(ExpressionSlicer(labelField.type))
 
                     val liveLocals = computeLivenessAtSuspensionPoints(originalBody)
 
@@ -837,10 +846,11 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                                                     val variable = localsMap[it.descriptor] ?: it
                                                     +irSetField(irGet(thisReceiver), localToPropertyMap[it.symbol]!!, irGet(variable))
                                                 }
-                                                +irCall(coroutineImplLabelSetterSymbol).apply {
+                                                +irSetField(
-                                                    dispatchReceiver = irGet(thisReceiver)
+                                                        irGet(thisReceiver),
-                                                    putValueArgument(0, irGet(suspensionPoint.suspensionPointIdParameter))
+                                                        labelField,
-                                                }
+                                                        irGet(suspensionPoint.suspensionPointIdParameter)
+                                                )
                                             }
                                         }
                                         restoreState.symbol -> {
@@ -865,11 +875,9 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                             startOffset       = startOffset,
                             endOffset         = endOffset,
                             type              = context.irBuiltIns.unitType,
-                            suspensionPointId = irCall(coroutineImplLabelGetterSymbol).apply {
+                            suspensionPointId = irGetField(irGet(function.dispatchReceiverParameter!!), labelField),
-                                dispatchReceiver = irGet(function.dispatchReceiverParameter!!)
-                            },
                             result            = irBlock(startOffset, endOffset) {
-                                +irThrowIfNotNull(exceptionArgument)    // Coroutine might start with an exception.
+                                +irThrowIfNotNull(irExceptionOrNull(irGet(resultArgument))) // Coroutine might start with an exception.
                                 statements.forEach { +it }
                             })
                     if (irFunction.returnType.isUnit())
@@ -1107,8 +1115,7 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                             },
                             resumeResult               = irBlock(startOffset, endOffset) {
                                 +irCall(restoreState)
-                                +irThrowIfNotNull(exceptionArgument)
+                                +irGetOrThrow(irGet(resultArgument))
-                                +irGet(dataArgument)
                             })
                     val expressionResult = when {
                         suspendCall.type.isUnit() -> irImplicitCoercionToUnit(suspensionPoint)
@@ -1188,7 +1195,11 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                 irIfThen(irEqeqeq(irGet(value), irCall(symbols.coroutineSuspendedGetter)),
                         irReturn(irGet(value)))
 
-        private fun IrBuilderWithScope.irThrowIfNotNull(exception: IrValueDeclaration) =
+        private fun IrBuilderWithScope.irThrowIfNotNull(exception: IrExpression) = irLetS(exception) {
+            irThrowIfNotNull(it.owner)
+        }
+
+        fun IrBuilderWithScope.irThrowIfNotNull(exception: IrValueDeclaration) =
                 irIfThen(irNot(irEqeqeq(irGet(exception), irNull())),
                         irThrow(irImplicitCast(irGet(exception), exception.type.makeNotNull())))
 
@@ -1200,6 +1211,30 @@ internal class SuspendFunctionsLowering(val context: Context): FileLoweringPass
                 }
     }
 
+    private fun IrBuilderWithScope.irGetOrThrow(successOrFailure: IrExpression): IrExpression {
+        // TODO: consider inlining getOrThrow function body here.
+        val successOrFailureClass = symbols.successOrFailure.owner
+        val getOrThrow = successOrFailureClass.simpleFunctions().single { it.name.asString() == "getOrThrow" }
+        return irCall(getOrThrow).apply {
+            dispatchReceiver = successOrFailure
+        }
+    }
+
+    private fun IrBuilderWithScope.irExceptionOrNull(successOrFailure: IrExpression): IrExpression {
+        val successOrFailureClass = symbols.successOrFailure.owner
+        val exceptionOrNull = successOrFailureClass.simpleFunctions().single { it.name.asString() == "exceptionOrNull" }
+        return irCall(exceptionOrNull).apply {
+            dispatchReceiver = successOrFailure
+        }
+    }
+
+    fun IrBlockBodyBuilder.irSuccess(value: IrExpression): IrCall {
+        val createSuccessOrFailure = symbols.successOrFailure.owner.constructors.single { it.isPrimary }
+        return irCall(createSuccessOrFailure).apply {
+            putValueArgument(0, value)
+        }
+    }
+
     private open class VariablesScopeTracker: IrElementVisitorVoid {
 
         protected val scopeStack = mutableListOf<MutableSet<IrVariable>>(mutableSetOf())

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/optimizations/DFGBuilder.kt

@@ -403,9 +403,9 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag
         }
     }
 
-    private val doResumeFunctionSymbol =
+    private val invokeSuspendFunctionSymbol =
-            context.ir.symbols.coroutineImpl.owner.declarations
+            context.ir.symbols.baseContinuationImpl.owner.declarations
-                    .filterIsInstance<IrSimpleFunction>().single { it.name.asString() == "doResume" }.symbol
+                    .filterIsInstance<IrSimpleFunction>().single { it.name.asString() == "invokeSuspend" }.symbol
 
     private val getContinuationSymbol = context.ir.symbols.getContinuation
     private val continuationType = getContinuationSymbol.owner.returnType
@@ -439,7 +439,7 @@ internal class ModuleDFGBuilder(val context: Context, val irModule: IrModuleFrag
 
             descriptor.isSuspend -> DataFlowIR.Node.Parameter(allParameters.size)
 
-            doResumeFunctionSymbol in descriptor.overriddenSymbols ->            // <this> is a CoroutineImpl inheritor.
+            descriptor.overrides(invokeSuspendFunctionSymbol.owner) ->           // <this> is a ContinuationImpl inheritor.
                 templateParameters[descriptor.dispatchReceiverParameter!!]       // It is its own continuation.
 
             else -> null

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanDescriptorSerializer.kt

@@ -259,7 +259,7 @@ class KonanDescriptorSerializer private constructor(
         if (requirement != null) {
             builder.addAllVersionRequirement(requirement)
         }
-        else if (descriptor.isSuspendOrHasSuspendTypesInSignature()) {
+        if (descriptor.isSuspendOrHasSuspendTypesInSignature()) {
             builder.addVersionRequirement(writeVersionRequirementDependingOnCoroutinesVersion())
         }
 
@@ -342,7 +342,7 @@ class KonanDescriptorSerializer private constructor(
         if (requirement != null) {
             builder.addAllVersionRequirement(requirement)
         }
-        else if (descriptor.isSuspendOrHasSuspendTypesInSignature()) {
+        if (descriptor.isSuspendOrHasSuspendTypesInSignature()) {
             builder.addVersionRequirement(writeVersionRequirementDependingOnCoroutinesVersion())
         }
 
@@ -380,7 +380,7 @@ class KonanDescriptorSerializer private constructor(
         if (requirement != null) {
             builder.addAllVersionRequirement(requirement)
         }
-        else if (descriptor.isSuspendOrHasSuspendTypesInSignature()) {
+        if (descriptor.isSuspendOrHasSuspendTypesInSignature()) {
             builder.addVersionRequirement(writeVersionRequirementDependingOnCoroutinesVersion())
         }
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanSerializerExtension.kt

@@ -17,6 +17,8 @@
 package org.jetbrains.kotlin.backend.konan.serialization
 
 import org.jetbrains.kotlin.backend.konan.Context
+import org.jetbrains.kotlin.config.LanguageFeature
+import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
 import org.jetbrains.kotlin.metadata.ProtoBuf
@@ -102,6 +104,9 @@ internal class KonanSerializerExtension(val context: Context, override val metad
         return IrSerializer( 
             context, inlineDescriptorTable, stringTable, serializer, descriptor).serializeInlineBody()
     }
+
+    override fun releaseCoroutines(): Boolean =
+            context.config.configuration.languageVersionSettings.supportsFeature(LanguageFeature.ReleaseCoroutines)
 }
 
 internal interface IrAwareExtension {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/ir/util/IrUtils2.kt

@@ -23,6 +23,7 @@ import org.jetbrains.kotlin.backend.konan.KonanBackendContext
 import org.jetbrains.kotlin.backend.konan.KonanCompilationException
 import org.jetbrains.kotlin.backend.konan.descriptors.synthesizedName
 import org.jetbrains.kotlin.backend.konan.irasdescriptors.*
+import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.descriptors.ClassDescriptor
 import org.jetbrains.kotlin.descriptors.FunctionDescriptor
@@ -50,6 +51,8 @@ import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.OverridingStrategy
 import org.jetbrains.kotlin.resolve.OverridingUtil
+import org.jetbrains.kotlin.resolve.calls.checkers.isRestrictsSuspensionReceiver
+import org.jetbrains.kotlin.resolve.descriptorUtil.builtIns
 import org.jetbrains.kotlin.types.*
 import org.jetbrains.kotlin.types.KotlinType
 import org.jetbrains.kotlin.types.typeUtil.immediateSupertypes
@@ -721,3 +724,17 @@ val IrType.isSimpleTypeWithQuestionMark: Boolean
 
 fun IrClass.defaultOrNullableType(hasQuestionMark: Boolean) =
         if (hasQuestionMark) this.defaultType.makeNullable() else this.defaultType
+
+fun FunctionDescriptor.createOverriddenDescriptor(owner: ClassDescriptor, final: Boolean = true): FunctionDescriptor {
+    return this.newCopyBuilder()
+            .setOwner(owner)
+            .setCopyOverrides(true)
+            .setModality(if (final) Modality.FINAL else Modality.OPEN)
+            .setDispatchReceiverParameter(owner.thisAsReceiverParameter)
+            .build()!!.apply {
+        overriddenDescriptors = listOf(this@createOverriddenDescriptor)
+    }
+}
+
+fun IrFunction.isRestrictedSuspendFunction(languageVersionSettings: LanguageVersionSettings): Boolean =
+        this.descriptor.extensionReceiverParameter?.type?.isRestrictsSuspensionReceiver(languageVersionSettings) == true

backend.native/tests/build.gradle

@@ -1329,16 +1329,19 @@ task coroutines_controlFlow_finally4(type: RunKonanTest) {
 }
 
 task coroutines_controlFlow_finally5(type: RunKonanTest) {
+    expectedFail = (project.testTarget == 'wasm32') // Uses exceptions.
     goldValue = "s1\nf2\nfinally\n1\n"
     source = "codegen/coroutines/controlFlow_finally5.kt"
 }
 
 task coroutines_controlFlow_finally6(type: RunKonanTest) {
+    expectedFail = (project.testTarget == 'wasm32') // Uses exceptions.
     goldValue = "s1\nfinally\nerror\n0\n"
     source = "codegen/coroutines/controlFlow_finally6.kt"
 }
 
 task coroutines_controlFlow_finally7(type: RunKonanTest) {
+    expectedFail = (project.testTarget == 'wasm32') // Uses exceptions.
     goldValue = "s1\nf2\nfinally1\ns2\nfinally2\n42\n"
     source = "codegen/coroutines/controlFlow_finally7.kt"
 }
@@ -1364,21 +1367,25 @@ task coroutines_controlFlow_tryCatch1(type: RunKonanTest) {
 }
 
 task coroutines_controlFlow_tryCatch2(type: RunKonanTest) {
+    expectedFail = (project.testTarget == 'wasm32') // Uses exceptions.
     goldValue = "s1\n42\n"
     source = "codegen/coroutines/controlFlow_tryCatch2.kt"
 }
 
 task coroutines_controlFlow_tryCatch3(type: RunKonanTest) {
+    expectedFail = (project.testTarget == 'wasm32') // Uses exceptions.
     goldValue = "s2\nf2\n1\n"
     source = "codegen/coroutines/controlFlow_tryCatch3.kt"
 }
 
 task coroutines_controlFlow_tryCatch4(type: RunKonanTest) {
+    expectedFail = (project.testTarget == 'wasm32') // Uses exceptions.
     goldValue = "s2\nf2\n1\n"
     source = "codegen/coroutines/controlFlow_tryCatch4.kt"
 }
 
 task coroutines_controlFlow_tryCatch5(type: RunKonanTest) {
+    expectedFail = (project.testTarget == 'wasm32') // Uses exceptions.
     goldValue = "s2\ns1\nError\n42\n"
     source = "codegen/coroutines/controlFlow_tryCatch5.kt"
 }

backend.native/tests/codegen/controlflow/for_loops_coroutines.kt

@@ -2,7 +2,7 @@ package codegen.controlflow.for_loops_coroutines
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
 
 @Test fun runTest() {
     val sq = buildSequence {

backend.native/tests/codegen/coroutines/anonymousObject.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.anonymousObject
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun suspendHere(): Int = suspendCoroutineOrReturn { x ->
+suspend fun suspendHere(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     x.resume(42)
     COROUTINE_SUSPENDED
 }

backend.native/tests/codegen/coroutines/controlFlow_finally1.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_finally1
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_finally2.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_finally2
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_finally3.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_finally3
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_finally4.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_finally4
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_finally5.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_finally5
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resumeWithException(Error())
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_finally6.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_finally6
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resumeWithException(Error("error"))
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_finally7.kt

@@ -2,22 +2,21 @@ package codegen.coroutines.controlFlow_finally7
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resumeWithException(Error())
     COROUTINE_SUSPENDED
 }
 
-suspend fun s2(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s2(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s2")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_if1.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_if1
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_if2.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_if2
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_inline1.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_inline1
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_inline2.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_inline2
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_inline3.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_inline3
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_tryCatch1.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_tryCatch1
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_tryCatch2.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.controlFlow_tryCatch2
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_tryCatch3.kt

@@ -2,22 +2,21 @@ package codegen.coroutines.controlFlow_tryCatch3
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED
 }
 
-suspend fun s2(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s2(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s2")
     x.resumeWithException(Error())
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_tryCatch4.kt

@@ -2,22 +2,21 @@ package codegen.coroutines.controlFlow_tryCatch4
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED
 }
 
-suspend fun s2(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s2(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s2")
     x.resumeWithException(Error())
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_tryCatch5.kt

@@ -2,22 +2,21 @@ package codegen.coroutines.controlFlow_tryCatch5
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED
 }
 
-suspend fun s2(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s2(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s2")
     x.resumeWithException(Error("Error"))
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_while1.kt

@@ -2,28 +2,27 @@ package codegen.coroutines.controlFlow_while1
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED
 }
 
-suspend fun s2(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s2(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s2")
     x.resumeWithException(Error("Error"))
     COROUTINE_SUSPENDED
 }
 
-suspend fun s3(value: Int): Int = suspendCoroutineOrReturn { x ->
+suspend fun s3(value: Int): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s3")
     x.resume(value)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/controlFlow_while2.kt

@@ -2,28 +2,27 @@ package codegen.coroutines.controlFlow_while2
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED
 }
 
-suspend fun s2(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s2(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s2")
     x.resumeWithException(Error("Error"))
     COROUTINE_SUSPENDED
 }
 
-suspend fun s3(value: Int): Int = suspendCoroutineOrReturn { x ->
+suspend fun s3(value: Int): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s3")
     x.resume(value)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/coroutineContext1.kt

@@ -2,13 +2,12 @@ package codegen.coroutines.coroutineContext1
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
 fun builder(c: suspend () -> Unit) {

backend.native/tests/codegen/coroutines/coroutineContext2.kt

@@ -2,13 +2,12 @@ package codegen.coroutines.coroutineContext2
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
 fun builder(c: suspend () -> Unit) {

backend.native/tests/codegen/coroutines/correctOrder1.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.correctOrder1
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Int = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(42)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/degenerate1.kt

@@ -2,13 +2,12 @@ package codegen.coroutines.degenerate1
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
 suspend fun s1() {

backend.native/tests/codegen/coroutines/degenerate2.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.degenerate2
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Unit = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Unit = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(Unit)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/returnsUnit1.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.returnsUnit1
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun s1(): Unit = suspendCoroutineOrReturn { x ->
+suspend fun s1(): Unit = suspendCoroutineUninterceptedOrReturn { x ->
     println("s1")
     x.resume(Unit)
     COROUTINE_SUSPENDED

backend.native/tests/codegen/coroutines/simple.kt

@@ -2,16 +2,15 @@ package codegen.coroutines.simple
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
-suspend fun suspendHere(): Int = suspendCoroutineOrReturn { x ->
+suspend fun suspendHere(): Int = suspendCoroutineUninterceptedOrReturn { x ->
     x.resume(42)
     COROUTINE_SUSPENDED
 }

backend.native/tests/codegen/coroutines/withReceiver.kt

@@ -2,17 +2,16 @@ package codegen.coroutines.withReceiver
 
 import kotlin.test.*
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
 class Controller {
-    suspend fun suspendHere(): Int = suspendCoroutineOrReturn { x ->
+    suspend fun suspendHere(): Int = suspendCoroutineUninterceptedOrReturn { x ->
         x.resume(42)
         COROUTINE_SUSPENDED
     }

buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy

@@ -151,59 +151,47 @@ abstract class KonanTest extends JavaExec {
     }
 
     void createCoroutineUtil(String file) {
-        StringBuilder text = new StringBuilder("import kotlin.coroutines.experimental.*\n")
+        StringBuilder text = new StringBuilder("import kotlin.coroutines.*\n")
         text.append(
                 """
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
 fun <T> handleResultContinuation(x: (T) -> Unit): Continuation<T> = object: Continuation<T> {
     override val context = EmptyCoroutineContext
-    override fun resumeWithException(exception: Throwable) {
+    override fun resumeWith(result: SuccessOrFailure<T>) { x(result.getOrThrow()) }
-        throw exception
-    }
-
-    override fun resume(data: T) = x(data)
 }
 
 fun handleExceptionContinuation(x: (Throwable) -> Unit): Continuation<Any?> = object: Continuation<Any?> {
     override val context = EmptyCoroutineContext
-    override fun resumeWithException(exception: Throwable) {
+    override fun resumeWith(result: SuccessOrFailure<Any?>) {
+        val exception = result.exceptionOrNull() ?: return
         x(exception)
     }
-
-    override fun resume(data: Any?) { }
 }
 """     )
         createFile(file, text.toString())
     }
 
     String createTextForHelpers() {
-        def coroutinesPackage = "kotlin.coroutines.experimental"
+        def coroutinesPackage = "kotlin.coroutines"
 
         def emptyContinuationBody =
             """
-                |override fun resume(data: Any?) {}
+                |override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-                |override fun resumeWithException(exception: Throwable) { throw exception }
             """.stripMargin()
 
         def handleResultContinuationBody = """
-                |override fun resumeWithException(exception: Throwable) {
+                |override fun resumeWith(result: SuccessOrFailure<T>) { x(result.getOrThrow()) }
-                |   throw exception
-                |}
-                |
-                |override fun resume(data: T) = x(data)
             """.stripMargin()
 
         def handleExceptionContinuationBody = """
-                |override fun resumeWithException(exception: Throwable) {
+                |override fun resumeWith(result: SuccessOrFailure<Any?>) {
-                |   x(exception)
+                |    val exception = result.exceptionOrNull() ?: return
+                |    x(exception)
                 |}
-                |
-                |override fun resume(data: Any?) {}
             """.stripMargin()
 
         return """
@@ -228,6 +216,16 @@ fun handleExceptionContinuation(x: (Throwable) -> Unit): Continuation<Any?> = ob
             |
             |abstract class ContinuationAdapter<in T> : Continuation<T> {
             |    override val context: CoroutineContext = EmptyCoroutineContext
+            |    override fun resumeWith(result: SuccessOrFailure<T>) {
+            |       if (result.isSuccess) {
+            |           resume(result.getOrThrow())
+            |       } else {
+            |           resumeWithException(result.exceptionOrNull()!!)
+            |       }
+            |    }
+            |
+            |    abstract fun resumeWithException(exception: Throwable)
+            |    abstract fun resume(value: T)
             |}
         """.stripMargin()
     }
@@ -741,7 +739,7 @@ class RunExternalTestGroup extends RunStandaloneKonanTest {
         for (String filePath : result) {
             def text = project.file(filePath).text
             if (text.contains('COROUTINES_PACKAGE')) {
-                text = text.replace('COROUTINES_PACKAGE', 'kotlin.coroutines.experimental')
+                text = text.replace('COROUTINES_PACKAGE', 'kotlin.coroutines')
             }
             def pkg = null
             if (text =~ packagePattern) {

runtime/src/main/kotlin/kotlin/coroutines/ContinuationImpl.kt

@@ -0,0 +1,133 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kotlin.coroutines.native.internal
+
+import kotlin.coroutines.*
+import kotlin.coroutines.intrinsics.CoroutineSingletons
+
+@SinceKotlin("1.3")
+internal abstract class BaseContinuationImpl(
+    // This is `public val` so that it is private on JVM and cannot be modified by untrusted code, yet
+    // it has a public getter (since even untrusted code is allowed to inspect its call stack).
+    public val completion: Continuation<Any?>?
+) : Continuation<Any?>, Serializable {
+    // This implementation is final. This fact is used to unroll resumeWith recursion.
+    public final override fun resumeWith(result: SuccessOrFailure<Any?>) {
+        // Invoke "resume" debug probe only once, even if previous frames are "resumed" in the loop below, too
+        probeCoroutineResumed(this)
+        // This loop unrolls recursion in current.resumeWith(param) to make saner and shorter stack traces on resume
+        var current = this
+        var param = result
+        while (true) {
+            with(current) {
+                val completion = completion!! // fail fast when trying to resume continuation without completion
+                val outcome: SuccessOrFailure<Any?> =
+                    try {
+                        val outcome = invokeSuspend(param)
+                        if (outcome === CoroutineSingletons.COROUTINE_SUSPENDED) return
+                        SuccessOrFailure.success(outcome)
+                    } catch (exception: Throwable) {
+                        SuccessOrFailure.failure(exception)
+                    }
+                releaseIntercepted() // this state machine instance is terminating
+                if (completion is BaseContinuationImpl) {
+                    // unrolling recursion via loop
+                    current = completion
+                    param = outcome
+                } else {
+                    // top-level completion reached -- invoke and return
+                    completion.resumeWith(outcome)
+                    return
+                }
+            }
+        }
+    }
+
+    protected abstract fun invokeSuspend(result: SuccessOrFailure<Any?>): Any?
+
+    protected open fun releaseIntercepted() {
+        // does nothing here, overridden in ContinuationImpl
+    }
+
+    public open fun create(completion: Continuation<*>): Continuation<Unit> {
+        throw UnsupportedOperationException("create(Continuation) has not been overridden")
+    }
+
+    public open fun create(value: Any?, completion: Continuation<*>): Continuation<Unit> {
+        throw UnsupportedOperationException("create(Any?;Continuation) has not been overridden")
+    }
+
+    public override fun toString(): String {
+        // todo: how continuation shall be rendered?
+        return "Continuation @ ${this::class.simpleName}"
+    }
+}
+
+@SinceKotlin("1.3")
+// State machines for named restricted suspend functions extend from this class
+internal abstract class RestrictedContinuationImpl(
+    completion: Continuation<Any?>?
+) : BaseContinuationImpl(completion) {
+    init {
+        completion?.let {
+            require(it.context === EmptyCoroutineContext) {
+                "Coroutines with restricted suspension must have EmptyCoroutineContext"
+            }
+        }
+    }
+
+    public override val context: CoroutineContext
+        get() = EmptyCoroutineContext
+}
+
+@SinceKotlin("1.3")
+// State machines for named suspend functions extend from this class
+internal abstract class ContinuationImpl(
+    completion: Continuation<Any?>?,
+    private val _context: CoroutineContext?
+) : BaseContinuationImpl(completion) {
+    constructor(completion: Continuation<Any?>?) : this(completion, completion?.context)
+
+    public override val context: CoroutineContext
+        get() = _context!! 
+
+    private var intercepted: Continuation<Any?>? = null
+
+    public fun intercepted(): Continuation<Any?> =
+        intercepted
+            ?: (context[ContinuationInterceptor]?.interceptContinuation(this) ?: this)
+                .also { intercepted = it }
+
+    protected override fun releaseIntercepted() {
+        val intercepted = intercepted
+        if (intercepted != null && intercepted !== this) {
+            context[ContinuationInterceptor]!!.releaseInterceptedContinuation(intercepted)
+        }
+        this.intercepted = CompletedContinuation // just in case
+    }
+}
+
+internal object CompletedContinuation : Continuation<Any?> {
+    override val context: CoroutineContext
+        get() = error("This continuation is already complete")
+
+    override fun resumeWith(result: SuccessOrFailure<Any?>) {
+        error("This continuation is already complete")
+    }
+
+    override fun toString(): String = "This continuation is already complete"
+}

runtime/src/main/kotlin/kotlin/coroutines/DebugProbes.kt

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package kotlin.coroutines.native.internal
+
+// TODO: support coroutines debugging in Kotlin/Native.
+
+import kotlin.coroutines.Continuation
+import kotlin.coroutines.intrinsics.*
+
+/**
+ * This probe is invoked when coroutine is being created and it can replace completion
+ * with its own wrapped object to intercept completion of this coroutine.
+ *
+ * This probe is invoked from stdlib implementation of [createCoroutineUnintercepted] function.
+ *
+ * Once created, coroutine is repeatedly [resumed][probeCoroutineResumed] and [suspended][probeCoroutineSuspended],
+ * until it is complete. On completion, the object that was returned by this probe is invoked.
+ *
+ * ```
+ * +-------+  probeCoroutineCreated +-----------+
+ * | START | ---------------------->| SUSPENDED |
+ * +-------+                        +-----------+
+ *                probeCoroutineResumed |   ^ probeCoroutineSuspended
+ *                                      V   |
+ *                                  +------------+ completion invoked  +-----------+
+ *                                  |   RUNNING  | ------------------->| COMPLETED |
+ *                                  +------------+                     +-----------+
+ * ```
+ *
+ * While the coroutine is resumed and suspended, it is represented by the pointer to its `frame`
+ * which always extends [BaseContinuationImpl] and represents a pointer to the topmost frame of the
+ * coroutine. Each [BaseContinuationImpl] object has [completion][BaseContinuationImpl.completion] reference
+ * that points either to another frame (extending [BaseContinuationImpl]) or to the completion object
+ * that was returned by this `probeCoroutineCreated` function.
+ *
+ * When coroutine is [suspended][probeCoroutineSuspended], then it is later [resumed][probeCoroutineResumed]
+ * with a reference to the same frame. However, while coroutine is running it can unwind its frames and
+ * invoke other suspending functions, so its next suspension can happen with a different frame pointer.
+ */
+@SinceKotlin("1.3")
+internal fun <T> probeCoroutineCreated(completion: Continuation<T>): Continuation<T> {
+    return completion
+}
+
+/**
+ * This probe is invoked when coroutine is resumed using [Continuation.resumeWith].
+ *
+ * This probe is invoked from stdlib implementation of [BaseContinuationImpl.resumeWith] function.
+ *
+ * Coroutines machinery implementation guarantees that the actual [frame] instance extends
+ * [BaseContinuationImpl] class, despite the fact that the declared type of [frame]
+ * parameter in this function is `Continuation<*>`. See [probeCoroutineCreated] for details.
+ */
+@SinceKotlin("1.3")
+internal fun probeCoroutineResumed(frame: Continuation<*>) {
+}
+
+/**
+ * This probe is invoked when coroutine is suspended using [suspendCoroutineUninterceptedOrReturn], that is
+ * when the corresponding `block` returns [COROUTINE_SUSPENDED].
+ *
+ * This probe is invoked from compiler-generated intrinsic for [suspendCoroutineUninterceptedOrReturn] function.
+ *
+ * Coroutines machinery implementation guarantees that the actual [frame] instance extends
+ * [BaseContinuationImpl] class, despite the fact that the declared type of [frame]
+ * parameter in this function is `Continuation<*>`. See [probeCoroutineCreated] for details.
+ */
+@SinceKotlin("1.3")
+internal fun probeCoroutineSuspended(frame: Continuation<*>) {
+}
+

runtime/src/main/kotlin/kotlin/coroutines/SafeContinuationNative.kt

@@ -16,7 +16,9 @@
 
 package kotlin.coroutines
 
-// TODO: implement them right.
+import kotlin.*
+import kotlin.coroutines.intrinsics.CoroutineSingletons.*
+
 @PublishedApi
 @SinceKotlin("1.3")
 internal actual class SafeContinuation<in T>
@@ -24,14 +26,37 @@ internal actual constructor(
         private val delegate: Continuation<T>,
         initialResult: Any?
 ) : Continuation<T> {
-
-    actual override val context: CoroutineContext
-    get() = TODO("unimplemented")
     @PublishedApi
-    internal actual constructor(delegate: Continuation<T>) : this(delegate, /*UNDECIDED*/ null)
+    internal actual constructor(delegate: Continuation<T>) : this(delegate, UNDECIDED)
+
+    public actual override val context: CoroutineContext
+        get() = delegate.context
+
+    private var result: Any? = initialResult
+
+    public actual override fun resumeWith(result: SuccessOrFailure<T>) {
+        val cur = this.result
+        when {
+            cur === UNDECIDED -> this.result = result.value
+            cur === COROUTINE_SUSPENDED -> {
+                this.result = RESUMED
+                delegate.resumeWith(result)
+            }
+            else -> throw IllegalStateException("Already resumed")
+        }
+    }
 
     @PublishedApi
-    internal actual fun getOrThrow(): Any? = TODO("unimplemented")
+    internal actual fun getOrThrow(): Any? {
-
+        val result = this.result
-    actual override fun resumeWith(result: SuccessOrFailure<T>):Unit = TODO("unimplemented")
+        if (result === UNDECIDED) {
+            this.result = COROUTINE_SUSPENDED
+            return COROUTINE_SUSPENDED
+        }
+        return when {
+            result === RESUMED -> COROUTINE_SUSPENDED // already called continuation, indicate COROUTINE_SUSPENDED upstream
+            result is SuccessOrFailure.Failure -> throw result.exception
+            else -> result // either COROUTINE_SUSPENDED or data
+        }
+    }
 }

runtime/src/main/kotlin/kotlin/coroutines/experimental/Intrinsics.kt

@@ -17,9 +17,6 @@
 package kotlin.coroutines.experimental.intrinsics
 
 import kotlin.coroutines.experimental.Continuation
-import kotlin.coroutines.experimental.CoroutineContext
-import kotlin.coroutines.experimental.processBareContinuationResume
-import kotlin.native.internal.*
 
 
 /**
@@ -34,13 +31,7 @@ import kotlin.native.internal.*
  */
 public actual fun <T> (suspend () -> T).createCoroutineUnchecked(
         completion: Continuation<T>
-): Continuation<Unit> =
+): Continuation<Unit> = errorExperimentalCoroutinesAreNoLongerSupported()
-        if (this !is CoroutineImpl)
-            buildContinuationByInvokeCall(completion) {
-                @Suppress("UNCHECKED_CAST") (this as Function1<Continuation<T>, Any?>).invoke(completion)
-            }
-        else
-            (this.create(completion) as CoroutineImpl).facade
 
 /**
  * Creates a coroutine with receiver type [R] and result type [T].
@@ -55,35 +46,7 @@ public actual fun <T> (suspend () -> T).createCoroutineUnchecked(
 public actual fun <R, T> (suspend R.() -> T).createCoroutineUnchecked(
         receiver: R,
         completion: Continuation<T>
-): Continuation<Unit> =
+): Continuation<Unit> = errorExperimentalCoroutinesAreNoLongerSupported()
-        if (this !is CoroutineImpl)
-            buildContinuationByInvokeCall(completion) {
-                @Suppress("UNCHECKED_CAST") (this as Function2<R, Continuation<T>, Any?>).invoke(receiver, completion)
-            }
-        else
-            (this.create(receiver, completion) as CoroutineImpl).facade
-
-// INTERNAL DEFINITIONS
-private inline fun <T> buildContinuationByInvokeCall(
-        completion: Continuation<T>,
-        crossinline block: () -> Any?
-): Continuation<Unit> {
-    val continuation =
-            object : Continuation<Unit> {
-                override val context: CoroutineContext
-                    get() = completion.context
-
-                override fun resume(value: Unit) {
-                    processBareContinuationResume(completion, block)
-                }
-
-                override fun resumeWithException(exception: Throwable) {
-                    completion.resumeWithException(exception)
-                }
-            }
-
-    return interceptContinuationIfNeeded(completion.context, continuation)
-}
 
 /**
  * Starts unintercepted coroutine without receiver and with result type [T] and executes it until its first suspension.
@@ -96,7 +59,7 @@ private inline fun <T> buildContinuationByInvokeCall(
 @kotlin.internal.InlineOnly
 public actual inline fun <T> (suspend () -> T).startCoroutineUninterceptedOrReturn(
         completion: Continuation<T>
-): Any? = (this as Function1<Continuation<T>, Any?>).invoke(completion)
+): Any? = errorExperimentalCoroutinesAreNoLongerSupported()
 
 /**
  * Starts unintercepted coroutine with receiver type [R] and result type [T] and executes it until its first suspension.
@@ -110,7 +73,7 @@ public actual inline fun <T> (suspend () -> T).startCoroutineUninterceptedOrRetu
 public actual inline fun <R, T> (suspend R.() -> T).startCoroutineUninterceptedOrReturn(
         receiver: R,
         completion: Continuation<T>
-): Any? = (this as Function2<R, Continuation<T>, Any?>).invoke(receiver, completion)
+): Any? = errorExperimentalCoroutinesAreNoLongerSupported()
 
 
 
@@ -119,6 +82,8 @@ public actual inline fun <R, T> (suspend R.() -> T).startCoroutineUninterceptedO
  * the execution was suspended and will not return any result immediately.
  */
 @SinceKotlin("1.1")
-public actual val COROUTINE_SUSPENDED: Any = CoroutineSuspendedMarker
+public actual val COROUTINE_SUSPENDED: Any get() = errorExperimentalCoroutinesAreNoLongerSupported()
 
-private object CoroutineSuspendedMarker
+@PublishedApi
+internal fun errorExperimentalCoroutinesAreNoLongerSupported(): Nothing =
+        error("kotlin.coroutines.experimental is no longer supported, please migrate to kotlin.coroutines")

runtime/src/main/kotlin/kotlin/coroutines/intrinsics/IntrinsicsNative.kt

@@ -17,35 +17,178 @@
 package kotlin.coroutines.intrinsics
 
 import kotlin.coroutines.*
-// TODO: implement them right.
+import kotlin.coroutines.native.internal.*
+import kotlin.native.internal.*
 
+/**
+ * Starts unintercepted coroutine without receiver and with result type [T] and executes it until its first suspension.
+ * Returns the result of the coroutine or throws its exception if it does not suspend or [COROUTINE_SUSPENDED] if it suspends.
+ * In the later case, the [completion] continuation is invoked when coroutine completes with result or exception.
+ * This function is designed to be used from inside of [suspendCoroutineOrReturn] to resume the execution of suspended
+ * coroutine using a reference to the suspending function.
+ */
+@Suppress("UNCHECKED_CAST")
+@kotlin.internal.InlineOnly
 public actual inline fun <T> (suspend () -> T).startCoroutineUninterceptedOrReturn(
         completion: Continuation<T>
-): Any? {
+): Any? = (this as Function1<Continuation<T>, Any?>).invoke(completion)
-    TODO("Unimplemented")
-}
 
+/**
+ * Starts unintercepted coroutine with receiver type [R] and result type [T] and executes it until its first suspension.
+ * Returns the result of the coroutine or throws its exception if it does not suspend or [COROUTINE_SUSPENDED] if it suspends.
+ * In the later case, the [completion] continuation is invoked when coroutine completes with result or exception.
+ * This function is designed to be used from inside of [suspendCoroutineOrReturn] to resume the execution of suspended
+ * coroutine using a reference to the suspending function.
+ */
+@Suppress("UNCHECKED_CAST")
+@kotlin.internal.InlineOnly
 public actual inline fun <R, T> (suspend R.() -> T).startCoroutineUninterceptedOrReturn(
         receiver: R,
         completion: Continuation<T>
-): Any? {
+): Any? = (this as Function2<R, Continuation<T>, Any?>).invoke(receiver, completion)
-    TODO("Unimplemented")
-}
 
+private object CoroutineSuspendedMarker
+
+
+/**
+ * Creates unintercepted coroutine without receiver and with result type [T].
+ * This function creates a new, fresh instance of suspendable computation every time it is invoked.
+ *
+ * To start executing the created coroutine, invoke `resume(Unit)` on the returned [Continuation] instance.
+ * The [completion] continuation is invoked when coroutine completes with result or exception.
+ *
+ * This function returns unintercepted continuation.
+ * Invocation of `resume(Unit)` starts coroutine immediately in the invoker's call stack without going through the
+ * [ContinuationInterceptor] that might be present in the completion's [CoroutineContext].
+ * It is invoker's responsibility to ensure that the proper invocation context is established.
+ * Note that [completion] of this function may get invoked in an arbitrary context.
+ *
+ * [Continuation.intercepted] can be used to acquire the intercepted continuation.
+ * Invocation of `resume(Unit)` on intercepted continuation guarantees that execution of
+ * both the coroutine and [completion] happens in the invocation context established by
+ * [ContinuationInterceptor].
+ *
+ * Repeated invocation of any resume function on the resulting continuation corrupts the
+ * state machine of the coroutine and may result in arbitrary behaviour or exception.
+ */
 @SinceKotlin("1.3")
 public actual fun <T> (suspend () -> T).createCoroutineUnintercepted(
         completion: Continuation<T>
 ): Continuation<Unit> {
-    TODO("Unimplemented")
+    val probeCompletion = probeCoroutineCreated(completion)
+    return if (this is BaseContinuationImpl)
+        create(probeCompletion)
+    else
+        createCoroutineFromSuspendFunction(probeCompletion) {
+            (this as Function1<Continuation<T>, Any?>).invoke(it)
+        }
 }
 
+/**
+ * Creates unintercepted coroutine with receiver type [R] and result type [T].
+ * This function creates a new, fresh instance of suspendable computation every time it is invoked.
+ *
+ * To start executing the created coroutine, invoke `resume(Unit)` on the returned [Continuation] instance.
+ * The [completion] continuation is invoked when coroutine completes with result or exception.
+ *
+ * This function returns unintercepted continuation.
+ * Invocation of `resume(Unit)` starts coroutine immediately in the invoker's call stack without going through the
+ * [ContinuationInterceptor] that might be present in the completion's [CoroutineContext].
+ * It is invoker's responsibility to ensure that the proper invocation context is established.
+ * Note that [completion] of this function may get invoked in an arbitrary context.
+ *
+ * [Continuation.intercepted] can be used to acquire the intercepted continuation.
+ * Invocation of `resume(Unit)` on intercepted continuation guarantees that execution of
+ * both the coroutine and [completion] happens in the invocation context established by
+ * [ContinuationInterceptor].
+ *
+ * Repeated invocation of any resume function on the resulting continuation corrupts the
+ * state machine of the coroutine and may result in arbitrary behaviour or exception.
+ */
 @SinceKotlin("1.3")
 public actual fun <R, T> (suspend R.() -> T).createCoroutineUnintercepted(
         receiver: R,
         completion: Continuation<T>
 ): Continuation<Unit> {
-    TODO("Unimplemented")
+    val probeCompletion = probeCoroutineCreated(completion)
+    return if (this is BaseContinuationImpl)
+        create(receiver, probeCompletion)
+    else {
+        createCoroutineFromSuspendFunction(probeCompletion) {
+            (this as Function2<R, Continuation<T>, Any?>).invoke(receiver, it)
+        }
+    }
 }
 
+/**
+ * Intercepts this continuation with [ContinuationInterceptor].
+ *
+ * This function shall be used on the immediate result of [createCoroutineUnintercepted] or [suspendCoroutineUninterceptedOrReturn],
+ * in which case it checks for [ContinuationInterceptor] in the continuation's [context][Continuation.context],
+ * invokes [ContinuationInterceptor.interceptContinuation], caches and returns result.
+ *
+ * If this function is invoked on other [Continuation] instances it returns `this` continuation unchanged.
+ */
 @SinceKotlin("1.3")
-public actual fun <T> Continuation<T>.intercepted(): Continuation<T> = TODO("unimplemented")
+public actual fun <T> Continuation<T>.intercepted(): Continuation<T> =
+        (this as? ContinuationImpl)?.intercepted() ?: this
+
+// INTERNAL DEFINITIONS
+
+/**
+ * This function is used when [createCoroutineUnintercepted] encounters suspending lambda that does not extend BaseContinuationImpl.
+ *
+ * It happens in two cases:
+ *   1. Callable reference to suspending function,
+ *   2. Suspending function reference implemented by Java code.
+ *
+ * We must wrap it into an instance that extends [BaseContinuationImpl], because that is an expectation of all coroutines machinery.
+ * As an optimization we use lighter-weight [RestrictedContinuationImpl] base class (it has less fields) if the context is
+ * [EmptyCoroutineContext], and a full-blown [ContinuationImpl] class otherwise.
+ *
+ * The instance of [BaseContinuationImpl] is passed to the [block] so that it can be passed to the corresponding invocation.
+ */
+@SinceKotlin("1.3")
+private inline fun <T> createCoroutineFromSuspendFunction(
+        completion: Continuation<T>,
+        crossinline block: (Continuation<T>) -> Any?
+): Continuation<Unit> {
+    val context = completion.context
+    // label == 0 when coroutine is not started yet (initially) or label == 1 when it was
+    return if (context === EmptyCoroutineContext)
+        object : RestrictedContinuationImpl(completion as Continuation<Any?>) {
+            private var label = 0
+
+            override fun invokeSuspend(result: SuccessOrFailure<Any?>): Any? =
+                    when (label) {
+                        0 -> {
+                            label = 1
+                            result.getOrThrow() // Rethrow exception if trying to start with exception (will be caught by BaseContinuationImpl.resumeWith
+                            block(this) // run the block, may return or suspend
+                        }
+                        1 -> {
+                            label = 2
+                            result.getOrThrow() // this is the result if the block had suspended
+                        }
+                        else -> error("This coroutine had already completed")
+                    }
+        }
+    else
+        object : ContinuationImpl(completion as Continuation<Any?>, context) {
+            private var label = 0
+
+            override fun invokeSuspend(result: SuccessOrFailure<Any?>): Any? =
+                    when (label) {
+                        0 -> {
+                            label = 1
+                            result.getOrThrow() // Rethrow exception if trying to start with exception (will be caught by BaseContinuationImpl.resumeWith
+                            block(this) // run the block, may return or suspend
+                        }
+                        1 -> {
+                            label = 2
+                            result.getOrThrow() // this is the result if the block had suspended
+                        }
+                        else -> error("This coroutine had already completed")
+                    }
+        }
+}

runtime/src/main/kotlin/kotlin/native/internal/Coroutines.kt

@@ -16,19 +16,14 @@
 
 package kotlin.native.internal
 
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 @kotlin.internal.InlineOnly
 @PublishedApi
 internal inline suspend fun <T> suspendCoroutineUninterceptedOrReturn(crossinline block: (Continuation<T>) -> Any?): T =
         returnIfSuspended<T>(block(getContinuation<T>()))
 
-@kotlin.internal.InlineOnly
-@PublishedApi
-internal inline fun <T> Continuation<T>.intercepted(): Continuation<T> =
-        normalizeContinuation<T>(this)
-
 @Intrinsic
 @PublishedApi
 internal external fun <T> getContinuation(): Continuation<T>
@@ -41,65 +36,3 @@ internal inline suspend fun getCoroutineContext(): CoroutineContext =
 @Intrinsic
 @PublishedApi
 internal external suspend fun <T> returnIfSuspended(@Suppress("UNUSED_PARAMETER") argument: Any?): T
-
-
-@ExportForCompiler
-internal fun <T> interceptContinuationIfNeeded(
-        context: CoroutineContext,
-        continuation: Continuation<T>
-) = context[ContinuationInterceptor]?.interceptContinuation(continuation) ?: continuation
-
-/**
- * @suppress
- */
-@ExportForCompiler
-@PublishedApi
-internal fun <T> normalizeContinuation(continuation: Continuation<T>): Continuation<T> =
-        (continuation as? CoroutineImpl)?.facade ?: continuation
-
-/**
- * @suppress
- */
-@ExportForCompiler
-abstract internal class CoroutineImpl(
-        protected var completion: Continuation<Any?>?
-) : Continuation<Any?> {
-
-    // label == -1 when coroutine cannot be started (it is just a factory object) or has already finished execution
-    // label == 0 in initial part of the coroutine
-    protected var label: NativePtr = if (completion != null) NativePtr.NULL else NativePtr.NULL + (-1L)
-
-    private val _context: CoroutineContext? = completion?.context
-
-    override val context: CoroutineContext
-        get() = _context!!
-
-    private var _facade: Continuation<Any?>? = null
-
-    val facade: Continuation<Any?> get() {
-        if (_facade == null) _facade = interceptContinuationIfNeeded(_context!!, this)
-        return _facade!!
-    }
-
-    override fun resume(value: Any?) {
-        processBareContinuationResume(completion!!) {
-            doResume(value, null)
-        }
-    }
-
-    override fun resumeWithException(exception: Throwable) {
-        processBareContinuationResume(completion!!) {
-            doResume(null, exception)
-        }
-    }
-
-    protected abstract fun doResume(data: Any?, exception: Throwable?): Any?
-
-    open fun create(completion: Continuation<*>): Continuation<Unit> {
-        throw IllegalStateException("create(Continuation) has not been overridden")
-    }
-
-    open fun create(value: Any?, completion: Continuation<*>): Continuation<Unit> {
-        throw IllegalStateException("create(Any?;Continuation) has not been overridden")
-    }
-}

samples/nonBlockingEchoServer/src/main/kotlin/EchoServer.kt

@@ -16,8 +16,8 @@
 
 import kotlinx.cinterop.*
 import platform.posix.*
-import kotlin.coroutines.experimental.*
+import kotlin.coroutines.*
-import kotlin.coroutines.experimental.intrinsics.*
+import kotlin.coroutines.intrinsics.*
 
 fun main(args: Array<String>) {
     if (args.size < 1) {
@@ -96,9 +96,8 @@ class Client(val clientFd: Int, val waitingList: MutableMap<Int, WaitingFor>) {
         if (posix_errno() != EWOULDBLOCK)
             throw IOException(getUnixError())
         // Save continuation and suspend.
-        return suspendCoroutineOrReturn { continuation ->
+        return suspendCoroutine { continuation ->
             waitingList.put(clientFd, WaitingFor.Read(data, dataLength, continuation))
-            COROUTINE_SUSPENDED
         }
     }
 
@@ -109,17 +108,15 @@ class Client(val clientFd: Int, val waitingList: MutableMap<Int, WaitingFor>) {
         if (posix_errno() != EWOULDBLOCK)
             throw IOException(getUnixError())
         // Save continuation and suspend.
-        return suspendCoroutineOrReturn { continuation ->
+        return suspendCoroutine { continuation ->
             waitingList.put(clientFd, WaitingFor.Write(data, length, continuation))
-            COROUTINE_SUSPENDED
         }
     }
 }
 
 open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
     companion object : EmptyContinuation()
-    override fun resume(value: Any?) {}
+    override fun resumeWith(result: SuccessOrFailure<Any?>) { result.getOrThrow() }
-    override fun resumeWithException(exception: Throwable) { throw exception }
 }
 
 fun acceptClientsAndRun(serverFd: Int, block: suspend Client.() -> Unit) {