Implement unit suspend functions tail-call optimisation

Unlike previously, this optimisation works on every callee return type.
Tail-calls inside unit functions can be either
INVOKE...
ARETURN
or
INVOKE
POP
GETSTATIC kotlin/Unit.INSTANCE
ARETURN
The first pattern is already covered. The second one is a bit tricky,
since we cannot just assume than the function is tail-call, we also need
to check whether the callee returned COROUTINE_SUSPENDED marker.
Thus, resulting bytecode of function's 'epilogue' look like
DUP
INVOKESTATIC getCOROUTINE_SUSPENDED
IF_ACMPNE LN
ARETURN
LN:
POP

 #KT-28938 Fixed

compiler/backend/src/org/jetbrains/kotlin/codegen/ExpressionCodegen.java

@@ -2533,8 +2533,6 @@ public class ExpressionCodegen extends KtVisitor<StackValue, StackValue> impleme
         callGenerator.genCall(callableMethod, resolvedCall, defaultMaskWasGenerated, this);
 
         if (isSuspendNoInlineCall) {
-            addReturnsUnitMarkerIfNecessary(v, resolvedCall);
-
             addSuspendMarker(v, false);
             addInlineMarker(v, false);
         }

compiler/backend/src/org/jetbrains/kotlin/codegen/coroutines/CoroutineTransformerMethodVisitor.kt

@@ -13,6 +13,7 @@ import org.jetbrains.kotlin.codegen.StackValue
 import org.jetbrains.kotlin.codegen.TransformationMethodVisitor
 import org.jetbrains.kotlin.codegen.inline.*
 import org.jetbrains.kotlin.codegen.optimization.DeadCodeEliminationMethodTransformer
+import org.jetbrains.kotlin.codegen.optimization.boxing.isUnitInstance
 import org.jetbrains.kotlin.codegen.optimization.common.*
 import org.jetbrains.kotlin.codegen.optimization.fixStack.FixStackMethodTransformer
 import org.jetbrains.kotlin.codegen.optimization.fixStack.top
@@ -31,9 +32,7 @@ import org.jetbrains.org.objectweb.asm.Opcodes
 import org.jetbrains.org.objectweb.asm.Type
 import org.jetbrains.org.objectweb.asm.commons.InstructionAdapter
 import org.jetbrains.org.objectweb.asm.tree.*
-import org.jetbrains.org.objectweb.asm.tree.analysis.Frame
-import org.jetbrains.org.objectweb.asm.tree.analysis.SourceInterpreter
-import org.jetbrains.org.objectweb.asm.tree.analysis.SourceValue
+import org.jetbrains.org.objectweb.asm.tree.analysis.*
 import kotlin.math.max
 
 private const val COROUTINES_DEBUG_METADATA_VERSION = 1
@@ -85,6 +84,9 @@ class CoroutineTransformerMethodVisitor(
     override fun performTransformations(methodNode: MethodNode) {
         removeFakeContinuationConstructorCall(methodNode)
 
+        // Remove redundant markers which came from compiled bytecode
+        cleanUpReturnsUnitMarkers(methodNode)
+
         replaceFakeContinuationsWithRealOnes(
             methodNode,
             if (isForNamedFunction) getLastParameterIndex(methodNode.desc, methodNode.access) else 0
@@ -105,13 +107,17 @@ class CoroutineTransformerMethodVisitor(
         val actualCoroutineStart = methodNode.instructions.first
 
         if (isForNamedFunction) {
-            ReturnUnitMethodTransformer.transform(containingClassInternalName, methodNode)
-
             if (putContinuationParameterToLvt) {
                 addCompletionParameterToLVT(methodNode)
             }
 
-            if (allSuspensionPointsAreTailCalls(containingClassInternalName, methodNode, suspensionPoints)) {
+            val examined = ExaminedMethodNode(
+                languageVersionSettings,
+                containingClassInternalName,
+                methodNode
+            )
+            if (examined.allSuspensionPointsAreTailCalls(suspensionPoints)) {
+                examined.replacePopsBeforeSafeUnitInstancesWithCoroutineSuspendedChecks()
                 dropSuspensionMarkers(methodNode, suspensionPoints)
                 return
             }
@@ -123,8 +129,6 @@ class CoroutineTransformerMethodVisitor(
             continuationIndex = methodNode.maxLocals++
 
             prepareMethodNodePreludeForNamedFunction(methodNode)
-        } else {
-            ReturnUnitMethodTransformer.cleanUpReturnsUnitMarkers(methodNode, ReturnUnitMethodTransformer.findReturnsUnitMarks(methodNode))
         }
 
         for (suspensionPoint in suspensionPoints) {
@@ -229,6 +233,12 @@ class CoroutineTransformerMethodVisitor(
         )
     }
 
+    private fun cleanUpReturnsUnitMarkers(methodNode: MethodNode) {
+        for (marker in methodNode.instructions.asSequence().filter(::isReturnsUnitMarker)) {
+            methodNode.instructions.removeAll(listOf(marker.previous, marker))
+        }
+    }
+
     private fun findSuspensionPointLineNumber(suspensionPoint: SuspensionPoint) =
         suspensionPoint.suspensionCallBegin.findPreviousOrNull { it is LineNumberNode } as LineNumberNode?
 
@@ -810,6 +820,88 @@ class CoroutineTransformerMethodVisitor(
     private data class SpilledVariableDescriptor(val fieldName: String, val variableName: String)
 }
 
+// TODO Use this in variable liveness analysis
+private class ExaminedMethodNode(
+    val languageVersionSettings: LanguageVersionSettings,
+    val containingClassInternalName: String,
+    val methodNode: MethodNode
+) {
+    // DO NOT REORDER: pops and areturns collecting depends on unit instances collecting
+    // Which, in turn depends on frames and cfg
+    val sourceFrames: Array<Frame<SourceValue>?> =
+        MethodTransformer.analyze(containingClassInternalName, methodNode, IgnoringCopyOperationSourceInterpreter())
+    val controlFlowGraph = ControlFlowGraph.build(methodNode)
+
+    private val safeUnitInstances = collectSafeUnitInstances()
+    private val popsBeforeSafeUnitInstances = collectPopsBeforeSafeUnitInstances()
+    private val areturnsAfterSafeUnitInstances = collectAreturnsAfterSafeUnitInstances()
+
+    private fun collectSafeUnitInstances() = methodNode.instructions.asSequence().filter { unit ->
+        unit.isUnitInstance() &&
+                methodNode.instructions.asSequence().any { insn ->
+                    insn.opcode == Opcodes.POP && insn.meaningfulSuccessors().let { succs ->
+                        succs.all { it.isUnitInstance() } && unit in succs
+                    }
+                } && unit.meaningfulSuccessors().all { it.opcode == Opcodes.ARETURN }
+    }.toList()
+
+    private fun collectPopsBeforeSafeUnitInstances() = methodNode.instructions.asSequence().filter { pop ->
+        pop.opcode == Opcodes.POP && pop.meaningfulSuccessors().all { it.isSafeUnitInstance() }
+    }.toList()
+
+    private fun collectAreturnsAfterSafeUnitInstances() = methodNode.instructions.asSequence().filter { areturn ->
+        areturn.opcode == Opcodes.ARETURN && sourceFrames[areturn.index()]?.top()?.insns?.all { it.isSafeUnitInstance() } == true
+    }
+
+    fun AbstractInsnNode.index() = methodNode.instructions.indexOf(this)
+
+    // GETSTATIC kotlin/Unit.INSTANCE is considered safe iff
+    // it is part of POP, PUSH Unit, ARETURN sequence.
+    fun AbstractInsnNode.isSafeUnitInstance(): Boolean = this in safeUnitInstances
+
+    fun AbstractInsnNode.isPopBeforeSafeUnitInstance(): Boolean = this in popsBeforeSafeUnitInstances
+    fun AbstractInsnNode.isAreturnAfterSafeUnitInstance(): Boolean = this in areturnsAfterSafeUnitInstances
+
+    private fun AbstractInsnNode.meaningfulSuccessors(): List<AbstractInsnNode> {
+        fun AbstractInsnNode.isMeaningful() = isMeaningful && opcode != Opcodes.NOP && opcode != Opcodes.GOTO && this !is LineNumberNode
+
+        val visited = arrayListOf<AbstractInsnNode>()
+        fun dfs(insn: AbstractInsnNode) {
+            if (insn in visited) return
+            visited += insn
+            if (!insn.isMeaningful()) {
+                for (succIndex in controlFlowGraph.getSuccessorsIndices(insn)) {
+                    dfs(methodNode.instructions[succIndex])
+                }
+            }
+        }
+
+        for (succIndex in controlFlowGraph.getSuccessorsIndices(this)) {
+            dfs(methodNode.instructions[succIndex])
+        }
+        return visited.filter { it.isMeaningful() }
+    }
+
+    fun replacePopsBeforeSafeUnitInstancesWithCoroutineSuspendedChecks() {
+        val basicAnalyser = Analyzer(BasicInterpreter())
+        basicAnalyser.analyze(containingClassInternalName, methodNode)
+        val typedFrames = basicAnalyser.frames
+
+        for (pop in popsBeforeSafeUnitInstances) {
+            if (!isUnreachable(pop.index(), sourceFrames) && typedFrames[pop.index()]?.top()?.isReference == true) {
+                val label = Label()
+                methodNode.instructions.insertBefore(pop, withInstructionAdapter {
+                    dup()
+                    loadCoroutineSuspendedMarker(languageVersionSettings)
+                    ifacmpne(label)
+                    areturn(AsmTypes.OBJECT_TYPE)
+                    mark(label)
+                })
+            }
+        }
+    }
+}
+
 internal fun InstructionAdapter.generateContinuationConstructorCall(
     objectTypeForState: Type?,
     methodNode: MethodNode,
@@ -939,13 +1031,8 @@ private fun getAllParameterTypes(desc: String, hasDispatchReceiver: Boolean, thi
     listOfNotNull(if (!hasDispatchReceiver) null else Type.getObjectType(thisName)).toTypedArray() +
             Type.getArgumentTypes(desc)
 
-private fun allSuspensionPointsAreTailCalls(
-    thisName: String,
-    methodNode: MethodNode,
-    suspensionPoints: List<SuspensionPoint>
-): Boolean {
-    val sourceFrames = MethodTransformer.analyze(thisName, methodNode, IgnoringCopyOperationSourceInterpreter())
-    val safelyReachableReturns = findSafelyReachableReturns(methodNode, sourceFrames)
+private fun ExaminedMethodNode.allSuspensionPointsAreTailCalls(suspensionPoints: List<SuspensionPoint>): Boolean {
+    val safelyReachableReturns = findSafelyReachableReturns()
 
     val instructions = methodNode.instructions
     return suspensionPoints.all { suspensionPoint ->
@@ -963,7 +1050,7 @@ private fun allSuspensionPointsAreTailCalls(
         if (insideTryBlock) return@all false
 
         safelyReachableReturns[endIndex + 1]?.all { returnIndex ->
-            sourceFrames[returnIndex].top().sure {
+            sourceFrames[returnIndex]?.top().sure {
                 "There must be some value on stack to return"
             }.insns.any { sourceInsn ->
                 sourceInsn?.let(instructions::indexOf) in beginIndex..endIndex
@@ -985,20 +1072,24 @@ internal class IgnoringCopyOperationSourceInterpreter : SourceInterpreter(Opcode
  *
  * @return indices of safely reachable returns for each instruction in the method node
  */
-private fun findSafelyReachableReturns(methodNode: MethodNode, sourceFrames: Array<Frame<SourceValue?>?>): Array<Set<Int>?> {
-    val controlFlowGraph = ControlFlowGraph.build(methodNode)
-
+private fun ExaminedMethodNode.findSafelyReachableReturns(): Array<Set<Int>?> {
     val insns = methodNode.instructions
-    val reachableReturnsIndices = Array<Set<Int>?>(insns.size()) init@ { index ->
+    val reachableReturnsIndices = Array<Set<Int>?>(insns.size()) init@{ index ->
         val insn = insns[index]
 
-        if (insn.opcode == Opcodes.ARETURN) {
+        if (insn.opcode == Opcodes.ARETURN && !insn.isAreturnAfterSafeUnitInstance()) {
             if (isUnreachable(index, sourceFrames)) return@init null
             return@init setOf(index)
         }
 
-        if (!insn.isMeaningful || insn.opcode in SAFE_OPCODES || insn.isInvisibleInDebugVarInsn(methodNode) ||
-            isInlineMarker(insn)) {
+        // Since POP, PUSH Unit, ARETURN behaves like normal return in terms of tail-call optimization, set return index to POP
+        if (insn.isPopBeforeSafeUnitInstance()) {
+            return@init setOf(index)
+        }
+
+        if (!insn.isMeaningful || insn.opcode in SAFE_OPCODES || insn.isInvisibleInDebugVarInsn(methodNode) || isInlineMarker(insn)
+            || insn.isSafeUnitInstance() || insn.isAreturnAfterSafeUnitInstance()
+        ) {
             setOf<Int>()
         } else null
     }
@@ -1029,7 +1120,8 @@ private fun findSafelyReachableReturns(methodNode: MethodNode, sourceFrames: Arr
 }
 
 // Check whether this instruction is unreachable, i.e. there is no path leading to this instruction
-internal fun isUnreachable(index: Int, sourceFrames: Array<Frame<SourceValue?>?>) = sourceFrames[index] == null
+internal fun isUnreachable(index: Int, sourceFrames: Array<Frame<SourceValue>?>): Boolean =
+    sourceFrames.size <= index || sourceFrames[index] == null
 
 private fun AbstractInsnNode?.isInvisibleInDebugVarInsn(methodNode: MethodNode): Boolean {
     val insns = methodNode.instructions

compiler/backend/src/org/jetbrains/kotlin/codegen/coroutines/RedundantLocalsEliminationMethodTransformer.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2018 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
@@ -12,8 +12,10 @@ import org.jetbrains.kotlin.codegen.optimization.common.isMeaningful
 import org.jetbrains.kotlin.codegen.optimization.common.removeAll
 import org.jetbrains.kotlin.codegen.optimization.transformer.MethodTransformer
 import org.jetbrains.kotlin.config.LanguageVersionSettings
+import org.jetbrains.kotlin.utils.keysToMap
 import org.jetbrains.org.objectweb.asm.Opcodes
 import org.jetbrains.org.objectweb.asm.tree.*
+import org.jetbrains.org.objectweb.asm.tree.analysis.SourceInterpreter
 
 // Inliner emits a lot of locals during inlining.
 // Remove all of them since these locals are
@@ -222,4 +224,22 @@ class RedundantLocalsEliminationMethodTransformer(private val languageVersionSet
         assert(this is VarInsnNode)
         return (this as VarInsnNode).`var`
     }
-}
+}
+
+private fun findSourceInstructions(
+    internalClassName: String,
+    methodNode: MethodNode,
+    insns: Collection<AbstractInsnNode>,
+    ignoreCopy: Boolean
+): Map<AbstractInsnNode, Collection<AbstractInsnNode>> {
+    val frames = MethodTransformer.analyze(
+        internalClassName,
+        methodNode,
+        if (ignoreCopy) IgnoringCopyOperationSourceInterpreter() else SourceInterpreter()
+    )
+    return insns.keysToMap {
+        val index = methodNode.instructions.indexOf(it)
+        if (isUnreachable(index, frames)) return@keysToMap emptySet<AbstractInsnNode>()
+        frames[index].getStack(0).insns
+    }
+}

compiler/backend/src/org/jetbrains/kotlin/codegen/coroutines/ReturnUnitMethodTransformer.kt

@@ -1,151 +0,0 @@
-/*
- * Copyright 2010-2018 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.codegen.coroutines
-
-import org.jetbrains.kotlin.codegen.inline.*
-import org.jetbrains.kotlin.codegen.optimization.boxing.isUnitInstance
-import org.jetbrains.kotlin.codegen.optimization.common.ControlFlowGraph
-import org.jetbrains.kotlin.codegen.optimization.common.asSequence
-import org.jetbrains.kotlin.codegen.optimization.common.isMeaningful
-import org.jetbrains.kotlin.codegen.optimization.common.removeAll
-import org.jetbrains.kotlin.codegen.optimization.transformer.MethodTransformer
-import org.jetbrains.kotlin.utils.addToStdlib.safeAs
-import org.jetbrains.kotlin.utils.keysToMap
-import org.jetbrains.kotlin.utils.sure
-import org.jetbrains.org.objectweb.asm.Opcodes
-import org.jetbrains.org.objectweb.asm.tree.*
-import org.jetbrains.org.objectweb.asm.tree.analysis.SourceInterpreter
-
-/*
- * Replace POP with ARETURN iff
- * 1) It is immediately followed by { GETSTATIC Unit.INSTANCE, ARETURN } sequences
- * 2) It is poping Unit
- *
- * Replace CHECKCAST Unit whit ARETURN iff
- * It is successed by ARETURN and it casts Unit
- */
-object ReturnUnitMethodTransformer : MethodTransformer() {
-    override fun transform(internalClassName: String, methodNode: MethodNode) {
-        val unitMarks = findReturnsUnitMarks(methodNode)
-        if (unitMarks.isEmpty()) return
-
-        replaceCheckcastUnitWithAreturn(methodNode, internalClassName)
-        replacePopWithAreturn(methodNode, internalClassName)
-
-        cleanUpReturnsUnitMarkers(methodNode, unitMarks)
-    }
-
-    private fun replacePopWithAreturn(
-        methodNode: MethodNode,
-        internalClassName: String
-    ) {
-        val units = findReturnUnitSequences(methodNode)
-        if (units.isEmpty()) return
-
-        replaceSafeInsnsWithUnit(methodNode, internalClassName, units) { it.opcode == Opcodes.POP }
-    }
-
-    private fun replaceCheckcastUnitWithAreturn(methodNode: MethodNode, internalClassName: String) {
-        val areturns = methodNode.instructions.asSequence().filter { it.opcode == Opcodes.ARETURN }.toList()
-
-        replaceSafeInsnsWithUnit(methodNode, internalClassName, areturns) { it.opcode == Opcodes.CHECKCAST }
-    }
-
-    // Find all instructions, which can be safely replaced with ARETURN and replace them with ARETURN for tail-call optimization
-    private fun replaceSafeInsnsWithUnit(
-        methodNode: MethodNode,
-        internalClassName: String,
-        safeSuccessors: Collection<AbstractInsnNode>,
-        predicate: (AbstractInsnNode) -> Boolean
-    ) {
-        val insns = methodNode.instructions.asSequence().filter(predicate).toList()
-        val successors = findSuccessors(methodNode, insns)
-        val sourceInsns = findSourceInstructions(internalClassName, methodNode, insns, ignoreCopy = true)
-        val safeInsns = filterOutUnsafes(successors, safeSuccessors, sourceInsns)
-        safeInsns.forEach { methodNode.instructions.set(it, InsnNode(Opcodes.ARETURN)) }
-    }
-
-    // Return list of instructions, which can be safely replaced by ARETURNs
-    private fun filterOutUnsafes(
-        successors: Map<AbstractInsnNode, Collection<AbstractInsnNode>>,
-        safeSuccessors: Collection<AbstractInsnNode>,
-        sources: Map<AbstractInsnNode, Collection<AbstractInsnNode>>
-    ): Collection<AbstractInsnNode> {
-        return successors.filter { (insn, successors) ->
-            successors.all { it in safeSuccessors } &&
-                    sources[insn].sure { "Sources of $insn cannot be null" }.all(::isSuspendingCallReturningUnit)
-        }.keys
-    }
-
-    // Find instructions which do something on stack, ignoring markers
-    // Return map {insn => list of found instructions}
-    private fun findSuccessors(
-        methodNode: MethodNode,
-        insns: Collection<AbstractInsnNode>
-    ): Map<AbstractInsnNode, Collection<AbstractInsnNode>> {
-        val cfg = ControlFlowGraph.build(methodNode)
-
-        return insns.keysToMap { findSuccessorsDFS(it, cfg, methodNode) }
-    }
-
-    // Find all meaningful successors of insn
-    private fun findSuccessorsDFS(insn: AbstractInsnNode, cfg: ControlFlowGraph, methodNode: MethodNode): Collection<AbstractInsnNode> {
-        val visited = hashSetOf<AbstractInsnNode>()
-
-        fun dfs(current: AbstractInsnNode): Collection<AbstractInsnNode> {
-            if (!visited.add(current)) return emptySet()
-
-            return cfg.getSuccessorsIndices(current).flatMap {
-                val succ = methodNode.instructions[it]
-                when {
-                    !succ.isMeaningful || succ is JumpInsnNode || succ.opcode == Opcodes.NOP -> dfs(succ)
-                    succ.isUnitInstance() -> {
-                        // There can be multiple chains of { UnitInstance, POP } after inlining. Ignore them
-                        val newSuccessors = dfs(succ)
-                        if (newSuccessors.all { it.opcode == Opcodes.POP }) newSuccessors.flatMap { dfs(it) }
-                        else setOf(succ)
-                    }
-                    else -> setOf(succ)
-                }
-            }
-        }
-
-        return dfs(insn)
-    }
-
-    private fun isSuspendingCallReturningUnit(node: AbstractInsnNode): Boolean =
-        node.safeAs<MethodInsnNode>()?.next?.next?.let(::isReturnsUnitMarker) == true
-
-    // Find { GETSTATIC kotlin/Unit.INSTANCE, ARETURN } sequences
-    // Result is list of GETSTATIC kotlin/Unit.INSTANCE instructions
-    private fun findReturnUnitSequences(methodNode: MethodNode): Collection<AbstractInsnNode> =
-        methodNode.instructions.asSequence().filter { it.isUnitInstance() && it.next?.opcode == Opcodes.ARETURN }.toList()
-
-    internal fun findReturnsUnitMarks(methodNode: MethodNode): Collection<AbstractInsnNode> =
-        methodNode.instructions.asSequence().filter(::isReturnsUnitMarker).toList()
-
-    internal fun cleanUpReturnsUnitMarkers(methodNode: MethodNode, unitMarks: Collection<AbstractInsnNode>) {
-        unitMarks.forEach { methodNode.instructions.removeAll(listOf(it.previous, it)) }
-    }
-}
-
-internal fun findSourceInstructions(
-    internalClassName: String,
-    methodNode: MethodNode,
-    insns: Collection<AbstractInsnNode>,
-    ignoreCopy: Boolean
-): Map<AbstractInsnNode, Collection<AbstractInsnNode>> {
-    val frames = MethodTransformer.analyze(
-        internalClassName,
-        methodNode,
-        if (ignoreCopy) IgnoringCopyOperationSourceInterpreter() else SourceInterpreter()
-    )
-    return insns.keysToMap {
-        val index = methodNode.instructions.indexOf(it)
-        if (isUnreachable(index, frames)) return@keysToMap emptySet<AbstractInsnNode>()
-        frames[index].getStack(0).insns
-    }
-}

compiler/backend/src/org/jetbrains/kotlin/codegen/inline/inlineCodegenUtils.kt

@@ -403,26 +403,6 @@ fun addInlineMarker(v: InstructionAdapter, isStartNotEnd: Boolean) {
     )
 }
 
-internal fun addReturnsUnitMarkerIfNecessary(v: InstructionAdapter, resolvedCall: ResolvedCall<*>) {
-    val wrapperDescriptor = resolvedCall.candidateDescriptor.safeAs<FunctionDescriptor>() ?: return
-    val unsubstitutedDescriptor = wrapperDescriptor.unwrapInitialDescriptorForSuspendFunction()
-
-    val typeSubstitutor = TypeSubstitutor.create(
-        unsubstitutedDescriptor.typeParameters
-            .withIndex()
-            .associateBy({ it.value.typeConstructor }) {
-                TypeProjectionImpl(resolvedCall.typeArguments[wrapperDescriptor.typeParameters[it.index]] ?: return)
-            }
-    )
-
-    val substitutedDescriptor = unsubstitutedDescriptor.substitute(typeSubstitutor) ?: return
-    val returnType = substitutedDescriptor.returnType ?: return
-
-    if (KotlinBuiltIns.isUnit(returnType)) {
-        addReturnsUnitMarker(v)
-    }
-}
-
 fun addSuspendMarker(v: InstructionAdapter, isStartNotEnd: Boolean) {
     v.emitInlineMarker(if (isStartNotEnd) INLINE_MARKER_BEFORE_SUSPEND_ID else INLINE_MARKER_AFTER_SUSPEND_ID)
 }
@@ -431,10 +411,6 @@ internal fun addFakeContinuationConstructorCallMarker(v: InstructionAdapter, isS
     v.emitInlineMarker(if (isStartNotEnd) INLINE_MARKER_BEFORE_FAKE_CONTINUATION_CONSTRUCTOR_CALL else INLINE_MARKER_AFTER_FAKE_CONTINUATION_CONSTRUCTOR_CALL)
 }
 
-private fun addReturnsUnitMarker(v: InstructionAdapter) {
-    v.emitInlineMarker(INLINE_MARKER_RETURNS_UNIT)
-}
-
 /* There are contexts when the continuation does not yet exist, for example, in inline lambdas, which are going to
  * be inlined into suspendable functions.
  * In such cases we just generate the marker which is going to be replaced with real continuation on generating state machine.

compiler/testData/codegen/box/coroutines/tailCallOptimizations/crossinline.txt

@@ -17,12 +17,7 @@ public final class CrossinlineKt$box$1$filter$$inlined$source$1$1 {
 }
 
 @kotlin.Metadata
-@kotlin.coroutines.jvm.internal.DebugMetadata
 public final class CrossinlineKt$box$1$filter$$inlined$source$1$lambda$1$1 {
-    field L$0: java.lang.Object
-    field L$1: java.lang.Object
-    field L$2: java.lang.Object
-    field L$3: java.lang.Object
     field label: int
     synthetic field result: java.lang.Object
     synthetic final field this$0: CrossinlineKt$box$1$filter$$inlined$source$1$lambda$1
@@ -102,28 +97,11 @@ public final class CrossinlineKt$box$1$invokeSuspend$$inlined$filter$1$1 {
     public final @org.jetbrains.annotations.Nullable method invokeSuspend(@org.jetbrains.annotations.NotNull p0: java.lang.Object): java.lang.Object
 }
 
-@kotlin.Metadata
-@kotlin.coroutines.jvm.internal.DebugMetadata
-public final class CrossinlineKt$box$1$invokeSuspend$$inlined$filter$1$2$1 {
-    field L$0: java.lang.Object
-    field L$1: java.lang.Object
-    field L$2: java.lang.Object
-    field L$3: java.lang.Object
-    field label: int
-    synthetic field result: java.lang.Object
-    synthetic final field this$0: CrossinlineKt$box$1$invokeSuspend$$inlined$filter$1$2
-    inner class CrossinlineKt$box$1$invokeSuspend$$inlined$filter$1$2
-    inner class CrossinlineKt$box$1$invokeSuspend$$inlined$filter$1$2$1
-    public method <init>(p0: CrossinlineKt$box$1$invokeSuspend$$inlined$filter$1$2, p1: kotlin.coroutines.Continuation): void
-    public final @org.jetbrains.annotations.Nullable method invokeSuspend(@org.jetbrains.annotations.NotNull p0: java.lang.Object): java.lang.Object
-}
-
 @kotlin.Metadata
 public final class CrossinlineKt$box$1$invokeSuspend$$inlined$filter$1$2 {
     synthetic final field $this_source$inlined: Sink
     synthetic final field this$0: CrossinlineKt$box$1$invokeSuspend$$inlined$filter$1
     inner class CrossinlineKt$box$1$invokeSuspend$$inlined$filter$1$2
-    inner class CrossinlineKt$box$1$invokeSuspend$$inlined$filter$1$2$1
     public method <init>(p0: Sink, p1: CrossinlineKt$box$1$invokeSuspend$$inlined$filter$1): void
     public method close(@org.jetbrains.annotations.Nullable p0: java.lang.Throwable): void
     public @org.jetbrains.annotations.Nullable method send(p0: java.lang.Object, @org.jetbrains.annotations.NotNull p1: kotlin.coroutines.Continuation): java.lang.Object
@@ -205,12 +183,7 @@ public final class CrossinlineKt$filter$$inlined$source$1$1 {
 }
 
 @kotlin.Metadata
-@kotlin.coroutines.jvm.internal.DebugMetadata
 public final class CrossinlineKt$filter$$inlined$source$1$lambda$1$1 {
-    field L$0: java.lang.Object
-    field L$1: java.lang.Object
-    field L$2: java.lang.Object
-    field L$3: java.lang.Object
     field label: int
     synthetic field result: java.lang.Object
     synthetic final field this$0: CrossinlineKt$filter$$inlined$source$1$lambda$1

compiler/testData/codegen/box/coroutines/tailCallOptimizations/crossinline_1_2.txt

@@ -17,29 +17,11 @@ public final class CrossinlineKt$box$1$doResume$$inlined$filter$1$1 {
     synthetic final method setLabel(p0: int): void
 }
 
-@kotlin.Metadata
-public final class CrossinlineKt$box$1$doResume$$inlined$filter$1$2$1 {
-    field L$0: java.lang.Object
-    field L$1: java.lang.Object
-    field L$2: java.lang.Object
-    field L$3: java.lang.Object
-    synthetic field data: java.lang.Object
-    synthetic field exception: java.lang.Throwable
-    synthetic final field this$0: CrossinlineKt$box$1$doResume$$inlined$filter$1$2
-    inner class CrossinlineKt$box$1$doResume$$inlined$filter$1$2
-    inner class CrossinlineKt$box$1$doResume$$inlined$filter$1$2$1
-    public method <init>(p0: CrossinlineKt$box$1$doResume$$inlined$filter$1$2, p1: kotlin.coroutines.experimental.Continuation): void
-    public final @org.jetbrains.annotations.Nullable method doResume(@org.jetbrains.annotations.Nullable p0: java.lang.Object, @org.jetbrains.annotations.Nullable p1: java.lang.Throwable): java.lang.Object
-    synthetic final method getLabel(): int
-    synthetic final method setLabel(p0: int): void
-}
-
 @kotlin.Metadata
 public final class CrossinlineKt$box$1$doResume$$inlined$filter$1$2 {
     synthetic final field $this_source$inlined: Sink
     synthetic final field this$0: CrossinlineKt$box$1$doResume$$inlined$filter$1
     inner class CrossinlineKt$box$1$doResume$$inlined$filter$1$2
-    inner class CrossinlineKt$box$1$doResume$$inlined$filter$1$2$1
     public method <init>(p0: Sink, p1: CrossinlineKt$box$1$doResume$$inlined$filter$1): void
     public method close(@org.jetbrains.annotations.Nullable p0: java.lang.Throwable): void
     public @org.jetbrains.annotations.Nullable method send(p0: java.lang.Object, @org.jetbrains.annotations.NotNull p1: kotlin.coroutines.experimental.Continuation): java.lang.Object
@@ -84,10 +66,6 @@ public final class CrossinlineKt$box$1$filter$$inlined$source$1$1 {
 
 @kotlin.Metadata
 public final class CrossinlineKt$box$1$filter$$inlined$source$1$lambda$1$1 {
-    field L$0: java.lang.Object
-    field L$1: java.lang.Object
-    field L$2: java.lang.Object
-    field L$3: java.lang.Object
     synthetic field data: java.lang.Object
     synthetic field exception: java.lang.Throwable
     synthetic final field this$0: CrossinlineKt$box$1$filter$$inlined$source$1$lambda$1
@@ -212,10 +190,6 @@ public final class CrossinlineKt$filter$$inlined$source$1$1 {
 
 @kotlin.Metadata
 public final class CrossinlineKt$filter$$inlined$source$1$lambda$1$1 {
-    field L$0: java.lang.Object
-    field L$1: java.lang.Object
-    field L$2: java.lang.Object
-    field L$3: java.lang.Object
     synthetic field data: java.lang.Object
     synthetic field exception: java.lang.Throwable
     synthetic final field this$0: CrossinlineKt$filter$$inlined$source$1$lambda$1

compiler/testData/codegen/box/coroutines/tailCallOptimizations/innerObjectRetransformation.txt

@@ -26,26 +26,9 @@ public final class flow/InnerObjectRetransformationKt$check$$inlined$collect$1 {
     public @org.jetbrains.annotations.Nullable method emit(@org.jetbrains.annotations.NotNull p0: java.lang.Object, @org.jetbrains.annotations.NotNull p1: kotlin.coroutines.Continuation): java.lang.Object
 }
 
-@kotlin.Metadata
-@kotlin.coroutines.jvm.internal.DebugMetadata
-public final class flow/InnerObjectRetransformationKt$check$$inlined$flow$1$1 {
-    field L$0: java.lang.Object
-    field L$1: java.lang.Object
-    field L$2: java.lang.Object
-    field L$3: java.lang.Object
-    field label: int
-    synthetic field result: java.lang.Object
-    synthetic final field this$0: flow.InnerObjectRetransformationKt$check$$inlined$flow$1
-    inner class flow/InnerObjectRetransformationKt$check$$inlined$flow$1
-    inner class flow/InnerObjectRetransformationKt$check$$inlined$flow$1$1
-    public method <init>(p0: flow.InnerObjectRetransformationKt$check$$inlined$flow$1, p1: kotlin.coroutines.Continuation): void
-    public final @org.jetbrains.annotations.Nullable method invokeSuspend(@org.jetbrains.annotations.NotNull p0: java.lang.Object): java.lang.Object
-}
-
 @kotlin.Metadata
 public final class flow/InnerObjectRetransformationKt$check$$inlined$flow$1 {
     inner class flow/InnerObjectRetransformationKt$check$$inlined$flow$1
-    inner class flow/InnerObjectRetransformationKt$check$$inlined$flow$1$1
     public method <init>(): void
     public @org.jetbrains.annotations.Nullable method collect(@org.jetbrains.annotations.NotNull p0: flow.FlowCollector, @org.jetbrains.annotations.NotNull p1: kotlin.coroutines.Continuation): java.lang.Object
 }
@@ -112,12 +95,7 @@ public final class flow/InnerObjectRetransformationKt$flow$1 {
 }
 
 @kotlin.Metadata
-@kotlin.coroutines.jvm.internal.DebugMetadata
 public final class flow/InnerObjectRetransformationKt$flowWith$$inlined$flow$1$1 {
-    field L$0: java.lang.Object
-    field L$1: java.lang.Object
-    field L$2: java.lang.Object
-    field L$3: java.lang.Object
     field label: int
     synthetic field result: java.lang.Object
     synthetic final field this$0: flow.InnerObjectRetransformationKt$flowWith$$inlined$flow$1
@@ -139,12 +117,7 @@ public final class flow/InnerObjectRetransformationKt$flowWith$$inlined$flow$1 {
 }
 
 @kotlin.Metadata
-@kotlin.coroutines.jvm.internal.DebugMetadata
 public final class flow/InnerObjectRetransformationKt$flowWith$$inlined$flow$2$1 {
-    field L$0: java.lang.Object
-    field L$1: java.lang.Object
-    field L$2: java.lang.Object
-    field L$3: java.lang.Object
     field label: int
     synthetic field result: java.lang.Object
     synthetic final field this$0: flow.InnerObjectRetransformationKt$flowWith$$inlined$flow$2

compiler/testData/codegen/box/coroutines/tailCallOptimizations/innerObjectRetransformation_1_2.txt

@@ -24,27 +24,9 @@ public final class flow/InnerObjectRetransformationKt$check$$inlined$collect$1 {
     public @org.jetbrains.annotations.Nullable method emit(@org.jetbrains.annotations.NotNull p0: java.lang.Object, @org.jetbrains.annotations.NotNull p1: kotlin.coroutines.experimental.Continuation): java.lang.Object
 }
 
-@kotlin.Metadata
-public final class flow/InnerObjectRetransformationKt$check$$inlined$flow$1$1 {
-    field L$0: java.lang.Object
-    field L$1: java.lang.Object
-    field L$2: java.lang.Object
-    field L$3: java.lang.Object
-    synthetic field data: java.lang.Object
-    synthetic field exception: java.lang.Throwable
-    synthetic final field this$0: flow.InnerObjectRetransformationKt$check$$inlined$flow$1
-    inner class flow/InnerObjectRetransformationKt$check$$inlined$flow$1
-    inner class flow/InnerObjectRetransformationKt$check$$inlined$flow$1$1
-    public method <init>(p0: flow.InnerObjectRetransformationKt$check$$inlined$flow$1, p1: kotlin.coroutines.experimental.Continuation): void
-    public final @org.jetbrains.annotations.Nullable method doResume(@org.jetbrains.annotations.Nullable p0: java.lang.Object, @org.jetbrains.annotations.Nullable p1: java.lang.Throwable): java.lang.Object
-    synthetic final method getLabel(): int
-    synthetic final method setLabel(p0: int): void
-}
-
 @kotlin.Metadata
 public final class flow/InnerObjectRetransformationKt$check$$inlined$flow$1 {
     inner class flow/InnerObjectRetransformationKt$check$$inlined$flow$1
-    inner class flow/InnerObjectRetransformationKt$check$$inlined$flow$1$1
     public method <init>(): void
     public @org.jetbrains.annotations.Nullable method collect(@org.jetbrains.annotations.NotNull p0: flow.FlowCollector, @org.jetbrains.annotations.NotNull p1: kotlin.coroutines.experimental.Continuation): java.lang.Object
 }
@@ -117,10 +99,6 @@ public final class flow/InnerObjectRetransformationKt$flow$1 {
 
 @kotlin.Metadata
 public final class flow/InnerObjectRetransformationKt$flowWith$$inlined$flow$1$1 {
-    field L$0: java.lang.Object
-    field L$1: java.lang.Object
-    field L$2: java.lang.Object
-    field L$3: java.lang.Object
     synthetic field data: java.lang.Object
     synthetic field exception: java.lang.Throwable
     synthetic final field this$0: flow.InnerObjectRetransformationKt$flowWith$$inlined$flow$1
@@ -145,10 +123,6 @@ public final class flow/InnerObjectRetransformationKt$flowWith$$inlined$flow$1 {
 
 @kotlin.Metadata
 public final class flow/InnerObjectRetransformationKt$flowWith$$inlined$flow$2$1 {
-    field L$0: java.lang.Object
-    field L$1: java.lang.Object
-    field L$2: java.lang.Object
-    field L$3: java.lang.Object
     synthetic field data: java.lang.Object
     synthetic field exception: java.lang.Throwable
     synthetic final field this$0: flow.InnerObjectRetransformationKt$flowWith$$inlined$flow$2

compiler/testData/codegen/box/coroutines/tailCallOptimizations/tailSuspendUnitFun.kt

@@ -17,9 +17,9 @@ suspend fun lambdaAsParameterNotTailCall(c: suspend ()->Unit) { c(); c() }
 suspend fun lambdaAsParameterReturn(c: suspend ()->Unit) { return c() }
 suspend fun returnsInt() = 42.also { TailCallOptimizationChecker.saveStackTrace() }
 // This should not be tail-call, since the caller should push Unit.INSTANCE on stack
-suspend fun callsIntNotTailCall() { returnsInt() }
+suspend fun callsIntTailCall() { returnsInt() }
 suspend fun multipleExitPoints(b: Boolean) { if (b) empty() else withoutReturn() }
-suspend fun multipleExitPointsNotTailCall(b: Boolean) { if (b) empty() else returnsInt() }
+suspend fun multipleExitPointsTailCall(b: Boolean) { if (b) empty() else returnsInt() }
 
 fun ordinary() = 1
 inline fun ordinaryInline() { ordinary() }
@@ -95,14 +95,14 @@ fun box(): String {
         lambdaAsParameterReturn { TailCallOptimizationChecker.saveStackTrace() }
         TailCallOptimizationChecker.checkNoStateMachineIn("lambdaAsParameterReturn")
 
-        callsIntNotTailCall()
-        TailCallOptimizationChecker.checkStateMachineIn("callsIntNotTailCall")
+        callsIntTailCall()
+        TailCallOptimizationChecker.checkNoStateMachineIn("callsIntTailCall")
 
         multipleExitPoints(false)
         TailCallOptimizationChecker.checkNoStateMachineIn("multipleExitPoints")
 
-        multipleExitPointsNotTailCall(false)
-        TailCallOptimizationChecker.checkStateMachineIn("multipleExitPointsNotTailCall")
+        multipleExitPointsTailCall(false)
+        TailCallOptimizationChecker.checkNoStateMachineIn("multipleExitPointsTailCall")
 
         multipleExitPointsWithOrdinaryInline(true)
         TailCallOptimizationChecker.checkNoStateMachineIn("multipleExitPointsWithOrdinaryInline")
@@ -120,7 +120,7 @@ fun box(): String {
         TailCallOptimizationChecker.checkNoStateMachineIn("useGenericInferType")
 
         useNullableUnit()
-        TailCallOptimizationChecker.checkStateMachineIn("useNullableUnit")
+        TailCallOptimizationChecker.checkNoStateMachineIn("useNullableUnit")
 
         useRunRunRunRunRun()
         TailCallOptimizationChecker.checkNoStateMachineIn("useRunRunRunRunRun")

compiler/testData/codegen/box/coroutines/tailCallOptimizations/unreachable.kt

@@ -40,8 +40,8 @@ suspend fun lambdaAsParameterReturn(c: suspend () -> Unit) {
 }
 
 suspend fun returnsInt() = 42.also { TailCallOptimizationChecker.saveStackTrace() }
-// This should not be tail-call, since the caller should push Unit.INSTANCE on stack
-suspend fun callsIntNotTailCall() {
+
+suspend fun callsIntTailCall() {
     returnsInt()
     return
     empty()
@@ -53,7 +53,7 @@ suspend fun multipleExitPoints(b: Boolean) {
     empty()
 }
 
-suspend fun multipleExitPointsNotTailCall(b: Boolean) {
+suspend fun multipleExitPointsTailCall(b: Boolean) {
     if (b) empty() else returnsInt()
     return
     empty()
@@ -153,14 +153,14 @@ fun box(): String {
         lambdaAsParameterReturn { TailCallOptimizationChecker.saveStackTrace() }
         TailCallOptimizationChecker.checkNoStateMachineIn("lambdaAsParameterReturn")
 
-        callsIntNotTailCall()
-        TailCallOptimizationChecker.checkStateMachineIn("callsIntNotTailCall")
+        callsIntTailCall()
+        TailCallOptimizationChecker.checkNoStateMachineIn("callsIntTailCall")
 
         multipleExitPoints(false)
         TailCallOptimizationChecker.checkNoStateMachineIn("multipleExitPoints")
 
-        multipleExitPointsNotTailCall(false)
-        TailCallOptimizationChecker.checkStateMachineIn("multipleExitPointsNotTailCall")
+        multipleExitPointsTailCall(false)
+        TailCallOptimizationChecker.checkNoStateMachineIn("multipleExitPointsTailCall")
 
         multipleExitPointsWithOrdinaryInline(true)
         TailCallOptimizationChecker.checkNoStateMachineIn("multipleExitPointsWithOrdinaryInline")
@@ -178,7 +178,7 @@ fun box(): String {
         TailCallOptimizationChecker.checkNoStateMachineIn("useGenericInferType")
 
         useNullableUnit()
-        TailCallOptimizationChecker.checkStateMachineIn("useNullableUnit")
+        TailCallOptimizationChecker.checkNoStateMachineIn("useNullableUnit")
 
         useRunRunRunRunRun()
         TailCallOptimizationChecker.checkNoStateMachineIn("useRunRunRunRunRun")

compiler/testData/codegen/boxInline/suspend/inlineSuspendContinuation.kt

@@ -101,6 +101,6 @@ fun box() : String {
             checkContinuation(savedContinuation!!)
         }
     }
-    if (!continuationChanged) return "FAIL 5"
+    if (continuationChanged) return "FAIL 5"
     return "OK"
 }

compiler/tests-common/tests/org/jetbrains/kotlin/coroutineTestUtil.kt

@@ -96,11 +96,11 @@ fun createTextForHelpers(isReleaseCoroutines: Boolean, checkStateMachine: Boolea
 
         fun check(numberOfSuspensions: Int) {
             for (i in 1..numberOfSuspensions) {
-                if (counter != i) error("Wrong state-machine generated: suspendHere called should be called exactly once in one state. Expected " + i + ", got " + counter)
+                if (counter != i) error("Wrong state-machine generated: suspendHere should be called exactly once in one state. Expected " + i + ", got " + counter)
                 proceed()
             }
             if (counter != numberOfSuspensions)
-                error("Wrong state-machine generated: suspendHere called should be called exactly once in one state. Expected " + numberOfSuspensions + ", got " + counter)
+                error("Wrong state-machine generated: wrong number of overall suspensions. Expected " + numberOfSuspensions + ", got " + counter)
             if (finished) error("Wrong state-machine generated: it is finished early")
             proceed()
             if (!finished) error("Wrong state-machine generated: it is not finished yet")