JVM_IR: Support continuation crossing inline lambda boundaries

I.E. generate fake continuation markers.

compiler/backend/src/org/jetbrains/kotlin/codegen/inline/inlineCodegenUtils.kt

@@ -416,7 +416,7 @@ internal fun addFakeContinuationConstructorCallMarker(v: InstructionAdapter, isS
  * In such cases we just generate the marker which is going to be replaced with real continuation on generating state machine.
  * See [CoroutineTransformerMethodVisitor] for more info.
  */
-internal fun addFakeContinuationMarker(v: InstructionAdapter) {
+fun addFakeContinuationMarker(v: InstructionAdapter) {
     v.emitInlineMarker(INLINE_MARKER_FAKE_CONTINUATION)
     v.aconst(null)
 }

compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/JvmBackendContext.kt

@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.backend.common.ir.Ir
 import org.jetbrains.kotlin.backend.common.phaser.PhaseConfig
 import org.jetbrains.kotlin.backend.jvm.codegen.IrTypeMapper
 import org.jetbrains.kotlin.backend.jvm.codegen.MethodSignatureMapper
+import org.jetbrains.kotlin.backend.jvm.codegen.createFakeContinuation
 import org.jetbrains.kotlin.backend.jvm.descriptors.JvmDeclarationFactory
 import org.jetbrains.kotlin.backend.jvm.descriptors.JvmSharedVariablesManager
 import org.jetbrains.kotlin.backend.jvm.intrinsics.IrIntrinsicMethods
@@ -24,6 +25,7 @@ import org.jetbrains.kotlin.incremental.components.NoLookupLocation
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
+import org.jetbrains.kotlin.ir.expressions.IrExpression
 import org.jetbrains.kotlin.ir.symbols.IrClassSymbol
 import org.jetbrains.kotlin.ir.symbols.IrFunctionSymbol
 import org.jetbrains.kotlin.ir.symbols.IrLocalDelegatedPropertySymbol
@@ -84,6 +86,7 @@ class JvmBackendContext(
     val suspendLambdaToOriginalFunctionMap = mutableMapOf<IrClass, IrFunction>()
     val continuationClassBuilders = mutableMapOf<IrClass, ClassBuilder>()
     val suspendFunctionViews = mutableMapOf<IrFunction, IrFunction>()
+    val FAKE_CONTINUATION: IrExpression = createFakeContinuation(this)
 
     val staticDefaultStubs = mutableMapOf<IrFunctionSymbol, IrFunction>()
 

compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/codegen/CoroutineCodegen.kt

@@ -27,6 +27,7 @@ import org.jetbrains.kotlin.ir.expressions.IrExpression
 import org.jetbrains.kotlin.ir.expressions.IrGetValue
 import org.jetbrains.kotlin.ir.expressions.copyTypeArgumentsFrom
 import org.jetbrains.kotlin.ir.expressions.impl.IrCallImpl
+import org.jetbrains.kotlin.ir.expressions.impl.IrErrorExpressionImpl
 import org.jetbrains.kotlin.ir.expressions.impl.IrGetValueImpl
 import org.jetbrains.kotlin.ir.symbols.impl.IrSimpleFunctionSymbolImpl
 import org.jetbrains.kotlin.ir.types.createType
@@ -146,13 +147,17 @@ private fun IrFunction.suspendFunctionView(context: JvmBackendContext): IrFuncti
 
             override fun visitCall(expression: IrCall): IrExpression {
                 if (!expression.isSuspend) return super.visitCall(expression)
-                return super.visitCall(expression.createSuspendFunctionCallViewIfNeeded(context, it))
+                return super.visitCall(expression.createSuspendFunctionCallViewIfNeeded(context, it, callerIsInlineLambda = false))
             }
         })
     }
 }
 
-internal fun IrCall.createSuspendFunctionCallViewIfNeeded(context: JvmBackendContext, caller: IrFunction): IrCall {
+internal fun IrCall.createSuspendFunctionCallViewIfNeeded(
+    context: JvmBackendContext,
+    caller: IrFunction,
+    callerIsInlineLambda: Boolean
+): IrCall {
     if (!isSuspend) return this
     val view = (symbol.owner as IrSimpleFunction).getOrCreateSuspendFunctionViewIfNeeded(context)
     if (view == symbol.owner) return this
@@ -164,9 +169,19 @@ internal fun IrCall.createSuspendFunctionCallViewIfNeeded(context: JvmBackendCon
             it.putValueArgument(i, getValueArgument(i))
         }
         val continuationParameter =
-            if (caller.isInvokeSuspendOfLambda(context) || caller.isInvokeSuspendOfContinuation(context)) caller.dispatchReceiverParameter!!
+            when {
-            else caller.valueParameters.last()
+                caller.isInvokeSuspendOfLambda(context) || caller.isInvokeSuspendOfContinuation(context) ->
-        val continuation = IrGetValueImpl(UNDEFINED_OFFSET, UNDEFINED_OFFSET, continuationParameter.symbol)
+                    IrGetValueImpl(UNDEFINED_OFFSET, UNDEFINED_OFFSET, caller.dispatchReceiverParameter!!.symbol)
-        it.putValueArgument(valueArgumentsCount, continuation)
+                callerIsInlineLambda -> context.FAKE_CONTINUATION
+                else -> IrGetValueImpl(UNDEFINED_OFFSET, UNDEFINED_OFFSET, caller.valueParameters.last().symbol)
+            }
+        it.putValueArgument(valueArgumentsCount, continuationParameter)
     }
-}
+}
+
+internal fun createFakeContinuation(context: JvmBackendContext): IrExpression = IrErrorExpressionImpl(
+    UNDEFINED_OFFSET,
+    UNDEFINED_OFFSET,
+    context.ir.symbols.continuationClass.createType(true, listOf(makeTypeProjection(context.irBuiltIns.anyNType, Variance.INVARIANT))),
+    "FAKE_CONTINUATION"
+)

compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/codegen/ExpressionCodegen.kt

@@ -158,7 +158,11 @@ class ExpressionCodegen(
 
     // TODO remove
     fun gen(expression: IrExpression, type: Type, irType: IrType, data: BlockInfo): StackValue {
-        expression.accept(this, data).coerce(type, irType).materialize()
+        if (expression === context.FAKE_CONTINUATION) {
+            addFakeContinuationMarker(mv)
+        } else {
+            expression.accept(this, data).coerce(type, irType).materialize()
+        }
         return StackValue.onStack(type, irType.toKotlinType())
     }
 
@@ -299,7 +303,7 @@ class ExpressionCodegen(
         visitStatementContainer(expression, data).coerce(expression.type)
 
     override fun visitCall(expression: IrCall, data: BlockInfo): PromisedValue {
-        return visitFunctionAccess(expression.createSuspendFunctionCallViewIfNeeded(context, irFunction), data)
+        return visitFunctionAccess(expression.createSuspendFunctionCallViewIfNeeded(context, irFunction, isInlineLambda), data)
     }
 
     override fun visitFunctionAccess(expression: IrFunctionAccessExpression, data: BlockInfo): PromisedValue {
@@ -666,10 +670,12 @@ class ExpressionCodegen(
             }
             arity == 2 && expression.arguments[0].type.isStringClassType() -> {
                 // Call the stringPlus intrinsic
-                for (argument in expression.arguments) {
+                for ((index, argument) in expression.arguments.withIndex()) {
                     val result = argument.accept(this, data).boxInlineClasses(argument.type).materialized
                     if (result.type.sort != Type.OBJECT) {
                         result.genToString(mv)
+                    } else if (index == 0) {
+                        result.coerce(context.irBuiltIns.stringType).materialize()
                     }
                 }
                 mv.invokestatic(

compiler/testData/codegen/box/coroutines/controlFlow/forWithStep.kt

@@ -1,5 +1,4 @@
 // KJS_WITH_FULL_RUNTIME
-// IGNORE_BACKEND: JVM_IR
 // WITH_RUNTIME
 // WITH_COROUTINES
 // COMMON_COROUTINES_TEST

compiler/testData/codegen/box/coroutines/inlinedTryCatchFinally.kt

@@ -1,5 +1,4 @@
 // KJS_WITH_FULL_RUNTIME
-// IGNORE_BACKEND: JVM_IR
 // WITH_RUNTIME
 // WITH_COROUTINES
 // COMMON_COROUTINES_TEST

compiler/testData/codegen/box/coroutines/kt15016.kt

@@ -1,4 +1,3 @@
-// IGNORE_BACKEND: JVM_IR
 // WITH_RUNTIME
 // WITH_COROUTINES
 // COMMON_COROUTINES_TEST

compiler/testData/codegen/box/coroutines/suspendFromInlineLambda.kt

@@ -1,4 +1,3 @@
-// IGNORE_BACKEND: JVM_IR
 // WITH_RUNTIME
 // WITH_COROUTINES
 // COMMON_COROUTINES_TEST

compiler/testData/codegen/box/coroutines/tailCallOptimizations/also.kt

@@ -1,4 +1,3 @@
-// IGNORE_BACKEND: JVM_IR
 // TARGET_BACKEND: JVM
 // FULL_JDK
 // WITH_RUNTIME

compiler/testData/codegen/box/coroutines/tailCallOptimizations/inlineWithoutStateMachine_ir.kt

@@ -0,0 +1,42 @@
+// WITH_RUNTIME
+// WITH_COROUTINES
+// IGNORE_BACKEND: JVM
+// TARGET_BACKEND: JVM
+import helpers.*
+// CHECK_BYTECODE_LISTING
+// CHECK_NEW_COUNT: function=suspendHere count=0
+// CHECK_NEW_COUNT: function=complexSuspend count=0
+import kotlin.coroutines.*
+import kotlin.coroutines.intrinsics.*
+
+inline suspend fun suspendThere(v: String): String = suspendCoroutineUninterceptedOrReturn { x ->
+    x.resume(v)
+    COROUTINE_SUSPENDED
+}
+
+// TODO: Somehow we still generate continuations for tail-call function, but we don't use them.
+suspend fun suspendHere(): String = suspendThere("O")
+
+// There is a kind of redundant state machine generated for complexSuspend:
+// it's basically has the only suspend call just before return, but there is
+// a redundant CHECKCAST String in the run's lambda, so we have to insert the state machine.
+// TODO: Think of avoiding such redundant casts
+suspend fun complexSuspend(): String {
+    return run {
+        suspendThere("K")
+    }
+}
+
+fun builder(c: suspend () -> Unit) {
+    c.startCoroutine(EmptyContinuation)
+}
+
+fun box(): String {
+    var result = ""
+
+    builder {
+        result = suspendHere() + complexSuspend()
+    }
+
+    return result
+}

compiler/testData/codegen/box/coroutines/tailCallOptimizations/inlineWithoutStateMachine_ir.txt

@@ -0,0 +1,48 @@
+@kotlin.Metadata
+@kotlin.coroutines.jvm.internal.DebugMetadata
+final class InlineWithoutStateMachine_irKt$$complexSuspend$Continuation {
+    field L$0: java.lang.Object
+    field L$1: java.lang.Object
+    field L$2: java.lang.Object
+    field label: int
+    @org.jetbrains.annotations.NotNull field result: java.lang.Object
+    public method <init>(@org.jetbrains.annotations.Nullable p0: kotlin.coroutines.Continuation): void
+    public final @org.jetbrains.annotations.Nullable method invokeSuspend(@org.jetbrains.annotations.NotNull p0: java.lang.Object): java.lang.Object
+}
+
+@kotlin.Metadata
+@kotlin.coroutines.jvm.internal.DebugMetadata
+final class InlineWithoutStateMachine_irKt$$suspendHere$Continuation {
+    field L$0: java.lang.Object
+    field L$1: java.lang.Object
+    field L$2: java.lang.Object
+    field label: int
+    @org.jetbrains.annotations.NotNull field result: java.lang.Object
+    public method <init>(@org.jetbrains.annotations.Nullable p0: kotlin.coroutines.Continuation): void
+    public final @org.jetbrains.annotations.Nullable method invokeSuspend(@org.jetbrains.annotations.NotNull p0: java.lang.Object): java.lang.Object
+}
+
+@kotlin.coroutines.jvm.internal.DebugMetadata
+@kotlin.Metadata
+final class InlineWithoutStateMachine_irKt$box$1 {
+    private field $result: kotlin.jvm.internal.Ref$ObjectRef
+    field L$0: java.lang.Object
+    field L$1: java.lang.Object
+    private field label: int
+    inner class InlineWithoutStateMachine_irKt$box$1
+    public method <init>(@org.jetbrains.annotations.NotNull p0: kotlin.jvm.internal.Ref$ObjectRef): void
+    public method <init>(@org.jetbrains.annotations.NotNull p0: kotlin.jvm.internal.Ref$ObjectRef, @org.jetbrains.annotations.Nullable p1: kotlin.coroutines.Continuation): void
+    public final @org.jetbrains.annotations.NotNull method create(@org.jetbrains.annotations.NotNull p0: kotlin.coroutines.Continuation): kotlin.coroutines.Continuation
+    public final method invoke(p0: java.lang.Object): java.lang.Object
+    public final @org.jetbrains.annotations.Nullable method invokeSuspend(@org.jetbrains.annotations.NotNull p0: java.lang.Object): java.lang.Object
+}
+
+@kotlin.Metadata
+public final class InlineWithoutStateMachine_irKt {
+    inner class InlineWithoutStateMachine_irKt$box$1
+    public final static @org.jetbrains.annotations.NotNull method box(): java.lang.String
+    public final static method builder(@org.jetbrains.annotations.NotNull p0: kotlin.jvm.functions.Function1): void
+    public final static @org.jetbrains.annotations.Nullable method complexSuspend(@org.jetbrains.annotations.NotNull p0: kotlin.coroutines.Continuation): java.lang.Object
+    public final static @org.jetbrains.annotations.Nullable method suspendHere(@org.jetbrains.annotations.NotNull p0: kotlin.coroutines.Continuation): java.lang.Object
+    public final static @org.jetbrains.annotations.Nullable method suspendThere(@org.jetbrains.annotations.NotNull p0: java.lang.String, @org.jetbrains.annotations.NotNull p1: kotlin.coroutines.Continuation): java.lang.Object
+}

compiler/testData/codegen/box/coroutines/tailCallOptimizations/tryCatchTailCall.kt

@@ -1,4 +1,3 @@
-// IGNORE_BACKEND: JVM_IR
 // TARGET_BACKEND: JVM
 // FULL_JDK
 // WITH_RUNTIME

compiler/tests/org/jetbrains/kotlin/codegen/BlackBoxCodegenTestGenerated.java

@@ -8339,6 +8339,11 @@ public class BlackBoxCodegenTestGenerated extends AbstractBlackBoxCodegenTest {
                 runTestWithPackageReplacement("compiler/testData/codegen/box/coroutines/tailCallOptimizations/inlineWithoutStateMachine.kt", "kotlin.coroutines");
             }
 
+            @TestMetadata("inlineWithoutStateMachine_ir.kt")
+            public void testInlineWithoutStateMachine_ir() throws Exception {
+                runTest("compiler/testData/codegen/box/coroutines/tailCallOptimizations/inlineWithoutStateMachine_ir.kt");
+            }
+
             @TestMetadata("innerObjectRetransformation.kt")
             public void testInnerObjectRetransformation_1_2() throws Exception {
                 runTestWithPackageReplacement("compiler/testData/codegen/box/coroutines/tailCallOptimizations/innerObjectRetransformation.kt", "kotlin.coroutines.experimental");

compiler/tests/org/jetbrains/kotlin/codegen/LightAnalysisModeTestGenerated.java

@@ -8297,6 +8297,11 @@ public class LightAnalysisModeTestGenerated extends AbstractLightAnalysisModeTes
         @TestDataPath("$PROJECT_ROOT")
         @RunWith(JUnit3RunnerWithInners.class)
         public static class TailCallOptimizations extends AbstractLightAnalysisModeTest {
+            @TestMetadata("inlineWithoutStateMachine_ir.kt")
+            public void ignoreInlineWithoutStateMachine_ir() throws Exception {
+                runTest("compiler/testData/codegen/box/coroutines/tailCallOptimizations/inlineWithoutStateMachine_ir.kt");
+            }
+
             private void runTest(String testDataFilePath) throws Exception {
                 KotlinTestUtils.runTest(this::doTest, TargetBackend.JVM, testDataFilePath);
             }

compiler/tests/org/jetbrains/kotlin/codegen/ir/IrBlackBoxCodegenTestGenerated.java

@@ -7289,6 +7289,11 @@ public class IrBlackBoxCodegenTestGenerated extends AbstractIrBlackBoxCodegenTes
                 runTestWithPackageReplacement("compiler/testData/codegen/box/coroutines/tailCallOptimizations/inlineWithoutStateMachine.kt", "kotlin.coroutines");
             }
 
+            @TestMetadata("inlineWithoutStateMachine_ir.kt")
+            public void testInlineWithoutStateMachine_ir() throws Exception {
+                runTest("compiler/testData/codegen/box/coroutines/tailCallOptimizations/inlineWithoutStateMachine_ir.kt");
+            }
+
             @TestMetadata("innerObjectRetransformation.kt")
             public void testInnerObjectRetransformation_1_3() throws Exception {
                 runTestWithPackageReplacement("compiler/testData/codegen/box/coroutines/tailCallOptimizations/innerObjectRetransformation.kt", "kotlin.coroutines");

js/js.tests/test/org/jetbrains/kotlin/js/test/ir/semantics/IrJsCodegenBoxTestGenerated.java

@@ -6219,6 +6219,11 @@ public class IrJsCodegenBoxTestGenerated extends AbstractIrJsCodegenBoxTest {
                 runTestWithPackageReplacement("compiler/testData/codegen/box/coroutines/tailCallOptimizations/inlineWithoutStateMachine.kt", "kotlin.coroutines");
             }
 
+            @TestMetadata("inlineWithoutStateMachine_ir.kt")
+            public void testInlineWithoutStateMachine_ir() throws Exception {
+                runTest("compiler/testData/codegen/box/coroutines/tailCallOptimizations/inlineWithoutStateMachine_ir.kt");
+            }
+
             @TestMetadata("innerObjectRetransformation.kt")
             public void testInnerObjectRetransformation_1_3() throws Exception {
                 runTestWithPackageReplacement("compiler/testData/codegen/box/coroutines/tailCallOptimizations/innerObjectRetransformation.kt", "kotlin.coroutines");

js/js.tests/test/org/jetbrains/kotlin/js/test/semantics/JsCodegenBoxTestGenerated.java

@@ -7239,6 +7239,11 @@ public class JsCodegenBoxTestGenerated extends AbstractJsCodegenBoxTest {
                 runTestWithPackageReplacement("compiler/testData/codegen/box/coroutines/tailCallOptimizations/inlineWithoutStateMachine.kt", "kotlin.coroutines");
             }
 
+            @TestMetadata("inlineWithoutStateMachine_ir.kt")
+            public void testInlineWithoutStateMachine_ir() throws Exception {
+                runTest("compiler/testData/codegen/box/coroutines/tailCallOptimizations/inlineWithoutStateMachine_ir.kt");
+            }
+
             @TestMetadata("innerObjectRetransformation.kt")
             public void testInnerObjectRetransformation_1_2() throws Exception {
                 runTestWithPackageReplacement("compiler/testData/codegen/box/coroutines/tailCallOptimizations/innerObjectRetransformation.kt", "kotlin.coroutines.experimental");