Detect inline cycles faster

E.g. in the following code

    fun x() {}
    inline fun f() { x(); g() }
    inline fun g() { x(); f() }

the old implementation of inline cycle detection bailed out after
generating 3 calls of x() in each function, while the new one stops
after 2. In other words, code generation for a single function is no
longer reentered.

compiler/backend/src/org/jetbrains/kotlin/codegen/FunctionGenerationStrategy.java

@@ -81,7 +81,12 @@ public abstract class FunctionGenerationStrategy {
                 @NotNull MemberCodegen<?> parentCodegen
         ) {
             ExpressionCodegen codegen = new ExpressionCodegen(mv, frameMap, signature.getReturnType(), context, state, parentCodegen);
-            doGenerateBody(codegen, signature);
+            state.getGlobalInlineContext().enterDeclaration(context.getFunctionDescriptor());
+            try {
+                doGenerateBody(codegen, signature);
+            } finally {
+                state.getGlobalInlineContext().exitDeclaration();
+            }
         }
 
         @Override

compiler/backend/src/org/jetbrains/kotlin/codegen/InlineCycleReporter.kt

@@ -1,51 +0,0 @@
-/*
- * Copyright 2010-2015 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.codegen
-
-import org.jetbrains.kotlin.codegen.inline.InlineCall
-import org.jetbrains.kotlin.diagnostics.DiagnosticSink
-import org.jetbrains.kotlin.diagnostics.Errors
-
-class InlineCycleReporter(private val diagnostics: DiagnosticSink) {
-
-    private val processingFunctions = linkedMapOf<Any, InlineCall>()
-
-    fun enterIntoInlining(call: InlineCall?): Boolean {
-        // null call for default method inlining
-        val id = call?.id
-        if (id != null) {
-            if (processingFunctions.contains(id)) {
-                val cycle = processingFunctions.values.dropWhile { it.id != id }
-                for (cycleCall in cycle) {
-                    val callPsiElement = cycleCall.callElement
-                    if (callPsiElement != null) {
-                        diagnostics.report(Errors.INLINE_CALL_CYCLE.on(callPsiElement, cycleCall.calleeDescriptor))
-                    }
-                }
-                return false
-            }
-            processingFunctions[id] = call
-        }
-        return true
-    }
-
-    fun exitFromInliningOf(call: InlineCall?) {
-        if (call != null) {
-            processingFunctions.remove(call.id)
-        }
-    }
-}

compiler/backend/src/org/jetbrains/kotlin/codegen/inline/GlobalInlineContext.kt

@@ -5,24 +5,48 @@
 
 package org.jetbrains.kotlin.codegen.inline
 
-import org.jetbrains.kotlin.codegen.InlineCycleReporter
+import com.intellij.psi.PsiElement
+import org.jetbrains.kotlin.descriptors.CallableDescriptor
 import org.jetbrains.kotlin.diagnostics.DiagnosticSink
-import org.jetbrains.kotlin.resolve.calls.model.ResolvedCall
+import org.jetbrains.kotlin.diagnostics.Errors
 import java.util.*
 
-class GlobalInlineContext(diagnostics: DiagnosticSink) {
-
-    private val inlineCycleReporter: InlineCycleReporter = InlineCycleReporter(diagnostics)
+class GlobalInlineContext(private val diagnostics: DiagnosticSink) {
+    // Ordered set of declarations and inline calls being generated right now.
+    // No call in it should point to a declaration that's before it in the stack.
+    private val inlineCallsAndDeclarations = LinkedList<Any? /* CallableDescriptor | PsiElement? */>()
+    private val inlineDeclarationSet = mutableSetOf<CallableDescriptor>()
 
     private val typesUsedInInlineFunctions = LinkedList<MutableSet<String>>()
 
-    fun enterIntoInlining(call: InlineCall?) =
-        inlineCycleReporter.enterIntoInlining(call).also {
-            if (it) typesUsedInInlineFunctions.push(hashSetOf())
-        }
+    fun enterDeclaration(descriptor: CallableDescriptor) {
+        assert(descriptor.original !in inlineDeclarationSet) { "entered inlining cycle on $descriptor" }
+        inlineDeclarationSet.add(descriptor.original)
+        inlineCallsAndDeclarations.add(descriptor.original)
+    }
 
-    fun exitFromInliningOf(call: InlineCall?) {
-        inlineCycleReporter.exitFromInliningOf(call)
+    fun exitDeclaration() {
+        inlineDeclarationSet.remove(inlineCallsAndDeclarations.removeLast())
+    }
+
+    fun enterIntoInlining(callee: CallableDescriptor?, element: PsiElement?): Boolean {
+        if (callee != null && callee.original in inlineDeclarationSet) {
+            element?.let { diagnostics.report(Errors.INLINE_CALL_CYCLE.on(it, callee.original)) }
+            for ((call, callTarget) in inlineCallsAndDeclarations.dropWhile { it != callee.original }.zipWithNext()) {
+                // Every call element should be followed by the callee's descriptor.
+                if (call is PsiElement && callTarget is CallableDescriptor) {
+                    diagnostics.report(Errors.INLINE_CALL_CYCLE.on(call, callTarget))
+                }
+            }
+            return false
+        }
+        inlineCallsAndDeclarations.add(element)
+        typesUsedInInlineFunctions.push(hashSetOf())
+        return true
+    }
+
+    fun exitFromInlining() {
+        inlineCallsAndDeclarations.removeLast()
         val pop = typesUsedInInlineFunctions.pop()
         typesUsedInInlineFunctions.peek()?.addAll(pop)
     }

compiler/backend/src/org/jetbrains/kotlin/codegen/inline/InlineCall.kt

@@ -1,32 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.codegen.inline
-
-import com.intellij.psi.PsiElement
-import org.jetbrains.kotlin.descriptors.CallableDescriptor
-import org.jetbrains.kotlin.resolve.calls.model.ResolvedCall
-
-interface InlineCall {
-    val id: Any
-    val calleeDescriptor: CallableDescriptor
-    val callElement: PsiElement?
-}
-
-class InlineCallImpl(
-    override val calleeDescriptor: CallableDescriptor,
-    override val callElement: PsiElement
-) : InlineCall {
-
-    override val id: Any
-        get() = callElement
-
-    companion object {
-        fun of(resolvedCall: ResolvedCall<*>?) =
-            resolvedCall?.run {
-                InlineCallImpl(resultingDescriptor.original, call.callElement)
-            }
-    }
-}

compiler/backend/src/org/jetbrains/kotlin/codegen/inline/PsiInlineCodegen.kt

@@ -63,8 +63,7 @@ class PsiInlineCodegen(
         callDefault: Boolean,
         codegen: ExpressionCodegen
     ) {
-        val inlineCall = InlineCallImpl.of(resolvedCall)
-        if (!state.globalInlineContext.enterIntoInlining(inlineCall)) {
+        if (!state.globalInlineContext.enterIntoInlining(resolvedCall?.resultingDescriptor, resolvedCall?.call?.callElement)) {
             generateStub(resolvedCall, codegen)
             return
         }
@@ -72,7 +71,7 @@ class PsiInlineCodegen(
             val registerLineNumber = registerLineNumberAfterwards(resolvedCall)
             performInline(resolvedCall?.typeArguments?.keys?.toList(), callDefault, callDefault, codegen.typeSystem, registerLineNumber)
         } finally {
-            state.globalInlineContext.exitFromInliningOf(inlineCall)
+            state.globalInlineContext.exitFromInlining()
         }
     }
 

compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/codegen/FunctionCodegen.kt

@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.backend.common.lower.BOUND_RECEIVER_PARAMETER
 import org.jetbrains.kotlin.backend.common.lower.BOUND_VALUE_PARAMETER
 import org.jetbrains.kotlin.backend.jvm.JvmBackendContext
 import org.jetbrains.kotlin.backend.jvm.JvmLoweredDeclarationOrigin
+import org.jetbrains.kotlin.backend.jvm.lower.suspendFunctionOriginal
 import org.jetbrains.kotlin.codegen.AsmUtil
 import org.jetbrains.kotlin.codegen.mangleNameIfNeeded
 import org.jetbrains.kotlin.codegen.state.GenerationState
@@ -110,7 +111,12 @@ open class FunctionCodegen(
                     irFunction, classCodegen, methodVisitor, flags, signature, getContinuation, psiElement()
                 )
             }
-            ExpressionCodegen(irFunction, signature, frameMap, InstructionAdapter(methodVisitor), classCodegen, inlinedInto).generate()
+            context.state.globalInlineContext.enterDeclaration(irFunction.suspendFunctionOriginal().descriptor)
+            try {
+                ExpressionCodegen(irFunction, signature, frameMap, InstructionAdapter(methodVisitor), classCodegen, inlinedInto).generate()
+            } finally {
+                context.state.globalInlineContext.exitDeclaration()
+            }
             methodVisitor.visitMaxs(-1, -1)
         }
         methodVisitor.visitEnd()

compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/codegen/IrInlineCodegen.kt

@@ -5,14 +5,13 @@
 
 package org.jetbrains.kotlin.backend.jvm.codegen
 
-import com.intellij.psi.PsiElement
 import org.jetbrains.kotlin.codegen.*
 import org.jetbrains.kotlin.backend.jvm.JvmBackendContext
 import org.jetbrains.kotlin.backend.jvm.ir.isInlineParameter
 import org.jetbrains.kotlin.backend.jvm.ir.isLambda
+import org.jetbrains.kotlin.backend.jvm.lower.suspendFunctionOriginal
 import org.jetbrains.kotlin.codegen.inline.*
 import org.jetbrains.kotlin.codegen.state.GenerationState
-import org.jetbrains.kotlin.descriptors.CallableDescriptor
 import org.jetbrains.kotlin.descriptors.FunctionDescriptor
 import org.jetbrains.kotlin.descriptors.ValueParameterDescriptor
 import org.jetbrains.kotlin.ir.declarations.*
@@ -145,36 +144,16 @@ class IrInlineCodegen(
         invocationParamBuilder.markValueParametersStart()
     }
 
-    private inner class IrInlineCall(
-        private val irFunctionAccessExpression: IrFunctionAccessExpression
-    ) : InlineCall {
-
-        override val calleeDescriptor: CallableDescriptor =
-            irFunctionAccessExpression.symbol.descriptor.original
-
-        override val callElement: PsiElement?
-            get() =
-                codegen.context.psiSourceManager.findPsiElement(irFunctionAccessExpression, function)
-                    ?: codegen.context.psiSourceManager.findPsiElement(function)
-
-        override val id: Any
-            get() = irFunctionAccessExpression
-
-        override fun toString(): String = irFunctionAccessExpression.render()
-    }
-
     override fun genCall(
         callableMethod: IrCallableMethod,
         codegen: ExpressionCodegen,
         expression: IrFunctionAccessExpression
     ) {
-        val inlineCall = IrInlineCall(expression)
-        if (!state.globalInlineContext.enterIntoInlining(inlineCall)) {
-            AsmUtil.genThrow(
-                codegen.v,
-                "java/lang/UnsupportedOperationException",
-                "Call is a part of inline call cycle: ${expression.render()}"
-            )
+        val element = codegen.context.psiSourceManager.findPsiElement(expression, codegen.irFunction)
+            ?: codegen.context.psiSourceManager.findPsiElement(codegen.irFunction)
+        if (!state.globalInlineContext.enterIntoInlining(expression.symbol.owner.suspendFunctionOriginal().descriptor, element)) {
+            val message = "Call is a part of inline call cycle: ${expression.render()}"
+            AsmUtil.genThrow(codegen.v, "java/lang/UnsupportedOperationException", message)
             return
         }
         try {
@@ -186,7 +165,7 @@ class IrInlineCodegen(
                 false
             )
         } finally {
-            state.globalInlineContext.exitFromInliningOf(inlineCall)
+            state.globalInlineContext.exitFromInlining()
         }
     }
 

compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/codegen/MethodSignatureMapper.kt

@@ -112,7 +112,7 @@ class MethodSignatureMapper(private val context: JvmBackendContext) {
         val newName = JvmCodegenUtil.sanitizeNameIfNeeded(name, context.state.languageVersionSettings)
 
         if (function.isTopLevel) {
-            if (Visibilities.isPrivate(if (function.isSuspend) function.suspendFunctionOriginal().visibility else function.visibility) &&
+            if (Visibilities.isPrivate(function.suspendFunctionOriginal().visibility) &&
                 newName != "<clinit>" && (function.parent as? IrClass)?.attributeOwnerId in context.multifileFacadeForPart
             ) {
                 return "$newName$${function.parentAsClass.name.asString()}"

compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/lower/AddContinuationLowering.kt

@@ -639,10 +639,8 @@ private fun IrFunction.suspendFunctionViewOrStub(context: JvmBackendContext): Ir
     return context.suspendFunctionOriginalToView.getOrPut(suspendFunctionOriginal()) { createSuspendFunctionStub(context) }
 }
 
-internal fun IrFunction.suspendFunctionOriginal(): IrFunction {
-    require(isSuspend && this is IrSimpleFunction)
-    return attributeOwnerId as IrFunction
-}
+internal fun IrFunction.suspendFunctionOriginal(): IrFunction =
+    if (this is IrSimpleFunction && isSuspend) attributeOwnerId as IrFunction else this
 
 private fun IrFunction.createSuspendFunctionStub(context: JvmBackendContext): IrFunction {
     require(this.isSuspend && this is IrSimpleFunction)

compiler/testData/diagnostics/testsWithJvmBackend/indirectInlineCycle.diag.txt

@@ -0,0 +1,6 @@
+/indirectInlineCycle.kt:8:24: error: the 'inlineFun2' invocation is a part of inline cycle
+        fun method() { inlineFun2(p) }
+                       ^
+/indirectInlineCycle.kt:13:5: error: the 'inlineFun1' invocation is a part of inline cycle
+    inlineFun1(p)
+    ^

compiler/testData/diagnostics/testsWithJvmBackend/indirectInlineCycle.kt

@@ -0,0 +1,11 @@
+// !RENDER_DIAGNOSTICS_FULL_TEXT
+
+inline fun inlineFun1(crossinline p: () -> Unit) {
+    object {
+        fun method() { <!INLINE_CALL_CYCLE, INLINE_CALL_CYCLE!>inlineFun2(p)<!> }
+    }
+}
+
+inline fun inlineFun2(crossinline p: () -> Unit) {
+    <!INLINE_CALL_CYCLE, INLINE_CALL_CYCLE!>inlineFun1(p)<!>
+}

compiler/testData/diagnostics/testsWithJvmBackend/indirectInlineCycle.txt

@@ -0,0 +1,4 @@
+package
+
+public inline fun inlineFun1(/*0*/ crossinline p: () -> kotlin.Unit): kotlin.Unit
+public inline fun inlineFun2(/*0*/ crossinline p: () -> kotlin.Unit): kotlin.Unit

compiler/testData/diagnostics/testsWithJvmBackend/suspendInlineCycle.diag.txt

@@ -0,0 +1,6 @@
+/suspendInlineCycle.kt:8:5: error: the 'inlineFun2' invocation is a part of inline cycle
+    inlineFun2(p)
+    ^
+/suspendInlineCycle.kt:13:5: error: the 'inlineFun1' invocation is a part of inline cycle
+    inlineFun1(p)
+    ^

compiler/testData/diagnostics/testsWithJvmBackend/suspendInlineCycle.kt

@@ -0,0 +1,11 @@
+// !RENDER_DIAGNOSTICS_FULL_TEXT
+// Note: 4 diagnostics per call because there are 2 synthetic $$forInline methods.
+suspend inline fun inlineFun1(p: () -> Unit) {
+    p()
+    <!INLINE_CALL_CYCLE, INLINE_CALL_CYCLE, INLINE_CALL_CYCLE, INLINE_CALL_CYCLE!>inlineFun2(p)<!>
+}
+
+suspend inline fun inlineFun2(p: () -> Unit) {
+    p()
+    <!INLINE_CALL_CYCLE, INLINE_CALL_CYCLE, INLINE_CALL_CYCLE, INLINE_CALL_CYCLE!>inlineFun1(p)<!>
+}

compiler/testData/diagnostics/testsWithJvmBackend/suspendInlineCycle.txt

@@ -0,0 +1,4 @@
+package
+
+public suspend inline fun inlineFun1(/*0*/ p: () -> kotlin.Unit): kotlin.Unit
+public suspend inline fun inlineFun2(/*0*/ p: () -> kotlin.Unit): kotlin.Unit

compiler/tests/org/jetbrains/kotlin/checkers/DiagnosticsTestWithJvmIrBackendGenerated.java

@@ -29,11 +29,21 @@ public class DiagnosticsTestWithJvmIrBackendGenerated extends AbstractDiagnostic
         KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/diagnostics/testsWithJvmBackend"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JVM_IR, true);
     }
 
+    @TestMetadata("indirectInlineCycle.kt")
+    public void testIndirectInlineCycle() throws Exception {
+        runTest("compiler/testData/diagnostics/testsWithJvmBackend/indirectInlineCycle.kt");
+    }
+
     @TestMetadata("inlineCycle.kt")
     public void testInlineCycle() throws Exception {
         runTest("compiler/testData/diagnostics/testsWithJvmBackend/inlineCycle.kt");
     }
 
+    @TestMetadata("suspendInlineCycle.kt")
+    public void testSuspendInlineCycle() throws Exception {
+        runTest("compiler/testData/diagnostics/testsWithJvmBackend/suspendInlineCycle.kt");
+    }
+
     @TestMetadata("compiler/testData/diagnostics/testsWithJvmBackend/duplicateJvmSignature")
     @TestDataPath("$PROJECT_ROOT")
     @RunWith(JUnit3RunnerWithInners.class)

compiler/tests/org/jetbrains/kotlin/checkers/DiagnosticsTestWithOldJvmBackendGenerated.java

@@ -29,11 +29,21 @@ public class DiagnosticsTestWithOldJvmBackendGenerated extends AbstractDiagnosti
         KotlinTestUtils.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("compiler/testData/diagnostics/testsWithJvmBackend"), Pattern.compile("^(.+)\\.kt$"), null, TargetBackend.JVM_OLD, true);
     }
 
+    @TestMetadata("indirectInlineCycle.kt")
+    public void testIndirectInlineCycle() throws Exception {
+        runTest("compiler/testData/diagnostics/testsWithJvmBackend/indirectInlineCycle.kt");
+    }
+
     @TestMetadata("inlineCycle.kt")
     public void testInlineCycle() throws Exception {
         runTest("compiler/testData/diagnostics/testsWithJvmBackend/inlineCycle.kt");
     }
 
+    @TestMetadata("suspendInlineCycle.kt")
+    public void testSuspendInlineCycle() throws Exception {
+        runTest("compiler/testData/diagnostics/testsWithJvmBackend/suspendInlineCycle.kt");
+    }
+
     @TestMetadata("compiler/testData/diagnostics/testsWithJvmBackend/duplicateJvmSignature")
     @TestDataPath("$PROJECT_ROOT")
     @RunWith(JUnit3RunnerWithInners.class)