Debugger: Fix infinite recursion in inline callable searcher (EA-108938)

idea/jvm-debugger/jvm-debugger-core/src/org/jetbrains/kotlin/idea/debugger/DebuggerClassNameProvider.kt

@@ -104,12 +104,12 @@ class DebuggerClassNameProvider(
         }
 
         val result = getOrComputeClassNames(relevantElement) { element ->
-            getOuterClassNamesForElement(element)
+            getOuterClassNamesForElement(element, emptySet())
         }.toMutableSet()
 
         for (lambda in position.readAction(::getLambdasAtLineIfAny)) {
             result += getOrComputeClassNames(lambda) { element ->
-                getOuterClassNamesForElement(element)
+                getOuterClassNamesForElement(element, emptySet())
             }
         }
 
@@ -118,7 +118,10 @@ class DebuggerClassNameProvider(
 
     @PublishedApi
     @Suppress("NON_TAIL_RECURSIVE_CALL")
-    internal tailrec fun getOuterClassNamesForElement(element: PsiElement?): ComputedClassNames {
+    internal tailrec fun getOuterClassNamesForElement(element: PsiElement?, alreadyVisited: Set<PsiElement>): ComputedClassNames {
+        // 'alreadyVisited' is used in inline callable searcher to prevent infinite recursion.
+        // In normal cases we only go from leaves to topmost parents upwards.
+
         if (element == null) return EMPTY
 
         return when (element) {
@@ -135,9 +138,9 @@ class DebuggerClassNameProvider(
                 when {
                     enclosingElementForLocal != null ->
                         // A local class
-                        getOuterClassNamesForElement(enclosingElementForLocal)
+                        getOuterClassNamesForElement(enclosingElementForLocal, alreadyVisited)
                     runReadAction { element.isObjectLiteral() } ->
-                        getOuterClassNamesForElement(element.relevantParentInReadAction)
+                        getOuterClassNamesForElement(element.relevantParentInReadAction, alreadyVisited)
                     else ->
                         // Guaranteed to be non-local class or object
                         element.readAction { _ ->
@@ -157,12 +160,12 @@ class DebuggerClassNameProvider(
             is KtProperty -> {
                 val nonInlineClasses = if (runReadAction { isTopLevelInFileOrScript(element) }) {
                     // Top level property
-                    getOuterClassNamesForElement(element.relevantParentInReadAction)
+                    getOuterClassNamesForElement(element.relevantParentInReadAction, alreadyVisited)
                 } else {
                     val enclosingElementForLocal = runReadAction { KtPsiUtil.getEnclosingElementForLocalDeclaration(element) }
                     if (enclosingElementForLocal != null) {
                         // Local class
-                        getOuterClassNamesForElement(enclosingElementForLocal)
+                        getOuterClassNamesForElement(enclosingElementForLocal, alreadyVisited)
                     } else {
                         val containingClassOrFile = runReadAction {
                             PsiTreeUtil.getParentOfType(element, KtFile::class.java, KtClassOrObject::class.java)
@@ -170,12 +173,12 @@ class DebuggerClassNameProvider(
 
                         if (containingClassOrFile is KtObjectDeclaration && containingClassOrFile.isCompanionInReadAction) {
                             // Properties from the companion object can be placed in the companion object's containing class
-                            (getOuterClassNamesForElement(containingClassOrFile.relevantParentInReadAction) +
+                            (getOuterClassNamesForElement(containingClassOrFile.relevantParentInReadAction, alreadyVisited) +
-                                    getOuterClassNamesForElement(containingClassOrFile)).distinct()
+                                    getOuterClassNamesForElement(containingClassOrFile, alreadyVisited)).distinct()
                         } else if (containingClassOrFile != null) {
-                            getOuterClassNamesForElement(containingClassOrFile)
+                            getOuterClassNamesForElement(containingClassOrFile, alreadyVisited)
                         } else {
-                            getOuterClassNamesForElement(element.relevantParentInReadAction)
+                            getOuterClassNamesForElement(element.relevantParentInReadAction, alreadyVisited)
                         }
                     }
                 }
@@ -184,13 +187,16 @@ class DebuggerClassNameProvider(
                             element.isInlineInReadAction ||
                                     runReadAction { element.accessors.any { it.hasModifier(KtTokens.INLINE_KEYWORD) } })
                 ) {
-                    nonInlineClasses + inlineUsagesSearcher.findInlinedCalls(element) { this.getOuterClassNamesForElement(it) }
+                    val inlinedCalls = inlineUsagesSearcher.findInlinedCalls(element, alreadyVisited) { el, newAlreadyVisited ->
+                        this.getOuterClassNamesForElement(el, newAlreadyVisited)
+                    }
+                    nonInlineClasses + inlinedCalls
                 } else {
                     return NonCached(nonInlineClasses.classNames)
                 }
             }
             is KtNamedFunction -> {
-                val classNamesOfContainingDeclaration = getOuterClassNamesForElement(element.relevantParentInReadAction)
+                val classNamesOfContainingDeclaration = getOuterClassNamesForElement(element.relevantParentInReadAction, alreadyVisited)
 
                 var nonInlineClasses: ComputedClassNames = classNamesOfContainingDeclaration
 
@@ -205,7 +211,9 @@ class DebuggerClassNameProvider(
                     return NonCached(nonInlineClasses.classNames)
                 }
 
-                val inlineCallSiteClasses = inlineUsagesSearcher.findInlinedCalls(element) { this.getOuterClassNamesForElement(it) }
+                val inlineCallSiteClasses = inlineUsagesSearcher.findInlinedCalls(element, alreadyVisited) { el, newAlreadyVisited ->
+                    this.getOuterClassNamesForElement(el, newAlreadyVisited)
+                }
 
                 nonInlineClasses + inlineCallSiteClasses
             }
@@ -213,10 +221,11 @@ class DebuggerClassNameProvider(
                 val initializerOwner = runReadAction { element.containingDeclaration }
 
                 if (initializerOwner is KtObjectDeclaration && initializerOwner.isCompanionInReadAction) {
-                    return getOuterClassNamesForElement(runReadAction { initializerOwner.containingClassOrObject })
+                    val containingClass = runReadAction { initializerOwner.containingClassOrObject }
+                    return getOuterClassNamesForElement(containingClass, alreadyVisited)
                 }
 
-                getOuterClassNamesForElement(initializerOwner)
+                getOuterClassNamesForElement(initializerOwner, alreadyVisited)
             }
             is KtFunctionLiteral -> {
                 val typeMapper = KotlinDebuggerCaches.getOrCreateTypeMapper(element)
@@ -233,9 +242,9 @@ class DebuggerClassNameProvider(
                     return names
                 }
 
-                names + getOuterClassNamesForElement(element.relevantParentInReadAction)
+                names + getOuterClassNamesForElement(element.relevantParentInReadAction, alreadyVisited)
             }
-            else -> getOuterClassNamesForElement(element.relevantParentInReadAction)
+            else -> getOuterClassNamesForElement(element.relevantParentInReadAction, alreadyVisited)
         }
     }
 

idea/jvm-debugger/jvm-debugger-core/src/org/jetbrains/kotlin/idea/debugger/InlineCallableUsagesSearcher.kt

@@ -22,6 +22,7 @@ import com.intellij.openapi.progress.EmptyProgressIndicator
 import com.intellij.openapi.progress.ProgressManager
 import com.intellij.openapi.ui.MessageType
 import com.intellij.psi.PsiElement
+import com.intellij.psi.PsiReference
 import com.intellij.psi.search.GlobalSearchScope
 import com.intellij.psi.search.searches.ReferencesSearch
 import com.intellij.xdebugger.impl.XDebugSessionImpl
@@ -37,15 +38,15 @@ import org.jetbrains.kotlin.idea.util.application.runReadAction
 import org.jetbrains.kotlin.psi.KtDeclaration
 import org.jetbrains.kotlin.psi.KtElement
 import org.jetbrains.kotlin.psi.psiUtil.isAncestor
-import org.jetbrains.kotlin.psi.psiUtil.isInsideOf
 import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.resolve.inline.InlineUtil
 
 class InlineCallableUsagesSearcher(private val myDebugProcess: DebugProcess) {
     fun findInlinedCalls(
             declaration: KtDeclaration,
+            alreadyVisited: Set<PsiElement>,
             bindingContext: BindingContext = KotlinDebuggerCaches.getOrCreateTypeMapper(declaration).bindingContext,
-            transformer: (PsiElement) -> ComputedClassNames
+            transformer: (PsiElement, Set<PsiElement>) -> ComputedClassNames
     ): ComputedClassNames {
         if (!checkIfInline(declaration, bindingContext)) {
             return ComputedClassNames.EMPTY
@@ -55,13 +56,8 @@ class InlineCallableUsagesSearcher(private val myDebugProcess: DebugProcess) {
             val declarationName = runReadAction { declaration.name }
 
             val task = Runnable {
-                ReferencesSearch.search(declaration, getScopeForInlineDeclarationUsages(declaration)).forEach {
+                for (reference in ReferencesSearch.search(declaration, getScopeForInlineDeclarationUsages(declaration))) {
-                    if (!runReadAction { it.isImportUsage() }) {
+                    processReference(declaration, reference, alreadyVisited)?.let { searchResult += it }
-                        val usage = (it.element as? KtElement)?.let(::getRelevantElement)
-                        if (usage != null && !runReadAction { declaration.isAncestor(usage) }) {
-                            searchResult.add(usage)
-                        }
-                    }
                 }
             }
 
@@ -85,11 +81,27 @@ class InlineCallableUsagesSearcher(private val myDebugProcess: DebugProcess) {
                 ).notify(myDebugProcess.project)
             }
 
-            val results = searchResult.map { transformer(it) }
+            val newAlreadyVisited = HashSet<PsiElement>().apply {
+                addAll(alreadyVisited)
+                addAll(searchResult)
+                add(declaration)
+            }
+
+            val results = searchResult.map { transformer(it, newAlreadyVisited) }
             return ComputedClassNames(results.flatMap { it.classNames }, shouldBeCached = results.all { it.shouldBeCached })
         }
     }
 
+    private fun processReference(declaration: KtDeclaration, reference: PsiReference, alreadyVisited: Set<PsiElement>): PsiElement? {
+        if (runReadAction { reference.isImportUsage() }) {
+            return null
+        }
+
+        val usage = (reference.element as? KtElement)?.let(::getRelevantElement) ?: return null
+        val shouldAnalyze = runReadAction { !declaration.isAncestor(usage) && usage !in alreadyVisited }
+        return if (shouldAnalyze) usage else null
+    }
+
     private fun checkIfInline(declaration: KtDeclaration, bindingContext: BindingContext): Boolean {
         val descriptor = bindingContext.get(BindingContext.DECLARATION_TO_DESCRIPTOR, declaration) ?: return false
         return when (descriptor) {

idea/jvm-debugger/jvm-debugger-core/src/org/jetbrains/kotlin/idea/debugger/KotlinPositionManager.kt

@@ -237,7 +237,7 @@ class KotlinPositionManager(private val myDebugProcess: DebugProcess) : MultiReq
             }
 
             val internalClassNames = DebuggerClassNameProvider(myDebugProcess, alwaysReturnLambdaParentClass = false)
-                    .getOuterClassNamesForElement(literal.firstChild)
+                    .getOuterClassNamesForElement(literal.firstChild, emptySet())
                     .classNames
 
             if (internalClassNames.any { it == currentLocationClassName }) {