[EE-IR] Fragment Compiler Entrypoint in IR Backend

This commit introduces a new entrypoint for the IR backend, and starts
the work of accomodating the evaluator in psi2ir.

The evaluator expects a certain class and method structure of the
compiled fragment, but PSI is only supplied for the actual fragment.

So, to that end, this commit introduces a new "front end" of psi2ir
that "synthesizes" the module, class and method structure around the
fragment before calling into the existing psi2ir pipeline to obtain
the IR for the fragment.

The primary complication so far is handling the captured variables of
the fragment: they are mapped to parameters of the method surrounding
the fragment, and passed as arguments on evaluation. Hence, the IR
translation of the fragment needs to remap captured variables to the
appropriate parameter, in essentially all places they can be referred
to in the fragment (and hence in psi2ir). This commit introduces a
decorated symbol table that intercepts symbol look-ups and remaps as
appropriate.

Other cases that dispatches based on descriptor (see
`CallGenerator.kt`) needs other metadata to generate correct code.

It also introduces a shim in DeserializedContainerSource in the psi2ir
pipeline to facilitate facade class generation for the code that is
being debugged (which are generated as "external ir declarations").

Finally, in passing we resolve a small leftover from previous
refactoring that left an asssertion re. allowing IR to _assign_ to
parameters of methods.

compiler/ir/ir.psi2ir/src/org/jetbrains/kotlin/psi2ir/Psi2IrTranslator.kt

@@ -31,6 +31,9 @@ import org.jetbrains.kotlin.psi2ir.generators.GeneratorContext
 import org.jetbrains.kotlin.psi2ir.generators.GeneratorExtensions
 import org.jetbrains.kotlin.psi2ir.generators.ModuleGenerator
 import org.jetbrains.kotlin.psi2ir.generators.TypeTranslatorImpl
+import org.jetbrains.kotlin.psi2ir.generators.fragments.EvaluatorFragmentInfo
+import org.jetbrains.kotlin.psi2ir.generators.fragments.FragmentContext
+import org.jetbrains.kotlin.psi2ir.generators.fragments.FragmentModuleGenerator
 import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.utils.SmartList
 
@@ -52,7 +55,8 @@ class Psi2IrTranslator(
         moduleDescriptor: ModuleDescriptor,
         bindingContext: BindingContext,
         symbolTable: SymbolTable,
-        extensions: GeneratorExtensions = GeneratorExtensions()
+        extensions: GeneratorExtensions = GeneratorExtensions(),
+        fragmentContext: FragmentContext? = null
     ): GeneratorContext {
         val typeTranslator = TypeTranslatorImpl(symbolTable, languageVersionSettings, moduleDescriptor, extensions = extensions)
         return GeneratorContext(
@@ -64,6 +68,7 @@ class Psi2IrTranslator(
             extensions,
             typeTranslator,
             IrBuiltInsOverDescriptors(moduleDescriptor.builtIns, typeTranslator, symbolTable),
+            fragmentContext
         )
     }
 
@@ -72,9 +77,13 @@ class Psi2IrTranslator(
         ktFiles: Collection<KtFile>,
         irProviders: List<IrProvider>,
         linkerExtensions: Collection<IrDeserializer.IrLinkerExtension>,
-        expectDescriptorToSymbol: MutableMap<DeclarationDescriptor, IrSymbol>? = null
+        expectDescriptorToSymbol: MutableMap<DeclarationDescriptor, IrSymbol>? = null,
+        fragmentInfo: EvaluatorFragmentInfo? = null
     ): IrModuleFragment {
-        val moduleGenerator = ModuleGenerator(context, expectDescriptorToSymbol)
+        val moduleGenerator = fragmentInfo?.let {
+            FragmentModuleGenerator(context, it)
+        } ?: ModuleGenerator(context, expectDescriptorToSymbol)
+
         val irModule = moduleGenerator.generateModuleFragment(ktFiles)
 
         val deserializers = irProviders.filterIsInstance<IrDeserializer>()

compiler/ir/ir.psi2ir/src/org/jetbrains/kotlin/psi2ir/generators/CallGenerator.kt

@@ -89,8 +89,16 @@ class CallGenerator(statementGenerator: StatementGenerator) : StatementGenerator
         resolvedCall: ResolvedCall<*>?,
         origin: IrStatementOrigin?,
         smartCastIrType: IrType? = null
-    ): IrExpression =
-        when (descriptor) {
+    ): IrExpression {
+        context.fragmentContext?.capturedDescriptorToFragmentParameterMap?.get(descriptor)?.let {
+            val getValue = IrGetValueImpl(startOffset, endOffset, it.descriptor.type.toIrType(), it, origin)
+            return if (smartCastIrType != null) {
+                IrTypeOperatorCallImpl(startOffset, endOffset, smartCastIrType, IrTypeOperator.IMPLICIT_CAST, smartCastIrType, getValue)
+            } else {
+                getValue
+            }
+        }
+        return when (descriptor) {
             is FakeCallableDescriptorForObject ->
                 generateValueReference(startOffset, endOffset, descriptor.getReferencedDescriptor(), resolvedCall, origin, smartCastIrType)
             is TypeAliasDescriptor ->
@@ -117,6 +125,7 @@ class CallGenerator(statementGenerator: StatementGenerator) : StatementGenerator
             else ->
                 TODO("Unexpected callable descriptor: $descriptor ${descriptor::class.java.simpleName}")
         }
+    }
 
     private fun generateGetVariable(
         startOffset: Int,

compiler/ir/ir.psi2ir/src/org/jetbrains/kotlin/psi2ir/generators/GeneratorContext.kt

@@ -9,16 +9,18 @@ import org.jetbrains.kotlin.backend.common.SamTypeApproximator
 import org.jetbrains.kotlin.builtins.ReflectionTypes
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.descriptors.CallableDescriptor
+import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.NotFoundClasses
 import org.jetbrains.kotlin.ir.IrBuiltIns
 import org.jetbrains.kotlin.ir.builders.IrGeneratorContext
-import org.jetbrains.kotlin.ir.descriptors.IrBuiltInsOverDescriptors
 import org.jetbrains.kotlin.ir.expressions.IrDeclarationReference
+import org.jetbrains.kotlin.ir.symbols.IrValueParameterSymbol
 import org.jetbrains.kotlin.ir.util.SymbolTable
 import org.jetbrains.kotlin.ir.util.TypeTranslator
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.psi2ir.Psi2IrConfiguration
+import org.jetbrains.kotlin.psi2ir.generators.fragments.FragmentContext
 import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
 
@@ -31,7 +33,8 @@ class GeneratorContext private constructor(
     val extensions: GeneratorExtensions,
     val typeTranslator: TypeTranslator,
     override val irBuiltIns: IrBuiltIns,
-    internal val callToSubstitutedDescriptorMap: MutableMap<IrDeclarationReference, CallableDescriptor>
+    internal val callToSubstitutedDescriptorMap: MutableMap<IrDeclarationReference, CallableDescriptor>,
+    internal val fragmentContext: FragmentContext?,
 ) : IrGeneratorContext {
 
     constructor(
@@ -43,6 +46,7 @@ class GeneratorContext private constructor(
         extensions: GeneratorExtensions,
         typeTranslator: TypeTranslator,
         irBuiltIns: IrBuiltIns,
+        fragmentContext: FragmentContext? = null
     ) : this(
         configuration,
         moduleDescriptor,
@@ -52,7 +56,8 @@ class GeneratorContext private constructor(
         extensions,
         typeTranslator,
         irBuiltIns,
-        mutableMapOf()
+        mutableMapOf(),
+        fragmentContext,
     )
 
     val constantValueGenerator = typeTranslator.constantValueGenerator
@@ -76,8 +81,8 @@ class GeneratorContext private constructor(
             extensions,
             TypeTranslatorImpl(symbolTable, languageVersionSettings, moduleDescriptor, extensions = extensions, ktFile = ktFile),
             irBuiltIns,
-            callToSubstitutedDescriptorMap
+            callToSubstitutedDescriptorMap,
+            fragmentContext,
         )
     }
-
 }

compiler/ir/ir.psi2ir/src/org/jetbrains/kotlin/psi2ir/generators/ModuleGenerator.kt

@@ -34,12 +34,12 @@ import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.resolve.lazy.descriptors.findPackageFragmentForFile
 import org.jetbrains.kotlin.utils.addIfNotNull
 
-class ModuleGenerator(
+open class ModuleGenerator(
     override val context: GeneratorContext,
     private val expectDescriptorToSymbol: MutableMap<DeclarationDescriptor, IrSymbol>? = null
 ) : Generator {
 
-    fun generateModuleFragment(ktFiles: Collection<KtFile>): IrModuleFragment =
+    open fun generateModuleFragment(ktFiles: Collection<KtFile>): IrModuleFragment =
         IrModuleFragmentImpl(context.moduleDescriptor, context.irBuiltIns).also { irModule ->
             ktFiles.toSet().mapTo(irModule.files) { ktFile ->
                 val fileContext = context.createFileScopeContext(ktFile)

compiler/ir/ir.psi2ir/src/org/jetbrains/kotlin/psi2ir/generators/fragments/EvaluatorFragmentInfo.kt

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.psi2ir.generators.fragments
+
+import org.jetbrains.kotlin.descriptors.ClassDescriptor
+import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
+import org.jetbrains.kotlin.descriptors.FunctionDescriptor
+
+/**
+ *  Information for compilation of code fragments for `evaluate expression`
+ *
+ *  The expression evaluator works by wrapping a code fragment in a method.
+ *  The free variables of the fragment are closed over by the parameters of
+ *  that method, and finally the method is placed in a class.
+ *
+ *  This data structure contains "synthesized" descriptors for that class,
+ *  method and parameter lay-out.
+ */
+class EvaluatorFragmentInfo(
+    val classDescriptor: ClassDescriptor,
+    val methodDescriptor: FunctionDescriptor,
+    val parameters: List<DeclarationDescriptor>
+)

compiler/ir/ir.psi2ir/src/org/jetbrains/kotlin/psi2ir/generators/fragments/FragmentCompilerSymbolTableDecorator.kt

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.psi2ir.generators.fragments
+
+import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
+import org.jetbrains.kotlin.descriptors.ParameterDescriptor
+import org.jetbrains.kotlin.descriptors.ReceiverParameterDescriptor
+import org.jetbrains.kotlin.descriptors.ValueDescriptor
+import org.jetbrains.kotlin.ir.declarations.IrFactory
+import org.jetbrains.kotlin.ir.symbols.IrValueParameterSymbol
+import org.jetbrains.kotlin.ir.symbols.IrValueSymbol
+import org.jetbrains.kotlin.ir.util.IdSignatureComposer
+import org.jetbrains.kotlin.ir.util.NameProvider
+import org.jetbrains.kotlin.ir.util.SymbolTable
+import org.jetbrains.kotlin.psi2ir.generators.fragments.EvaluatorFragmentInfo
+import org.jetbrains.kotlin.resolve.scopes.receivers.ExtensionReceiver
+import org.jetbrains.kotlin.resolve.scopes.receivers.ImplicitClassReceiver
+import org.jetbrains.kotlin.resolve.scopes.receivers.ThisClassReceiver
+
+// Used from CodeFragmentCompiler for IDE Debugger Plug-In
+@Suppress("unused")
+class FragmentCompilerSymbolTableDecorator(
+    signatureComposer: IdSignatureComposer,
+    irFactory: IrFactory,
+    private val fragmentInfo: EvaluatorFragmentInfo,
+    nameProvider: NameProvider = NameProvider.DEFAULT,
+) : SymbolTable(signatureComposer, irFactory, nameProvider) {
+
+    override fun referenceValueParameter(descriptor: ParameterDescriptor): IrValueParameterSymbol {
+        if (descriptor !is ReceiverParameterDescriptor) return super.referenceValueParameter(descriptor)
+
+        val finderPredicate = when (val receiverValue = descriptor.value) {
+            is ExtensionReceiver -> { targetDescriptor: DeclarationDescriptor ->
+                receiverValue == (targetDescriptor as? ReceiverParameterDescriptor)?.value
+            }
+            is ThisClassReceiver -> { targetDescriptor: DeclarationDescriptor ->
+                receiverValue.classDescriptor == targetDescriptor.original
+            }
+            else -> TODO("Unimplemented")
+        }
+
+        val parameterPosition =
+            fragmentInfo.parameters.indexOfFirst(finderPredicate)
+        if (parameterPosition > -1) {
+            return super.referenceValueParameter(fragmentInfo.methodDescriptor.valueParameters[parameterPosition])
+        }
+        return super.referenceValueParameter(descriptor)
+    }
+
+    override fun referenceValue(value: ValueDescriptor): IrValueSymbol {
+        val parameterPosition =
+            fragmentInfo.parameters.indexOf(value)
+        if (parameterPosition > -1) {
+            return super.referenceValueParameter(fragmentInfo.methodDescriptor.valueParameters[parameterPosition])
+        }
+
+        return super.referenceValue(value)
+    }
+}

compiler/ir/ir.psi2ir/src/org/jetbrains/kotlin/psi2ir/generators/fragments/FragmentContext.kt

@@ -0,0 +1,13 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.psi2ir.generators.fragments
+
+import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
+import org.jetbrains.kotlin.ir.symbols.IrValueParameterSymbol
+
+class FragmentContext(
+    val capturedDescriptorToFragmentParameterMap: MutableMap<DeclarationDescriptor, IrValueParameterSymbol> = mutableMapOf()
+)

compiler/ir/ir.psi2ir/src/org/jetbrains/kotlin/psi2ir/generators/fragments/FragmentDeclarationGenerator.kt

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.psi2ir.generators.fragments
+
+import org.jetbrains.kotlin.descriptors.Modality
+import org.jetbrains.kotlin.descriptors.ValueParameterDescriptor
+import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
+import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.ir.symbols.impl.IrConstructorPublicSymbolImpl
+import org.jetbrains.kotlin.ir.util.createIrClassFromDescriptor
+import org.jetbrains.kotlin.ir.util.declareSimpleFunctionWithOverrides
+import org.jetbrains.kotlin.ir.util.defaultType
+import org.jetbrains.kotlin.ir.util.withScope
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.psi.KtBlockCodeFragment
+import org.jetbrains.kotlin.psi2ir.generators.Generator
+import org.jetbrains.kotlin.psi2ir.generators.GeneratorContext
+import org.jetbrains.kotlin.psi2ir.generators.createBodyGenerator
+import org.jetbrains.kotlin.types.KotlinType
+
+open class FragmentDeclarationGenerator(
+    override val context: GeneratorContext,
+    private val fragmentInfo: EvaluatorFragmentInfo
+) : Generator {
+
+    fun generateClassForCodeFragment(ktFile: KtBlockCodeFragment): IrClass {
+        val classDescriptor = fragmentInfo.classDescriptor
+        val startOffset = UNDEFINED_OFFSET
+        val endOffset = UNDEFINED_OFFSET
+
+        return context.symbolTable.declareClass(classDescriptor) {
+            context.irFactory.createIrClassFromDescriptor(
+                startOffset, endOffset,
+                IrDeclarationOrigin.DEFINED,
+                symbol = it,
+                classDescriptor,
+                context.symbolTable.nameProvider.nameForDeclaration(classDescriptor),
+                classDescriptor.visibility,
+                Modality.FINAL
+            )
+        }.buildWithScope { irClass ->
+            irClass.thisReceiver = context.symbolTable.declareValueParameter(
+                startOffset, endOffset,
+                IrDeclarationOrigin.INSTANCE_RECEIVER,
+                classDescriptor.thisAsReceiverParameter,
+                classDescriptor.thisAsReceiverParameter.type.toIrType()
+            )
+
+            generatePrimaryConstructor(irClass)
+
+            irClass.declarations.add(
+                generateFunctionForFragment(ktFile)
+            )
+        }
+    }
+
+    private fun generatePrimaryConstructor(irClass: IrClass) {
+        val constructor = context.irFactory.createConstructor(
+            startOffset = UNDEFINED_OFFSET,
+            endOffset = UNDEFINED_OFFSET,
+            origin = IrDeclarationOrigin.DEFINED,
+            symbol = IrConstructorPublicSymbolImpl(context.symbolTable.signaturer.composeSignature(irClass.descriptor)!!),
+            Name.special("<init>"),
+            irClass.visibility,
+            irClass.defaultType,
+            isInline = false,
+            isExternal = false,
+            isPrimary = true,
+            isExpect = false
+        )
+        constructor.parent = irClass
+        constructor.body = context.irFactory.createBlockBody(UNDEFINED_OFFSET, UNDEFINED_OFFSET)
+        irClass.addMember(constructor)
+    }
+
+    private fun generateFunctionForFragment(ktFile: KtBlockCodeFragment): IrSimpleFunction {
+        return context.symbolTable.declareSimpleFunctionWithOverrides(
+            UNDEFINED_OFFSET, UNDEFINED_OFFSET,
+            IrDeclarationOrigin.DEFINED,
+            fragmentInfo.methodDescriptor
+        ).buildWithScope { irFunction ->
+            irFunction.returnType = fragmentInfo.methodDescriptor.returnType!!.toIrType()
+            generateFragmentValueParameterDeclarations(irFunction)
+            irFunction.body = createBodyGenerator(irFunction.symbol).generateExpressionBody(ktFile.getContentElement())
+        }
+    }
+
+    private fun generateFragmentValueParameterDeclarations(irFunction: IrSimpleFunction) {
+        val functionDescriptor = irFunction.descriptor
+        functionDescriptor.valueParameters.forEachIndexed { index, valueParameterDescriptor ->
+            irFunction.valueParameters += declareParameter(valueParameterDescriptor).apply {
+                context.fragmentContext!!.capturedDescriptorToFragmentParameterMap[fragmentInfo.parameters[index]] = this.symbol
+            }
+        }
+    }
+
+    private fun declareParameter(descriptor: ValueParameterDescriptor): IrValueParameter {
+        // Parameter must be _assignable_:
+        // These parameters model the captured variables of the fragment. The captured
+        // _values_ are extracted from the call stack of the JVM being debugged, and supplied
+        // to the fragment evaluator via these parameters. Any modifications by the fragment
+        // are written directly to the parameter, and then extracted from the stack frame
+        // of the interpreter/JVM evaluating the fragment and written back into the call
+        // stack of the JVM being debugged.
+        return context.symbolTable.declareValueParameter(
+            UNDEFINED_OFFSET,
+            UNDEFINED_OFFSET,
+            IrDeclarationOrigin.DEFINED,
+            descriptor,
+            descriptor.type.toIrType(),
+            (descriptor as? ValueParameterDescriptor)?.varargElementType?.toIrType(),
+            null,
+            isAssignable = true
+        )
+    }
+
+    private fun KotlinType.toIrType() = context.typeTranslator.translateType(this)
+
+    private inline fun <T : IrDeclaration> T.buildWithScope(builder: (T) -> Unit): T =
+        also { irDeclaration: T ->
+            context.symbolTable.withScope(irDeclaration) {
+                builder(irDeclaration)
+            }
+        }
+
+}

compiler/ir/ir.psi2ir/src/org/jetbrains/kotlin/psi2ir/generators/fragments/FragmentModuleGenerator.kt

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.psi2ir.generators.fragments
+
+import org.jetbrains.kotlin.ir.PsiIrFileEntry
+import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.ir.declarations.impl.IrFileImpl
+import org.jetbrains.kotlin.ir.declarations.impl.IrModuleFragmentImpl
+import org.jetbrains.kotlin.ir.util.patchDeclarationParents
+import org.jetbrains.kotlin.psi.KtBlockCodeFragment
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.psi2ir.generators.GeneratorContext
+import org.jetbrains.kotlin.psi2ir.generators.ModuleGenerator
+import org.jetbrains.kotlin.resolve.lazy.descriptors.findPackageFragmentForFile
+
+class FragmentModuleGenerator(
+    override val context: GeneratorContext,
+    private val fragmentInfo: EvaluatorFragmentInfo
+) : ModuleGenerator(context, expectDescriptorToSymbol = null) {
+
+    override fun generateModuleFragment(
+        ktFiles: Collection<KtFile>,
+    ): IrModuleFragment {
+        val ktBlockCodeFragment = ktFiles.singleOrNull() as? KtBlockCodeFragment
+            ?: TODO("Multiple fragments in one compilation not understood and implemented yet")
+        return IrModuleFragmentImpl(context.moduleDescriptor, context.irBuiltIns).also { irModule ->
+            val irDeclarationGenerator = FragmentDeclarationGenerator(context, fragmentInfo)
+            irModule.files.add(
+                createEmptyIrFile(ktBlockCodeFragment).apply {
+                    declarations.add(
+                        irDeclarationGenerator.generateClassForCodeFragment(ktBlockCodeFragment)
+                    )
+                    patchDeclarationParents()
+                }
+            )
+        }
+    }
+
+    private fun createEmptyIrFile(ktFile: KtFile): IrFileImpl {
+        val fileEntry = PsiIrFileEntry(ktFile)
+        val packageFragmentDescriptor = context.moduleDescriptor.findPackageFragmentForFile(ktFile)!!
+        return IrFileImpl(fileEntry, packageFragmentDescriptor)
+    }
+}

compiler/ir/ir.psi2ir/src/org/jetbrains/kotlin/psi2ir/intermediate/VariableLValue.kt

@@ -48,7 +48,7 @@ class VariableLValue(
         IrSetValueImpl(
             startOffset, endOffset,
             context.irBuiltIns.unitType,
-            symbol.assertedCast<IrVariableSymbol> { "Not a variable: ${symbol.descriptor}" },
+            symbol,
             irExpression, origin
         )