JVM IR: fix support of cyclic module dependencies

Previous episode was at 987a3460 (KT-45915).

Apparently, it's not enough to run psi2ir for all modules, and then
backend for all modules. If there are several modules in the chunk, IR
in any one of them can reference IR of any other one after psi2ir.

So we would end up in a situation where we're running codegen for the
first module, but the second module is completely unlowered. This would
break some assumptions in the codegen, for example in KT-49575, codegen
would see a reference to a top-level function from another module, and
would fail because the function has no containing class (since file
facades have not been generated yet!), and thus must be an intrinsic,
yet no such intrinsic is known to the codegen.

The solution is to run lowerings first on all modules, and then run
codegen on all modules. The kind of error explained above shouldn't be
possible anymore, because lowerings have to deal both with lowered and
unlowered code from other files all the time, so lowerings can't assume
that reference from other module is lowered either (or unlowered).

The code is not very nice, but hopefully it can be improved as soon as
we get rid of the old JVM backend (and maybe later with the migration to
FIR too).

 #KT-49575 Fixed

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/FirKotlinToJvmBytecodeCompiler.kt

@@ -62,9 +62,7 @@ import org.jetbrains.kotlin.resolve.PlatformDependentAnalyzerServices
 import org.jetbrains.kotlin.resolve.jvm.platform.JvmPlatformAnalyzerServices
 import org.jetbrains.kotlin.resolve.multiplatform.isCommonSource
 import org.jetbrains.kotlin.utils.addToStdlib.runIf
-import org.jetbrains.kotlin.utils.newLinkedHashMapWithExpectedSize
 import java.io.File
-import kotlin.collections.set
 
 object FirKotlinToJvmBytecodeCompiler {
     fun compileModulesUsingFrontendIR(
@@ -83,11 +81,12 @@ object FirKotlinToJvmBytecodeCompiler {
             "ATTENTION!\n This build uses in-dev FIR: \n  -Xuse-fir"
         )
 
-        val outputs = newLinkedHashMapWithExpectedSize<Module, Pair<FirResult, GenerationState>>(chunk.size)
+        val outputs = ArrayList<Pair<FirResult, GenerationState>>(chunk.size)
         val targetIds = projectConfiguration.get(JVMConfigurationKeys.MODULES)?.map(::TargetId)
         val incrementalComponents = projectConfiguration.get(JVMConfigurationKeys.INCREMENTAL_COMPILATION_COMPONENTS)
         val isMultiModuleChunk = chunk.size > 1
 
+        // TODO: run lowerings for all modules in the chunk, then run codegen for all modules.
         for (module in chunk) {
             val moduleConfiguration = projectConfiguration.applyModuleProperties(module, buildFile)
             val context = CompilationContext(
@@ -105,19 +104,18 @@ object FirKotlinToJvmBytecodeCompiler {
                 (projectEnvironment as? PsiBasedProjectEnvironment)?.project?.let { IrGenerationExtension.getInstances(it) } ?: emptyList()
             )
             val generationState = context.compileModule() ?: return false
-            outputs[module] = generationState
+            outputs += generationState
         }
 
         val mainClassFqName: FqName? = runIf(chunk.size == 1 && projectConfiguration.get(JVMConfigurationKeys.OUTPUT_JAR) != null) {
-            val firResult = outputs.values.single().first
-            findMainClass(firResult)
+            findMainClass(outputs.single().first)
         }
 
         return writeOutputs(
             (projectEnvironment as? PsiBasedProjectEnvironment)?.project,
             projectConfiguration,
             chunk,
-            outputs.mapValues { (_, value) -> value.second },
+            outputs.map(Pair<FirResult, GenerationState>::second),
             mainClassFqName
         )
     }

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.kt

@@ -42,15 +42,15 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.JVMConfigurationKeys
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory
+import org.jetbrains.kotlin.diagnostics.impl.BaseDiagnosticsCollector
 import org.jetbrains.kotlin.ir.backend.jvm.jvmResolveLibraries
 import org.jetbrains.kotlin.load.kotlin.ModuleVisibilityManager
 import org.jetbrains.kotlin.modules.Module
 import org.jetbrains.kotlin.progress.ProgressIndicatorAndCompilationCanceledStatus
 import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.resolve.jvm.KotlinJavaPsiFacade
-import org.jetbrains.kotlin.utils.newLinkedHashMapWithExpectedSize
 import java.io.File
-import kotlin.collections.set
 
 object KotlinToJVMBytecodeCompiler {
     internal fun compileModules(
@@ -111,11 +111,12 @@ object KotlinToJVMBytecodeCompiler {
                 findMainClass(result.bindingContext, projectConfiguration.languageVersionSettings, environment.getSourceFiles())
             else null
 
-        val outputs = newLinkedHashMapWithExpectedSize<Module, GenerationState>(chunk.size)
-
         val localFileSystem = VirtualFileManager.getInstance().getFileSystem(StandardFileSystems.FILE_PROTOCOL)
 
         val (codegenFactory, wholeBackendInput) = convertToIr(environment, result)
+        val diagnosticsReporter = DiagnosticReporterFactory.createReporter()
+
+        val codegenInputs = ArrayList<CodegenFactory.CodegenInput>(chunk.size)
 
         for (module in chunk) {
             ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
@@ -125,13 +126,20 @@ object KotlinToJVMBytecodeCompiler {
             val moduleConfiguration = projectConfiguration.applyModuleProperties(module, buildFile)
 
             val backendInput = codegenFactory.getModuleChunkBackendInput(wholeBackendInput, ktFiles)
-            outputs[module] = generate(environment, moduleConfiguration, result, ktFiles, module, codegenFactory, backendInput)
+            codegenInputs += runLowerings(
+                environment, moduleConfiguration, result, ktFiles, module, codegenFactory, backendInput, diagnosticsReporter
+            )
+        }
+
+        val outputs = ArrayList<GenerationState>(chunk.size)
+
+        for (input in codegenInputs) {
+            outputs += runCodegen(input, input.state, result.bindingContext, diagnosticsReporter, environment.configuration)
         }
 
         return writeOutputs(environment.project, projectConfiguration, chunk, outputs, mainClassFqName)
     }
 
-
     internal fun configureSourceRoots(configuration: CompilerConfiguration, chunk: List<Module>, buildFile: File? = null) {
         for (module in chunk) {
             val commonSources = getBuildFilePaths(buildFile, module.getCommonSourceFiles()).toSet()
@@ -240,7 +248,12 @@ object KotlinToJVMBytecodeCompiler {
         result.throwIfError()
 
         val (codegenFactory, backendInput) = convertToIr(environment, result)
-        return generate(environment, environment.configuration, result, environment.getSourceFiles(), null, codegenFactory, backendInput)
+        val diagnosticsReporter = DiagnosticReporterFactory.createReporter()
+        val input = runLowerings(
+            environment, environment.configuration, result, environment.getSourceFiles(), null, codegenFactory, backendInput,
+            diagnosticsReporter
+        )
+        return runCodegen(input, input.state, result.bindingContext, diagnosticsReporter, environment.configuration)
     }
 
     private fun convertToIr(environment: KotlinCoreEnvironment, result: AnalysisResult): Pair<CodegenFactory, CodegenFactory.BackendInput> {
@@ -325,7 +338,7 @@ object KotlinToJVMBytecodeCompiler {
         override fun toString() = "All files under: $directories"
     }
 
-    private fun generate(
+    private fun runLowerings(
         environment: KotlinCoreEnvironment,
         configuration: CompilerConfiguration,
         result: AnalysisResult,
@@ -333,9 +346,8 @@ object KotlinToJVMBytecodeCompiler {
         module: Module?,
         codegenFactory: CodegenFactory,
         backendInput: CodegenFactory.BackendInput,
-    ): GenerationState {
-        val diagnosticsReporter = DiagnosticReporterFactory.createReporter()
-
+        diagnosticsReporter: BaseDiagnosticsCollector,
+    ): CodegenFactory.CodegenInput {
         val state = GenerationState.Builder(
             environment.project,
             ClassBuilderFactories.BINARIES,
@@ -352,30 +364,41 @@ object KotlinToJVMBytecodeCompiler {
 
         ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
 
-        val performanceManager = environment.configuration.get(CLIConfigurationKeys.PERF_MANAGER)
-        performanceManager?.notifyGenerationStarted()
+        environment.configuration.get(CLIConfigurationKeys.PERF_MANAGER)?.notifyGenerationStarted()
 
         state.beforeCompile()
 
         ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
 
-        codegenFactory.generateModule(state, backendInput)
+        return codegenFactory.invokeLowerings(state, backendInput)
+    }
+
+    private fun runCodegen(
+        codegenInput: CodegenFactory.CodegenInput,
+        state: GenerationState,
+        bindingContext: BindingContext,
+        diagnosticsReporter: BaseDiagnosticsCollector,
+        configuration: CompilerConfiguration,
+    ): GenerationState {
+        ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
+        state.codegenFactory.invokeCodegen(codegenInput)
 
         CodegenFactory.doCheckCancelled(state)
         state.factory.done()
 
-        performanceManager?.notifyGenerationFinished()
+        configuration.get(CLIConfigurationKeys.PERF_MANAGER)?.notifyGenerationFinished()
 
         ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
 
+        val messageCollector = configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
         AnalyzerWithCompilerReport.reportDiagnostics(
             FilteredJvmDiagnostics(
                 state.collectedExtraJvmDiagnostics,
-                result.bindingContext.diagnostics
+                bindingContext.diagnostics
             ),
-            environment.messageCollector
+            messageCollector
         )
-        FirDiagnosticsCompilerResultsReporter.reportToMessageCollector(diagnosticsReporter, environment.messageCollector)
+        FirDiagnosticsCompilerResultsReporter.reportToMessageCollector(diagnosticsReporter, messageCollector)
 
         ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
         return state

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/cliCompilerUtils.kt

@@ -119,16 +119,16 @@ fun writeOutputs(
     project: Project?,
     projectConfiguration: CompilerConfiguration,
     chunk: List<Module>,
-    outputs: Map<Module, GenerationState>,
+    outputs: List<GenerationState>,
     mainClassFqName: FqName?
 ): Boolean {
     try {
-        for ((_, state) in outputs) {
+        for (state in outputs) {
             ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
             writeOutput(state.configuration, state.factory, mainClassFqName)
         }
     } finally {
-        outputs.values.forEach(GenerationState::destroy)
+        outputs.forEach(GenerationState::destroy)
     }
 
     if (projectConfiguration.getBoolean(JVMConfigurationKeys.COMPILE_JAVA)) {