FIR: new cli pipeline with LightTree support

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/K2JVMCompiler.kt

@@ -32,16 +32,15 @@ import org.jetbrains.kotlin.cli.common.messages.OutputMessageUtil
 import org.jetbrains.kotlin.cli.common.modules.ModuleBuilder
 import org.jetbrains.kotlin.cli.common.modules.ModuleChunk
 import org.jetbrains.kotlin.cli.common.profiling.ProfilingCompilerPerformanceManager
-import org.jetbrains.kotlin.cli.jvm.compiler.CompileEnvironmentUtil
-import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
-import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
-import org.jetbrains.kotlin.cli.jvm.compiler.KotlinToJVMBytecodeCompiler
+import org.jetbrains.kotlin.cli.jvm.compiler.*
+import org.jetbrains.kotlin.cli.jvm.compiler.pipeline.compileModulesUsingFrontendIrAndLightTree
+import org.jetbrains.kotlin.cli.jvm.compiler.pipeline.createProjectEnvironment
 import org.jetbrains.kotlin.cli.jvm.config.ClassicFrontendSpecificJvmConfigurationKeys
 import org.jetbrains.kotlin.codegen.CompilationException
 import org.jetbrains.kotlin.config.*
 import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
-import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.components.InlineConstTracker
+import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.load.java.JavaClassesTracker
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
@@ -156,25 +155,35 @@ class K2JVMCompiler : CLICompiler<K2JVMCompilerArguments>() {
             }
 
             val chunk = moduleChunk.modules
-            KotlinToJVMBytecodeCompiler.configureSourceRoots(configuration, chunk, buildFile)
-            val environment = createCoreEnvironment(
-                rootDisposable, configuration, messageCollector,
-                chunk.map { input -> input.getModuleName() + "-" + input.getModuleType() }.let { names ->
-                    names.singleOrNull() ?: names.joinToString()
+            configuration.configureSourceRoots(chunk, buildFile)
+
+            if (configuration.getBoolean(CommonConfigurationKeys.USE_FIR)) {
+                val projectEnvironment =
+                    createProjectEnvironment(configuration, rootDisposable, EnvironmentConfigFiles.JVM_CONFIG_FILES, messageCollector)
+
+                compileModulesUsingFrontendIrAndLightTree(
+                    projectEnvironment, configuration, messageCollector, buildFile, chunk
+                )
+            } else {
+                val environment = createCoreEnvironment(
+                    rootDisposable, configuration, messageCollector,
+                    chunk.map { input -> input.getModuleName() + "-" + input.getModuleType() }.let { names ->
+                        names.singleOrNull() ?: names.joinToString()
+                    }
+                ) ?: return COMPILATION_ERROR
+                environment.registerJavacIfNeeded(arguments).let {
+                    if (!it) return COMPILATION_ERROR
                 }
-            ) ?: return COMPILATION_ERROR
-            environment.registerJavacIfNeeded(arguments).let {
-                if (!it) return COMPILATION_ERROR
+
+                if (environment.getSourceFiles().isEmpty() && !arguments.allowNoSourceFiles && buildFile == null) {
+                    if (arguments.version) return OK
+
+                    messageCollector.report(ERROR, "No source files")
+                    return COMPILATION_ERROR
+                }
+
+                KotlinToJVMBytecodeCompiler.compileModules(environment, buildFile, chunk)
             }
-
-            if (environment.getSourceFiles().isEmpty() && !arguments.allowNoSourceFiles && buildFile == null) {
-                if (arguments.version) return OK
-
-                messageCollector.report(ERROR, "No source files")
-                return COMPILATION_ERROR
-            }
-
-            KotlinToJVMBytecodeCompiler.compileModules(environment, buildFile, chunk)
             return OK
         } catch (e: CompilationException) {
             messageCollector.report(

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/FirKotlinToJvmBytecodeCompiler.kt

@@ -88,13 +88,13 @@ object FirKotlinToJvmBytecodeCompiler {
         val isMultiModuleChunk = chunk.size > 1
 
         // TODO: run lowerings for all modules in the chunk, then run codegen for all modules.
-        val project = (projectEnvironment as? PsiBasedProjectEnvironment)?.project
+        val project = (projectEnvironment as? VfsBasedProjectEnvironment)?.project
         for (module in chunk) {
             val moduleConfiguration = projectConfiguration.applyModuleProperties(module, buildFile)
             val context = CompilationContext(
                 module,
                 module.getSourceFiles(
-                    allSources, (projectEnvironment as? PsiBasedProjectEnvironment)?.localFileSystem, isMultiModuleChunk, buildFile
+                    allSources, (projectEnvironment as? VfsBasedProjectEnvironment)?.localFileSystem, isMultiModuleChunk, buildFile
                 ),
                 projectEnvironment,
                 messageCollector,
@@ -112,7 +112,7 @@ object FirKotlinToJvmBytecodeCompiler {
         }
 
         val mainClassFqName: FqName? = runIf(chunk.size == 1 && projectConfiguration.get(JVMConfigurationKeys.OUTPUT_JAR) != null) {
-            findMainClass(outputs.single().first)
+            findMainClass(outputs.single().first.fir)
         }
 
         return writeOutputs(
@@ -179,7 +179,7 @@ object FirKotlinToJvmBytecodeCompiler {
             AnalyzerWithCompilerReport.reportSyntaxErrors(ktFile, messageCollector).isHasErrors or errorsFound
         }
 
-        var sourceScope = (projectEnvironment as PsiBasedProjectEnvironment).getSearchScopeByPsiFiles(ktFiles) +
+        var sourceScope = (projectEnvironment as VfsBasedProjectEnvironment).getSearchScopeByPsiFiles(ktFiles) +
                 projectEnvironment.getSearchScopeForProjectJavaSources()
 
         var librariesScope = projectEnvironment.getSearchScopeForProjectLibraries()
@@ -307,7 +307,7 @@ object FirKotlinToJvmBytecodeCompiler {
         )
 
         val generationState = GenerationState.Builder(
-            (projectEnvironment as PsiBasedProjectEnvironment).project, ClassBuilderFactories.BINARIES,
+            (projectEnvironment as VfsBasedProjectEnvironment).project, ClassBuilderFactories.BINARIES,
             moduleFragment.descriptor, dummyBindingContext, ktFiles,
             moduleConfiguration
         ).codegenFactory(
@@ -366,38 +366,38 @@ object FirKotlinToJvmBytecodeCompiler {
         val firExtensionRegistrars: List<FirExtensionRegistrar>,
         val irGenerationExtensions: Collection<IrGenerationExtension>
     )
-
-    private fun findMainClass(firResult: FirResult): FqName? {
-        // TODO: replace with proper main function detector, KT-44557
-        val compatibleClasses = mutableListOf<FqName>()
-        val visitor = object : FirVisitorVoid() {
-            lateinit var file: FirFile
-
-            override fun visitElement(element: FirElement) {}
-
-            override fun visitFile(file: FirFile) {
-                this.file = file
-                file.acceptChildren(this)
-            }
-
-            override fun visitSimpleFunction(simpleFunction: FirSimpleFunction) {
-                if (simpleFunction.name.asString() != "main") return
-                if (simpleFunction.typeParameters.isNotEmpty()) return
-                when (simpleFunction.valueParameters.size) {
-                    0 -> {}
-                    1 -> {
-                        val parameterType = simpleFunction.valueParameters.single().returnTypeRef.coneType
-                        if (!parameterType.isArrayType || parameterType.arrayElementType()?.isString != true) return
-                    }
-                    else -> return
-                }
-
-                compatibleClasses += FqName.fromSegments(
-                    file.packageFqName.pathSegments().map { it.asString() } + "${file.name.removeSuffix(".kt").capitalize()}Kt"
-                )
-            }
-        }
-        firResult.fir.forEach { it.accept(visitor) }
-        return compatibleClasses.singleOrNull()
-    }
+}
+
+internal fun findMainClass(fir: List<FirFile>): FqName? {
+    // TODO: replace with proper main function detector, KT-44557
+    val compatibleClasses = mutableListOf<FqName>()
+    val visitor = object : FirVisitorVoid() {
+        lateinit var file: FirFile
+
+        override fun visitElement(element: FirElement) {}
+
+        override fun visitFile(file: FirFile) {
+            this.file = file
+            file.acceptChildren(this)
+        }
+
+        override fun visitSimpleFunction(simpleFunction: FirSimpleFunction) {
+            if (simpleFunction.name.asString() != "main") return
+            if (simpleFunction.typeParameters.isNotEmpty()) return
+            when (simpleFunction.valueParameters.size) {
+                0 -> {}
+                1 -> {
+                    val parameterType = simpleFunction.valueParameters.single().returnTypeRef.coneType
+                    if (!parameterType.isArrayType || parameterType.arrayElementType()?.isString != true) return
+                }
+                else -> return
+            }
+
+            compatibleClasses += FqName.fromSegments(
+                file.packageFqName.pathSegments().map { it.asString() } + "${file.name.removeSuffix(".kt").capitalize()}Kt"
+            )
+        }
+    }
+    fir.forEach { it.accept(visitor) }
+    return compatibleClasses.singleOrNull()
 }

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt

@@ -198,39 +198,11 @@ class KotlinCoreEnvironment private constructor(
     val configuration: CompilerConfiguration = initialConfiguration.apply { setupJdkClasspathRoots(configFiles) }.copy()
 
     init {
-        PersistentFSConstants::class.java.getDeclaredField("ourMaxIntellisenseFileSize")
-            .apply { isAccessible = true }
-            .setInt(null, FileUtilRt.LARGE_FOR_CONTENT_LOADING)
-
+        projectEnvironment.configureProjectEnvironment(configuration, configFiles)
         val project = projectEnvironment.project
-
-        val messageCollector = configuration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
-
-        if (configuration.getBoolean(JVMConfigurationKeys.USE_FAST_JAR_FILE_SYSTEM)) {
-            messageCollector?.report(
-                STRONG_WARNING,
-                "Using new faster version of JAR FS: it should make your build faster, but the new implementation is experimental"
-            )
-        }
-
-        (projectEnvironment as? ProjectEnvironment)?.registerExtensionsFromPlugins(configuration)
-        // otherwise consider that project environment is properly configured before passing to the environment
-        // TODO: consider some asserts to check important extension points
-
         project.registerService(DeclarationProviderFactoryService::class.java, CliDeclarationProviderFactoryService(sourceFiles))
 
-        val isJvm = configFiles == EnvironmentConfigFiles.JVM_CONFIG_FILES
-        project.registerService(ModuleVisibilityManager::class.java, CliModuleVisibilityManagerImpl(isJvm))
-
-        registerProjectServicesForCLI(projectEnvironment)
-
-        registerProjectServices(projectEnvironment.project)
-
-        for (extension in CompilerConfigurationExtension.getInstances(project)) {
-            extension.updateConfiguration(configuration)
-        }
-
-        sourceFiles += createKtFiles(project)
+        sourceFiles += createSourceFilesFromSourceRoots(configuration, project, getSourceRootsCheckingForDuplicates())
 
         collectAdditionalSources(project)
 
@@ -238,6 +210,8 @@ class KotlinCoreEnvironment private constructor(
 
         val javaFileManager = ServiceManager.getService(project, CoreJavaFileManager::class.java) as KotlinCliJavaFileManagerImpl
 
+        val messageCollector = configuration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
+
         val jdkHome = configuration.get(JVMConfigurationKeys.JDK_HOME)
         val releaseTarget = configuration.get(JVMConfigurationKeys.JDK_RELEASE)
         val javaModuleFinder = CliJavaModuleFinder(
@@ -449,9 +423,6 @@ class KotlinCoreEnvironment private constructor(
 
     fun getSourceFiles(): List<KtFile> = sourceFiles
 
-    private fun createKtFiles(project: Project): List<KtFile> =
-        createSourceFilesFromSourceRoots(configuration, project, getSourceRootsCheckingForDuplicates())
-
     internal fun report(severity: CompilerMessageSeverity, message: String) = configuration.report(severity, message)
 
     companion object {
@@ -585,6 +556,38 @@ class KotlinCoreEnvironment private constructor(
             }
         }
 
+        @JvmStatic
+        fun ProjectEnvironment.configureProjectEnvironment(
+            configuration: CompilerConfiguration,
+            configFiles: EnvironmentConfigFiles
+        ) {
+            PersistentFSConstants::class.java.getDeclaredField("ourMaxIntellisenseFileSize")
+                .apply { isAccessible = true }
+                .setInt(null, FileUtilRt.LARGE_FOR_CONTENT_LOADING)
+
+            if (configuration.getBoolean(JVMConfigurationKeys.USE_FAST_JAR_FILE_SYSTEM)) {
+                configuration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)?.report(
+                    STRONG_WARNING,
+                    "Using new faster version of JAR FS: it should make your build faster, but the new implementation is experimental"
+                )
+            }
+
+            registerExtensionsFromPlugins(configuration)
+            // otherwise consider that project environment is properly configured before passing to the environment
+            // TODO: consider some asserts to check important extension points
+
+            val isJvm = configFiles == EnvironmentConfigFiles.JVM_CONFIG_FILES
+            project.registerService(ModuleVisibilityManager::class.java, CliModuleVisibilityManagerImpl(isJvm))
+
+            registerProjectServicesForCLI(this)
+
+            registerProjectServices(project)
+
+            for (extension in CompilerConfigurationExtension.getInstances(project)) {
+                extension.updateConfiguration(configuration)
+            }
+        }
+
         private fun createApplicationEnvironment(
             parentDisposable: Disposable, configuration: CompilerConfiguration, unitTestMode: Boolean
         ): KotlinCoreApplicationEnvironment {
@@ -723,7 +726,7 @@ class KotlinCoreEnvironment private constructor(
             }
         }
 
-        private fun registerProjectServicesForCLI(@Suppress("UNUSED_PARAMETER") projectEnvironment: JavaCoreProjectEnvironment) {
+        fun registerProjectServicesForCLI(@Suppress("UNUSED_PARAMETER") projectEnvironment: JavaCoreProjectEnvironment) {
             /**
              * Note that Kapt may restart code analysis process, and CLI services should be aware of that.
              * Use PsiManager.getModificationTracker() to ensure that all the data you cached is still valid.

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.kt

@@ -85,7 +85,7 @@ object KotlinToJVMBytecodeCompiler {
         if (projectConfiguration.getBoolean(CommonConfigurationKeys.USE_FIR)) {
             val extendedAnalysisMode = projectConfiguration.getBoolean(CommonConfigurationKeys.USE_FIR_EXTENDED_CHECKERS)
             val projectEnvironment =
-                PsiBasedProjectEnvironment(
+                VfsBasedProjectEnvironment(
                     environment.project,
                     VirtualFileManager.getInstance().getFileSystem(StandardFileSystems.FILE_PROTOCOL)
                 ) { environment.createPackagePartProvider(it) }
@@ -139,52 +139,6 @@ object KotlinToJVMBytecodeCompiler {
         return writeOutputs(environment.project, projectConfiguration, chunk, outputs, mainClassFqName)
     }
 
-    internal fun configureSourceRoots(configuration: CompilerConfiguration, chunk: List<Module>, buildFile: File? = null) {
-        for (module in chunk) {
-            val commonSources = getBuildFilePaths(buildFile, module.getCommonSourceFiles()).toSet()
-
-            for (path in getBuildFilePaths(buildFile, module.getSourceFiles())) {
-                configuration.addKotlinSourceRoot(path, isCommon = path in commonSources)
-            }
-        }
-
-        for (module in chunk) {
-            for ((path, packagePrefix) in module.getJavaSourceRoots()) {
-                configuration.addJavaSourceRoot(File(path), packagePrefix)
-            }
-        }
-
-        val isJava9Module = chunk.any { module ->
-            module.getJavaSourceRoots().any { (path, packagePrefix) ->
-                val file = File(path)
-                packagePrefix == null &&
-                        (file.name == PsiJavaModule.MODULE_INFO_FILE ||
-                                (file.isDirectory && file.listFiles()!!.any { it.name == PsiJavaModule.MODULE_INFO_FILE }))
-            }
-        }
-
-        for (module in chunk) {
-            for (classpathRoot in module.getClasspathRoots()) {
-                if (isJava9Module) {
-                    configuration.add(CLIConfigurationKeys.CONTENT_ROOTS, JvmModulePathRoot(File(classpathRoot)))
-                }
-                configuration.add(CLIConfigurationKeys.CONTENT_ROOTS, JvmClasspathRoot(File(classpathRoot)))
-            }
-        }
-
-        for (module in chunk) {
-            val modularJdkRoot = module.modularJdkRoot
-            if (modularJdkRoot != null) {
-                // We use the SDK of the first module in the chunk, which is not always correct because some other module in the chunk
-                // might depend on a different SDK
-                configuration.put(JVMConfigurationKeys.JDK_HOME, File(modularJdkRoot))
-                break
-            }
-        }
-
-        configuration.addAll(JVMConfigurationKeys.MODULES, chunk)
-    }
-
     fun compileBunchOfSources(environment: KotlinCoreEnvironment): Boolean {
         val moduleVisibilityManager = ModuleVisibilityManager.SERVICE.getInstance(environment.project)
 
@@ -427,3 +381,49 @@ object KotlinToJVMBytecodeCompiler {
         return state
     }
 }
+
+internal fun CompilerConfiguration.configureSourceRoots(chunk: List<Module>, buildFile: File? = null) {
+    for (module in chunk) {
+        val commonSources = getBuildFilePaths(buildFile, module.getCommonSourceFiles()).toSet()
+
+        for (path in getBuildFilePaths(buildFile, module.getSourceFiles())) {
+            addKotlinSourceRoot(path, isCommon = path in commonSources)
+        }
+    }
+
+    for (module in chunk) {
+        for ((path, packagePrefix) in module.getJavaSourceRoots()) {
+            addJavaSourceRoot(File(path), packagePrefix)
+        }
+    }
+
+    val isJava9Module = chunk.any { module ->
+        module.getJavaSourceRoots().any { (path, packagePrefix) ->
+            val file = File(path)
+            packagePrefix == null &&
+                    (file.name == PsiJavaModule.MODULE_INFO_FILE ||
+                            (file.isDirectory && file.listFiles()!!.any { it.name == PsiJavaModule.MODULE_INFO_FILE }))
+        }
+    }
+
+    for (module in chunk) {
+        for (classpathRoot in module.getClasspathRoots()) {
+            if (isJava9Module) {
+                add(CLIConfigurationKeys.CONTENT_ROOTS, JvmModulePathRoot(File(classpathRoot)))
+            }
+            add(CLIConfigurationKeys.CONTENT_ROOTS, JvmClasspathRoot(File(classpathRoot)))
+        }
+    }
+
+    for (module in chunk) {
+        val modularJdkRoot = module.modularJdkRoot
+        if (modularJdkRoot != null) {
+            // We use the SDK of the first module in the chunk, which is not always correct because some other module in the chunk
+            // might depend on a different SDK
+            put(JVMConfigurationKeys.JDK_HOME, File(modularJdkRoot))
+            break
+        }
+    }
+
+    addAll(JVMConfigurationKeys.MODULES, chunk)
+}

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/VfsBasedProjectEnvironment.kt

@@ -42,11 +42,12 @@ class PsiBasedProjectFileSearchScope(val psiSearchScope: GlobalSearchScope) : Ab
         PsiBasedProjectFileSearchScope(GlobalSearchScope.notScope(psiSearchScope))
 }
 
-class PsiBasedProjectEnvironment(
+open class VfsBasedProjectEnvironment(
     val project: Project,
     val localFileSystem: VirtualFileSystem,
-    val getPackagePartProviderFn: (GlobalSearchScope) -> PackagePartProvider
+    private val getPackagePartProviderFn: (GlobalSearchScope) -> PackagePartProvider
 ) : AbstractProjectEnvironment {
+
     override fun getKotlinClassFinder(fileSearchScope: AbstractProjectFileSearchScope): KotlinClassFinder =
         VirtualFileFinderFactory.getInstance(project).create(fileSearchScope.asPsiSearchScope())
 
@@ -105,7 +106,7 @@ private fun AbstractProjectFileSearchScope.asPsiSearchScope() =
     }
 
 fun KotlinCoreEnvironment.toAbstractProjectEnvironment(): AbstractProjectEnvironment =
-    PsiBasedProjectEnvironment(
+    VfsBasedProjectEnvironment(
         project, VirtualFileManager.getInstance().getFileSystem(StandardFileSystems.FILE_PROTOCOL),
         { createPackagePartProvider(it) }
     )

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/coreEnvironmentUtils.kt

@@ -30,22 +30,20 @@ fun CompilerConfiguration.report(severity: CompilerMessageSeverity, message: Str
     get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)?.report(severity, message, location)
 }
 
-fun createSourceFilesFromSourceRoots(
+inline fun List<KotlinSourceRoot>.forAllFiles(
     configuration: CompilerConfiguration,
     project: Project,
-    sourceRoots: List<KotlinSourceRoot>,
-    reportLocation: CompilerMessageLocation? = null
-): MutableList<KtFile> {
+    reportLocation: CompilerMessageLocation? = null,
+    body: (VirtualFile, Boolean) -> Unit
+) {
     val localFileSystem = VirtualFileManager.getInstance()
         .getFileSystem(StandardFileSystems.FILE_PROTOCOL)
-    val psiManager = PsiManager.getInstance(project)
 
     val processedFiles = hashSetOf<VirtualFile>()
-    val result = mutableListOf<KtFile>()
 
     val virtualFileCreator = PreprocessedFileCreator(project)
 
-    for ((sourceRootPath, isCommon) in sourceRoots) {
+    for ((sourceRootPath, isCommon) in this) {
         val vFile = localFileSystem.findFileByPath(sourceRootPath)
         if (vFile == null) {
             val message = "Source file or directory not found: $sourceRootPath"
@@ -70,17 +68,28 @@ fun createSourceFilesFromSourceRoots(
 
             val virtualFile = localFileSystem.findFileByPath(file.absolutePath)?.let(virtualFileCreator::create)
             if (virtualFile != null && processedFiles.add(virtualFile)) {
-                val psiFile = psiManager.findFile(virtualFile)
-                if (psiFile is KtFile) {
-                    result.add(psiFile)
-                    if (isCommon) {
-                        psiFile.isCommonSource = true
-                    }
-                }
+                body(virtualFile, isCommon)
             }
         }
     }
+}
 
+fun createSourceFilesFromSourceRoots(
+    configuration: CompilerConfiguration,
+    project: Project,
+    sourceRoots: List<KotlinSourceRoot>,
+    reportLocation: CompilerMessageLocation? = null
+): MutableList<KtFile> {
+    val psiManager = PsiManager.getInstance(project)
+    val result = mutableListOf<KtFile>()
+    sourceRoots.forAllFiles(configuration, project, reportLocation) { virtualFile, isCommon ->
+        psiManager.findFile(virtualFile)?.let {
+            if (it is KtFile) {
+                it.isCommonSource = isCommon
+                result.add(it)
+            }
+        }
+    }
     return result
 }
 

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt

@@ -0,0 +1,445 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cli.jvm.compiler.pipeline
+
+import com.intellij.core.CoreJavaFileManager
+import com.intellij.ide.highlighter.JavaFileType
+import com.intellij.openapi.Disposable
+import com.intellij.openapi.components.ServiceManager
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.StandardFileSystems
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.openapi.vfs.VirtualFileManager
+import com.intellij.openapi.vfs.VirtualFileSystem
+import com.intellij.psi.PsiManager
+import com.intellij.psi.search.GlobalSearchScope
+import com.intellij.util.io.URLUtil
+import org.jetbrains.kotlin.analyzer.common.CommonPlatformAnalyzerServices
+import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
+import org.jetbrains.kotlin.backend.jvm.JvmGeneratorExtensionsImpl
+import org.jetbrains.kotlin.backend.jvm.JvmIrCodegenFactory
+import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
+import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots
+import org.jetbrains.kotlin.cli.common.fir.FirDiagnosticsCompilerResultsReporter
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.cli.jvm.compiler.*
+import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment.Companion.configureProjectEnvironment
+import org.jetbrains.kotlin.cli.jvm.config.*
+import org.jetbrains.kotlin.cli.jvm.index.JavaRoot
+import org.jetbrains.kotlin.cli.jvm.index.JvmDependenciesDynamicCompoundIndex
+import org.jetbrains.kotlin.cli.jvm.index.JvmDependenciesIndexImpl
+import org.jetbrains.kotlin.cli.jvm.index.SingleJavaFileRootsIndex
+import org.jetbrains.kotlin.cli.jvm.modules.CliJavaModuleFinder
+import org.jetbrains.kotlin.cli.jvm.modules.CliJavaModuleResolver
+import org.jetbrains.kotlin.codegen.ClassBuilderFactories
+import org.jetbrains.kotlin.codegen.CodegenFactory
+import org.jetbrains.kotlin.codegen.state.GenerationState
+import org.jetbrains.kotlin.config.*
+import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory
+import org.jetbrains.kotlin.fir.DependencyListForCliModule
+import org.jetbrains.kotlin.fir.FirSession
+import org.jetbrains.kotlin.fir.backend.jvm.FirJvmBackendClassResolver
+import org.jetbrains.kotlin.fir.backend.jvm.FirJvmBackendExtension
+import org.jetbrains.kotlin.fir.checkers.registerExtendedCommonCheckers
+import org.jetbrains.kotlin.fir.java.FirProjectSessionProvider
+import org.jetbrains.kotlin.fir.moduleData
+import org.jetbrains.kotlin.fir.pipeline.buildFirViaLightTree
+import org.jetbrains.kotlin.fir.pipeline.convertToIr
+import org.jetbrains.kotlin.fir.pipeline.runCheckers
+import org.jetbrains.kotlin.fir.pipeline.runResolution
+import org.jetbrains.kotlin.fir.session.FirSessionFactory
+import org.jetbrains.kotlin.fir.session.environment.AbstractProjectEnvironment
+import org.jetbrains.kotlin.fir.session.environment.AbstractProjectFileSearchScope
+import org.jetbrains.kotlin.load.kotlin.MetadataFinderFactory
+import org.jetbrains.kotlin.load.kotlin.PackagePartProvider
+import org.jetbrains.kotlin.load.kotlin.VirtualFileFinderFactory
+import org.jetbrains.kotlin.load.kotlin.incremental.IncrementalPackagePartProvider
+import org.jetbrains.kotlin.modules.Module
+import org.jetbrains.kotlin.modules.TargetId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.platform.CommonPlatforms
+import org.jetbrains.kotlin.platform.TargetPlatform
+import org.jetbrains.kotlin.platform.jvm.JvmPlatforms
+import org.jetbrains.kotlin.progress.ProgressIndicatorAndCompilationCanceledStatus
+import org.jetbrains.kotlin.resolve.ModuleAnnotationsResolver
+import org.jetbrains.kotlin.resolve.PlatformDependentAnalyzerServices
+import org.jetbrains.kotlin.resolve.jvm.modules.JavaModuleResolver
+import org.jetbrains.kotlin.resolve.jvm.platform.JvmPlatformAnalyzerServices
+import org.jetbrains.kotlin.utils.addToStdlib.runIf
+import java.io.File
+
+fun compileModulesUsingFrontendIrAndLightTree(
+    projectEnvironment: AbstractProjectEnvironment,
+    compilerConfiguration: CompilerConfiguration,
+    messageCollector: MessageCollector,
+    buildFile: File?,
+    chunk: List<Module>
+): Boolean {
+    require(projectEnvironment is VfsBasedProjectEnvironment) // TODO: abstract away this requirement
+    ProgressIndicatorAndCompilationCanceledStatus.checkCanceled()
+
+    messageCollector.report(
+        CompilerMessageSeverity.STRONG_WARNING,
+        "ATTENTION!\n This build uses in-dev FIR: \n  -Xuse-fir"
+    )
+
+    val outputs = mutableMapOf<Module, GenerationState>()
+    var mainClassFqName: FqName? = null
+
+    for (module in chunk) {
+        val moduleConfiguration = compilerConfiguration.copy().applyModuleProperties(module, buildFile).apply {
+            addAll(JVMConfigurationKeys.FRIEND_PATHS, module.getFriendPaths())
+        }
+        val platformSources = linkedSetOf<File>()
+        val commonSources = linkedSetOf<File>()
+
+        // !!
+        compilerConfiguration.kotlinSourceRoots.forAllFiles(compilerConfiguration, projectEnvironment.project) { virtualFile, isCommon ->
+            val file = File(virtualFile.canonicalPath ?: virtualFile.path)
+            if (!file.isFile) error("TODO: better error: file not found $virtualFile")
+            if (isCommon) commonSources.add(file)
+            else platformSources.add(file)
+        }
+
+        val diagnosticsReporter = DiagnosticReporterFactory.createReporter()
+
+        val (moduleMainClassName, generationState) = compileModule(
+            ModuleCompilerInput(
+                TargetId(module),
+                CommonPlatforms.defaultCommonPlatform, commonSources,
+                JvmPlatforms.unspecifiedJvmPlatform, platformSources,
+                moduleConfiguration
+            ),
+            ModuleCompilerEnvironment(projectEnvironment, diagnosticsReporter)
+        )
+        FirDiagnosticsCompilerResultsReporter.reportToMessageCollector(
+            diagnosticsReporter,
+            messageCollector,
+            moduleConfiguration.getBoolean(CLIConfigurationKeys.RENDER_DIAGNOSTIC_INTERNAL_NAME)
+        )
+        outputs[module] = generationState
+
+        // TODO: consider what to do if many modules contain main class
+        if (mainClassFqName == null) {
+            mainClassFqName = moduleMainClassName
+        }
+    }
+
+    return writeOutputs(
+        (projectEnvironment as? VfsBasedProjectEnvironment)?.project,
+        compilerConfiguration,
+        chunk,
+        outputs,
+        mainClassFqName
+    )
+}
+
+fun compileModule(
+    input: ModuleCompilerInput,
+    environment: ModuleCompilerEnvironment
+): ModuleCompilerOutput {
+    var sourcesScope = environment.projectEnvironment.getSearchScopeByIoFiles(input.platformSources) //!!
+    val sessionProvider = FirProjectSessionProvider()
+    val extendedAnalysisMode = input.configuration.getBoolean(CommonConfigurationKeys.USE_FIR_EXTENDED_CHECKERS)
+
+    val commonSession = runIf(
+        input.commonSources.isNotEmpty() && input.configuration.languageVersionSettings.supportsFeature(LanguageFeature.MultiPlatformProjects)
+    ) {
+        val commonSourcesScope = environment.projectEnvironment.getSearchScopeByIoFiles(input.commonSources) //!!
+        sourcesScope -= commonSourcesScope
+        createSession(
+            "${input.targetId.name}-common",
+            input.commonPlatform,
+            input.configuration,
+            environment.projectEnvironment,
+            commonSourcesScope,
+            CommonPlatformAnalyzerServices,
+            sessionProvider,
+            extendedAnalysisMode,
+            needRegisterJavaElementFinder = false
+        )
+    }
+
+    val session = createSession(
+        input.targetId.name,
+        input.platform,
+        input.configuration,
+        environment.projectEnvironment,
+        sourcesScope,
+        JvmPlatformAnalyzerServices,
+        sessionProvider,
+        extendedAnalysisMode,
+        needRegisterJavaElementFinder = true
+    ) {
+        if (commonSession != null) {
+            sourceDependsOnDependencies(listOf(commonSession.moduleData))
+        }
+        friendDependencies(input.configuration[JVMConfigurationKeys.FRIEND_PATHS] ?: emptyList())
+    }
+
+    // raw fir
+    val commonRawFir = commonSession?.buildFirViaLightTree(input.commonSources)
+    val rawFir = session.buildFirViaLightTree(input.platformSources)
+
+    // resolution
+    commonSession?.apply {
+        val (commonScopeSession, commonFir) = runResolution(commonRawFir!!)
+        // TODO: find out what to do with commonFir
+        runCheckers(commonScopeSession, commonFir, environment.diagnosticsReporter)
+    }
+
+    val (scopeSession, fir) = session.runResolution(rawFir)
+    // checkers
+    session.runCheckers(scopeSession, fir, environment.diagnosticsReporter)
+
+    val mainClassFqName: FqName? = runIf(input.configuration.get(JVMConfigurationKeys.OUTPUT_JAR) != null) {
+        findMainClass(fir)
+    }
+
+    val extensions = JvmGeneratorExtensionsImpl(input.configuration)
+
+    // fir2ir
+    val irGenerationExtensions = (environment.projectEnvironment as? VfsBasedProjectEnvironment)?.project?.let { IrGenerationExtension.getInstances(it) }
+    val (irModuleFragment, symbolTable, components) = session.convertToIr(scopeSession, fir, extensions, irGenerationExtensions ?: emptyList())
+
+    // IR
+    val codegenFactory = JvmIrCodegenFactory(
+        input.configuration,
+        input.configuration.get(CLIConfigurationKeys.PHASE_CONFIG),
+        jvmGeneratorExtensions = extensions
+    )
+    val dummyBindingContext = NoScopeRecordCliBindingTrace().bindingContext
+
+    val generationState = GenerationState.Builder(
+        (environment.projectEnvironment as VfsBasedProjectEnvironment).project, ClassBuilderFactories.BINARIES,
+        irModuleFragment.descriptor, dummyBindingContext, emptyList()/* !! */,
+        input.configuration
+    ).codegenFactory(
+        codegenFactory
+    ).targetId(
+        input.targetId
+    ).moduleName(
+        input.targetId.name
+    ).outDirectory(
+        input.configuration[JVMConfigurationKeys.OUTPUT_DIRECTORY]
+    ).onIndependentPartCompilationEnd(
+        createOutputFilesFlushingCallbackIfPossible(input.configuration)
+    ).isIrBackend(
+        true
+    ).jvmBackendClassResolver(
+        FirJvmBackendClassResolver(components)
+    ).diagnosticReporter(
+        environment.diagnosticsReporter
+    ).build()
+
+    generationState.beforeCompile()
+    codegenFactory.generateModuleInFrontendIRMode(
+        generationState, irModuleFragment, symbolTable, extensions, FirJvmBackendExtension(session, components)
+    )
+    CodegenFactory.doCheckCancelled(generationState)
+    generationState.factory.done()
+
+    return ModuleCompilerOutput(mainClassFqName, generationState)
+}
+
+fun createSession(
+    name: String,
+    platform: TargetPlatform,
+    moduleConfiguration: CompilerConfiguration,
+    projectEnvironment: AbstractProjectEnvironment,
+    sourceScope: AbstractProjectFileSearchScope,
+    analyzerServices: PlatformDependentAnalyzerServices,
+    sessionProvider: FirProjectSessionProvider?,
+    extendedAnalysisMode: Boolean,
+    needRegisterJavaElementFinder: Boolean,
+    dependenciesConfigurator: DependencyListForCliModule.Builder.() -> Unit = {},
+): FirSession {
+    var librariesScope = projectEnvironment.getSearchScopeForProjectLibraries()
+
+    val providerAndScopeForIncrementalCompilation =
+        createComponentsForIncrementalCompilation(moduleConfiguration, projectEnvironment, sourceScope)?.also {
+            librariesScope -= it.scope
+        }
+
+    return FirSessionFactory.createSessionWithDependencies(
+        Name.identifier(name),
+        platform,
+        analyzerServices,
+        externalSessionProvider = sessionProvider,
+        projectEnvironment,
+        moduleConfiguration.languageVersionSettings,
+        sourceScope,
+        librariesScope,
+        lookupTracker = moduleConfiguration.get(CommonConfigurationKeys.LOOKUP_TRACKER),
+        providerAndScopeForIncrementalCompilation,
+        extensionRegistrars = (projectEnvironment as? VfsBasedProjectEnvironment)?.let { FirExtensionRegistrar.getInstances(it.project) }
+            ?: emptyList(),
+        needRegisterJavaElementFinder = needRegisterJavaElementFinder,
+        dependenciesConfigurator = {
+            dependencies(moduleConfiguration.jvmClasspathRoots.map { it.toPath() })
+            dependencies(moduleConfiguration.jvmModularRoots.map { it.toPath() })
+            friendDependencies(moduleConfiguration[JVMConfigurationKeys.FRIEND_PATHS] ?: emptyList())
+            dependenciesConfigurator()
+        }
+    ) {
+        if (extendedAnalysisMode) {
+            registerExtendedCommonCheckers()
+        }
+    }
+}
+
+private fun createComponentsForIncrementalCompilation(
+    compilerConfiguration: CompilerConfiguration,
+    projectEnvironment: AbstractProjectEnvironment,
+    sourceScope: AbstractProjectFileSearchScope
+): FirSessionFactory.ProviderAndScopeForIncrementalCompilation? {
+    val targetIds = compilerConfiguration.get(JVMConfigurationKeys.MODULES)?.map(::TargetId)
+    val incrementalComponents = compilerConfiguration.get(JVMConfigurationKeys.INCREMENTAL_COMPILATION_COMPONENTS)
+    if (targetIds == null || incrementalComponents == null) return null
+    val directoryWithIncrementalPartsFromPreviousCompilation =
+        compilerConfiguration[JVMConfigurationKeys.OUTPUT_DIRECTORY]
+            ?: return null
+    val incrementalCompilationScope = directoryWithIncrementalPartsFromPreviousCompilation.walk()
+        .filter { it.extension == "class" }
+        .let { projectEnvironment.getSearchScopeByIoFiles(it.asIterable()) }
+        .takeIf { !it.isEmpty }
+        ?: return null
+    val packagePartProvider = IncrementalPackagePartProvider(
+        projectEnvironment.getPackagePartProvider(sourceScope),
+        targetIds.map(incrementalComponents::getIncrementalCache)
+    )
+    return FirSessionFactory.ProviderAndScopeForIncrementalCompilation(packagePartProvider, incrementalCompilationScope)
+}
+
+private class ProjectEnvironmentWithCoreEnvironmentEmulation(
+    project: Project,
+    localFileSystem: VirtualFileSystem,
+    getPackagePartProviderFn: (GlobalSearchScope) -> PackagePartProvider,
+    val initialRoots: List<JavaRoot>,
+    val configuration: CompilerConfiguration
+) : VfsBasedProjectEnvironment(project, localFileSystem, getPackagePartProviderFn) {
+
+    val packagePartProviders = mutableListOf<JvmPackagePartProvider>()
+
+    override fun getPackagePartProvider(fileSearchScope: AbstractProjectFileSearchScope): PackagePartProvider {
+        return super.getPackagePartProvider(fileSearchScope).also {
+            (it as? JvmPackagePartProvider)?.run {
+                (it as? JvmPackagePartProvider)?.run {
+                    addRoots(initialRoots, configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY))
+                    packagePartProviders += this
+                    (ModuleAnnotationsResolver.getInstance(project) as CliModuleAnnotationsResolver).addPackagePartProvider(this)
+                }
+            }
+        }
+    }
+}
+
+fun createProjectEnvironment(
+    configuration: CompilerConfiguration,
+    parentDisposable: Disposable,
+    configFiles: EnvironmentConfigFiles,
+    messageCollector: MessageCollector
+): VfsBasedProjectEnvironment {
+    setupIdeaStandaloneExecution()
+    val appEnv = KotlinCoreEnvironment.getOrCreateApplicationEnvironmentForProduction(parentDisposable, configuration)
+    // TODO: get rid of projEnv too - seems that all needed components could be easily extracted
+    val projectEnvironment = KotlinCoreEnvironment.ProjectEnvironment(parentDisposable, appEnv, configuration)
+
+    projectEnvironment.configureProjectEnvironment(configuration, configFiles)
+
+    val project = projectEnvironment.project
+    val localFileSystem = VirtualFileManager.getInstance().getFileSystem(StandardFileSystems.FILE_PROTOCOL)
+
+    val javaFileManager = ServiceManager.getService(project, CoreJavaFileManager::class.java) as KotlinCliJavaFileManagerImpl
+
+    val releaseTarget = configuration.get(JVMConfigurationKeys.JDK_RELEASE)
+
+    val javaModuleFinder =
+        CliJavaModuleFinder(configuration.get(JVMConfigurationKeys.JDK_HOME), messageCollector, javaFileManager, project, releaseTarget)
+
+    val outputDirectory =
+        configuration.get(JVMConfigurationKeys.MODULES)?.singleOrNull()?.getOutputDirectory()
+            ?: configuration.get(JVMConfigurationKeys.OUTPUT_DIRECTORY)?.absolutePath
+
+    val classpathRootsResolver = ClasspathRootsResolver(
+        PsiManager.getInstance(project),
+        messageCollector,
+        configuration.getList(JVMConfigurationKeys.ADDITIONAL_JAVA_MODULES),
+        { contentRootToVirtualFile(it, localFileSystem, projectEnvironment.jarFileSystem, messageCollector) },
+        javaModuleFinder,
+        !configuration.getBoolean(CLIConfigurationKeys.ALLOW_KOTLIN_PACKAGE),
+        outputDirectory?.let { localFileSystem.findFileByPath(it) },
+        javaFileManager,
+        releaseTarget
+    )
+
+    val (initialRoots, javaModules) =
+        classpathRootsResolver.convertClasspathRoots(configuration.getList(CLIConfigurationKeys.CONTENT_ROOTS))
+
+    val (roots, singleJavaFileRoots) =
+        initialRoots.partition { (file) -> file.isDirectory || file.extension != JavaFileType.DEFAULT_EXTENSION }
+
+    // REPL and kapt2 update classpath dynamically
+    val rootsIndex = JvmDependenciesDynamicCompoundIndex().apply {
+        addIndex(JvmDependenciesIndexImpl(roots))
+        indexedRoots.forEach {
+            projectEnvironment.addSourcesToClasspath(it.file)
+//            javaFileManager.addToClasspath(it.file)
+        }
+    }
+
+    project.registerService(
+        JavaModuleResolver::class.java,
+        CliJavaModuleResolver(classpathRootsResolver.javaModuleGraph, javaModules, javaModuleFinder.systemModules.toList(), project)
+    )
+
+    val finderFactory = CliVirtualFileFinderFactory(rootsIndex, releaseTarget != null)
+    project.registerService(MetadataFinderFactory::class.java, finderFactory)
+    project.registerService(VirtualFileFinderFactory::class.java, finderFactory)
+
+    return ProjectEnvironmentWithCoreEnvironmentEmulation(
+        project,
+        localFileSystem,
+        { JvmPackagePartProvider(configuration.languageVersionSettings, it) },
+        initialRoots, configuration
+    ).also {
+        javaFileManager.initialize(
+            rootsIndex,
+            it.packagePartProviders,
+            SingleJavaFileRootsIndex(singleJavaFileRoots),
+            configuration.getBoolean(JVMConfigurationKeys.USE_PSI_CLASS_FILES_READING)
+        )
+    }
+}
+
+private fun contentRootToVirtualFile(root: JvmContentRoot, locaFileSystem: VirtualFileSystem, jarFileSystem: VirtualFileSystem, messageCollector: MessageCollector): VirtualFile? =
+    when (root) {
+        // TODO: find out why non-existent location is not reported for JARs, add comment or fix
+        is JvmClasspathRoot ->
+            if (root.file.isFile) jarFileSystem.findJarRoot(root.file) else locaFileSystem.findExistingRoot(root, "Classpath entry", messageCollector)
+        is JvmModulePathRoot ->
+            if (root.file.isFile) jarFileSystem.findJarRoot(root.file) else locaFileSystem.findExistingRoot(root, "Java module root", messageCollector)
+        is JavaSourceRoot ->
+            locaFileSystem.findExistingRoot(root, "Java source root", messageCollector)
+        else ->
+            throw IllegalStateException("Unexpected root: $root")
+    }
+
+private fun VirtualFileSystem.findJarRoot(file: File): VirtualFile? =
+    findFileByPath("$file${URLUtil.JAR_SEPARATOR}")
+
+private fun VirtualFileSystem.findExistingRoot(root: JvmContentRoot, rootDescription: String, messageCollector: MessageCollector): VirtualFile? {
+    return findFileByPath(root.file.absolutePath).also {
+        if (it == null) {
+            messageCollector.report(
+                CompilerMessageSeverity.STRONG_WARNING,
+                "$rootDescription points to a non-existent location: ${root.file}"
+            )
+        }
+    }
+}

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipelineData.kt

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cli.jvm.compiler.pipeline
+
+import org.jetbrains.kotlin.codegen.state.GenerationState
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.diagnostics.impl.BaseDiagnosticsCollector
+import org.jetbrains.kotlin.fir.session.environment.AbstractProjectEnvironment
+import org.jetbrains.kotlin.modules.TargetId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.platform.TargetPlatform
+import java.io.File
+
+// ---
+
+data class ModuleCompilerInput(
+    val targetId: TargetId,
+    val commonPlatform: TargetPlatform,
+    val commonSources: Collection<File>,
+    val platform: TargetPlatform,
+    val platformSources: Collection<File>,
+    val configuration: CompilerConfiguration
+)
+
+data class ModuleCompilerEnvironment(
+    val projectEnvironment: AbstractProjectEnvironment,
+    val diagnosticsReporter: BaseDiagnosticsCollector
+)
+
+data class ModuleCompilerOutput(
+    val mainClassName: FqName?,
+    val generationState: GenerationState
+)
+
+// ---
+