FIR: extend cli pipeline with incremental compilation logic

use it in the IncrementalCompilationRunner
2021-11-13 20:10:25 +01:00
parent 6b61488099
commit 275135a1b2
20 changed files with 886 additions and 203 deletions
@@ -13,6 +13,9 @@ dependencies {
    api(project(":compiler:frontend.java"))
    api(project(":compiler:cli"))
    api(project(":compiler:cli-js"))
+    api(project(":compiler:fir:entrypoint"))
+    api(project(":compiler:ir.serialization.jvm"))
+    api(project(":compiler:backend.jvm.entrypoint"))
    api(project(":kotlin-build-common"))
    api(project(":daemon-common"))
    implementation(commonDependency("com.google.code.gson:gson"))
@@ -16,8 +16,8 @@ class DirtyFilesContainer(
 ) {
    private val myDirtyFiles = HashSet<File>()

-    fun toMutableList(): MutableList<File> =
-        ArrayList(myDirtyFiles)
+    fun toMutableLinkedSet(): LinkedHashSet<File> =
+        LinkedHashSet(myDirtyFiles)

    fun add(files: Iterable<File>, reason: String?) {
        val existingKotlinFiles = files.filter { it.isKotlinFile(sourceFilesExtensions) }
@@ -54,7 +54,7 @@ abstract class IncrementalCompilerRunner<
 ) {

    protected val cacheDirectory = File(workingDir, cacheDirName)
-    private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
+    protected val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
    protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME)
    private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME)
    protected open val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
@@ -297,14 +297,16 @@ abstract class IncrementalCompilerRunner<
        }

    protected abstract fun runCompiler(
-        sourcesToCompile: Set<File>,
+        sourcesToCompile: List<File>,
        args: Args,
        caches: CacheManager,
        services: Services,
-        messageCollector: MessageCollector
-    ): ExitCode
+        messageCollector: MessageCollector,
+        allSources: List<File>,
+        isIncremental: Boolean
+    ): Pair<ExitCode, Collection<File>>

-    private fun compileIncrementally(
+    protected open fun compileIncrementally(
        args: Args,
        caches: CacheManager,
        allKotlinSources: List<File>,
@@ -322,11 +324,11 @@ abstract class IncrementalCompilerRunner<

        val dirtySources = when (compilationMode) {
            is CompilationMode.Incremental -> {
-                compilationMode.dirtyFiles.toMutableList()
+                compilationMode.dirtyFiles.toMutableLinkedSet()
            }
            is CompilationMode.Rebuild -> {
                reporter.addAttribute(compilationMode.reason)
-                allKotlinSources.toMutableList()
+                LinkedHashSet(allKotlinSources)
            }
        }

@@ -362,10 +364,18 @@ abstract class IncrementalCompilerRunner<
            val bufferingMessageCollector = BufferingMessageCollector()
            val messageCollectorAdapter = MessageCollectorToOutputItemsCollectorAdapter(bufferingMessageCollector, outputItemsCollector)

-            exitCode = reporter.measure(BuildTime.COMPILATION_ROUND) {
-                runCompiler(sourcesToCompile.toSet(), args, caches, services, messageCollectorAdapter)
+            val compiledSources = reporter.measure(BuildTime.COMPILATION_ROUND) {
+                runCompiler(
+                    sourcesToCompile, args, caches, services, messageCollectorAdapter,
+                    allKotlinSources, compilationMode is CompilationMode.Incremental
+                )
+            }.let { (ec, compiled) ->
+                exitCode = ec
+                compiled
            }

+            dirtySources.addAll(compiledSources)
+
            val generatedFiles = outputItemsCollector.outputs.map(SimpleOutputItem::toGeneratedFile)
            if (compilationMode is CompilationMode.Incremental) {
                // todo: feels dirty, can this be refactored?
@@ -378,7 +388,7 @@ abstract class IncrementalCompilerRunner<
                }
            }

-            reporter.reportCompileIteration(compilationMode is CompilationMode.Incremental, sourcesToCompile, exitCode)
+            reporter.reportCompileIteration(compilationMode is CompilationMode.Incremental, compiledSources, exitCode)
            bufferingMessageCollector.flush(originalMessageCollector)

            if (exitCode != ExitCode.OK) break
@@ -0,0 +1,356 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import com.intellij.ide.highlighter.JavaFileType
+import com.intellij.openapi.util.Disposer
+import com.intellij.psi.PsiJavaModule
+import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
+import org.jetbrains.kotlin.backend.jvm.JvmGeneratorExtensionsImpl
+import org.jetbrains.kotlin.backend.jvm.serialization.JvmIdSignatureDescriptor
+import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
+import org.jetbrains.kotlin.build.report.BuildReporter
+import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
+import org.jetbrains.kotlin.cli.common.ExitCode
+import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
+import org.jetbrains.kotlin.cli.common.computeKotlinPaths
+import org.jetbrains.kotlin.cli.common.config.addKotlinSourceRoot
+import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots
+import org.jetbrains.kotlin.cli.common.environment.setIdeaIoUseFallback
+import org.jetbrains.kotlin.cli.common.fir.FirDiagnosticsCompilerResultsReporter
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.common.messages.GroupingMessageCollector
+import org.jetbrains.kotlin.cli.common.messages.IrMessageCollector
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.cli.common.modules.ModuleBuilder
+import org.jetbrains.kotlin.cli.common.setupCommonArguments
+import org.jetbrains.kotlin.cli.jvm.*
+import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
+import org.jetbrains.kotlin.cli.jvm.compiler.VfsBasedProjectEnvironment
+import org.jetbrains.kotlin.cli.jvm.compiler.findMainClass
+import org.jetbrains.kotlin.cli.jvm.compiler.forAllFiles
+import org.jetbrains.kotlin.cli.jvm.compiler.pipeline.*
+import org.jetbrains.kotlin.cli.jvm.config.ClassicFrontendSpecificJvmConfigurationKeys
+import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot
+import org.jetbrains.kotlin.cli.jvm.config.JvmModulePathRoot
+import org.jetbrains.kotlin.cli.jvm.config.addJavaSourceRoot
+import org.jetbrains.kotlin.cli.jvm.plugins.PluginCliParser
+import org.jetbrains.kotlin.config.*
+import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory
+import org.jetbrains.kotlin.fir.FirSession
+import org.jetbrains.kotlin.fir.backend.Fir2IrConverter
+import org.jetbrains.kotlin.fir.backend.jvm.Fir2IrJvmSpecialAnnotationSymbolProvider
+import org.jetbrains.kotlin.fir.backend.jvm.FirJvmKotlinMangler
+import org.jetbrains.kotlin.fir.backend.jvm.FirJvmVisibilityConverter
+import org.jetbrains.kotlin.fir.languageVersionSettings
+import org.jetbrains.kotlin.fir.moduleData
+import org.jetbrains.kotlin.fir.resolve.providers.firProvider
+import org.jetbrains.kotlin.fir.resolve.providers.impl.FirProviderImpl
+import org.jetbrains.kotlin.fir.session.environment.AbstractProjectFileSearchScope
+import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
+import org.jetbrains.kotlin.incremental.components.InlineConstTracker
+import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
+import org.jetbrains.kotlin.ir.backend.jvm.serialization.JvmDescriptorMangler
+import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl
+import org.jetbrains.kotlin.ir.util.IrMessageLogger
+import org.jetbrains.kotlin.load.java.JavaClassesTracker
+import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
+import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion
+import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil
+import org.jetbrains.kotlin.modules.TargetId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.platform.CommonPlatforms
+import org.jetbrains.kotlin.platform.jvm.JvmPlatforms
+import org.jetbrains.kotlin.progress.CompilationCanceledException
+import java.io.File
+
+class IncrementalFirJvmCompilerRunner(
+    workingDir: File,
+    reporter: BuildReporter,
+    buildHistoryFile: File,
+    outputFiles: Collection<File>,
+    modulesApiHistory: ModulesApiHistory,
+    kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
+    classpathChanges: ClasspathChanges
+) : IncrementalJvmCompilerRunner(
+    workingDir,
+    reporter,
+    false,
+    buildHistoryFile,
+    outputFiles,
+    modulesApiHistory,
+    kotlinSourceFilesExtensions,
+    classpathChanges
+) {
+
+    override fun runCompiler(
+        sourcesToCompile: List<File>,
+        args: K2JVMCompilerArguments,
+        caches: IncrementalJvmCachesManager,
+        services: Services,
+        messageCollector: MessageCollector,
+        allSources: List<File>,
+        isIncremental: Boolean
+    ): Pair<ExitCode, Collection<File>> {
+//        val isIncremental = true // TODO
+        val collector = GroupingMessageCollector(messageCollector, args.allWarningsAsErrors)
+        // from K2JVMCompiler (~)
+        val moduleName = args.moduleName ?: JvmProtoBufUtil.DEFAULT_MODULE_NAME
+        val targetId = TargetId(moduleName, "java-production") // TODO: get rid of magic constant
+
+        val dirtySources = linkedSetOf<File>().apply { addAll(sourcesToCompile) }
+
+        // TODO: probably shoudl be passed along with sourcesToCompile
+        // TODO: file path normalization
+        val commonSources = args.commonSources?.mapTo(mutableSetOf(), ::File).orEmpty()
+
+        val exitCode = ExitCode.OK
+        val allCompiledSources = LinkedHashSet<File>()
+        val rootDisposable = Disposer.newDisposable()
+
+        try {
+            // - configuration
+            val configuration = CompilerConfiguration().apply {
+
+                put(CLIConfigurationKeys.ORIGINAL_MESSAGE_COLLECTOR_KEY, messageCollector)
+                put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, collector)
+                put(IrMessageLogger.IR_MESSAGE_LOGGER, IrMessageCollector(collector))
+
+                setupCommonArguments(args) { JvmMetadataVersion(*it) }
+
+                if (IncrementalCompilation.isEnabledForJvm()) {
+                    putIfNotNull(CommonConfigurationKeys.LOOKUP_TRACKER, services[LookupTracker::class.java])
+
+                    putIfNotNull(CommonConfigurationKeys.EXPECT_ACTUAL_TRACKER, services[ExpectActualTracker::class.java])
+
+                    putIfNotNull(CommonConfigurationKeys.INLINE_CONST_TRACKER, services[InlineConstTracker::class.java])
+
+                    putIfNotNull(
+                        JVMConfigurationKeys.INCREMENTAL_COMPILATION_COMPONENTS,
+                        services[IncrementalCompilationComponents::class.java]
+                    )
+
+                    putIfNotNull(ClassicFrontendSpecificJvmConfigurationKeys.JAVA_CLASSES_TRACKER, services[JavaClassesTracker::class.java])
+                }
+
+                setupJvmSpecificArguments(args)
+            }
+
+            val paths = computeKotlinPaths(collector, args)
+            if (collector.hasErrors()) return ExitCode.COMPILATION_ERROR to emptyList()
+
+            // -- plugins
+            val pluginClasspaths: Iterable<String> = args.pluginClasspaths?.asIterable() ?: emptyList()
+            val pluginOptions = args.pluginOptions?.toMutableList() ?: ArrayList()
+            // TODO: add scripting support when ready in FIR
+            val pluginLoadResult = PluginCliParser.loadPluginsSafe(pluginClasspaths, pluginOptions, configuration)
+            if (pluginLoadResult != ExitCode.OK) return pluginLoadResult to emptyList()
+            // -- /plugins
+
+            with(configuration) {
+                configureJavaModulesContentRoots(args)
+                configureStandardLibs(paths, args)
+                configureAdvancedJvmOptions(args)
+                configureKlibPaths(args)
+
+                val destination = File(args.destination ?: ".")
+                if (destination.path.endsWith(".jar")) {
+                    put(JVMConfigurationKeys.OUTPUT_JAR, destination)
+                } else {
+                    put(JVMConfigurationKeys.OUTPUT_DIRECTORY, destination)
+                }
+                addAll(JVMConfigurationKeys.MODULES, listOf(ModuleBuilder(targetId.name, destination.path, targetId.type)))
+
+                configureBaseRoots(args)
+                configureSourceRootsFromSources(allSources, commonSources, args.javaPackagePrefix)
+            }
+            // - /configuration
+
+            setIdeaIoUseFallback()
+
+            // -AbstractProjectEnvironment-
+            val projectEnvironment =
+                createProjectEnvironment(configuration, rootDisposable, EnvironmentConfigFiles.JVM_CONFIG_FILES, messageCollector)
+
+            // -sources
+            val allPlatformSourceFiles = linkedSetOf<File>() // TODO: get from caller
+            val allCommonSourceFiles = linkedSetOf<File>()
+
+            configuration.kotlinSourceRoots.forAllFiles(configuration, projectEnvironment.project) { virtualFile, isCommon ->
+                val file = File(virtualFile.canonicalPath ?: virtualFile.path)
+                if (!file.isFile) error("TODO: better error: file not found $virtualFile")
+                if (isCommon) allCommonSourceFiles.add(file)
+                else allPlatformSourceFiles.add(file)
+            }
+
+            val diagnosticsReporter = DiagnosticReporterFactory.createReporter()
+            val compilerEnvironment = ModuleCompilerEnvironment(projectEnvironment, diagnosticsReporter)
+
+            // !! main class - maybe from cache?
+            var mainClassFqName: FqName? = null
+
+            var incrementalExcludesScope: AbstractProjectFileSearchScope? = null
+
+            fun firIncrementalCycle(): ModuleCompilerAnalyzedOutput? {
+                while (true) {
+
+                    val compilerInput = ModuleCompilerInput(
+                        targetId,
+                        CommonPlatforms.defaultCommonPlatform, dirtySources.filter { it in allCommonSourceFiles },
+                        JvmPlatforms.unspecifiedJvmPlatform, dirtySources.filter { it in allPlatformSourceFiles },
+                        configuration
+                    )
+
+                    val analysisResults =
+                        compileModuleToAnalyzedFir(
+                            compilerInput,
+                            compilerEnvironment,
+                            emptyList(),
+                            incrementalExcludesScope,
+                        )
+
+                    // TODO: consider what to do if many compilations find a main class
+                    if (mainClassFqName == null && configuration.get(JVMConfigurationKeys.OUTPUT_JAR) != null) {
+                        mainClassFqName = findMainClass(analysisResults.fir)
+                    }
+
+                    allCompiledSources.addAll(dirtySources)
+
+                    if (diagnosticsReporter.hasErrors) {
+                        FirDiagnosticsCompilerResultsReporter.reportToMessageCollector(
+                            diagnosticsReporter,
+                            collector,
+                            configuration.getBoolean(CLIConfigurationKeys.RENDER_DIAGNOSTIC_INTERNAL_NAME)
+                        )
+                        return null
+                    }
+
+                    val newDirtySources =
+                        collectNewDirtySources(analysisResults, targetId, configuration, caches, allCompiledSources, reporter)
+
+                    if (!isIncremental || newDirtySources.isEmpty()) return analysisResults
+
+                    caches.platformCache.markDirty(newDirtySources)
+                    val newDirtyFilesOutputsScope =
+                        projectEnvironment.getSearchScopeByIoFiles(caches.inputsCache.getOutputForSourceFiles(newDirtySources))
+                    incrementalExcludesScope = incrementalExcludesScope.let {
+                        when {
+                            newDirtyFilesOutputsScope.isEmpty -> it
+                            it == null || it.isEmpty -> newDirtyFilesOutputsScope
+                            else -> it + newDirtyFilesOutputsScope
+                        }
+                    }
+                    caches.inputsCache.removeOutputForSourceFiles(newDirtySources)
+                    dirtySources.addAll(newDirtySources)
+                    projectEnvironment.localFileSystem.refresh(false)
+                }
+            }
+
+            val cycleResult = firIncrementalCycle() ?: return ExitCode.COMPILATION_ERROR to allCompiledSources
+
+            val extensions = JvmGeneratorExtensionsImpl(configuration)
+            val irGenerationExtensions =
+                (projectEnvironment as? VfsBasedProjectEnvironment)?.project?.let { IrGenerationExtension.getInstances(it) }.orEmpty()
+            val mangler = JvmDescriptorMangler(null)
+            val signaturer = JvmIdSignatureDescriptor(JvmDescriptorMangler(null))
+            val allCommonFirFiles = cycleResult.session.moduleData.dependsOnDependencies
+                .map { it.session }
+                .filter { it.kind == FirSession.Kind.Source }
+                .flatMap { (it.firProvider as FirProviderImpl).getAllFirFiles() }
+
+            val (irModuleFragment, symbolTable, components) = Fir2IrConverter.createModuleFragment(
+                cycleResult.session, cycleResult.scopeSession, cycleResult.fir + allCommonFirFiles,
+                cycleResult.session.languageVersionSettings, mangler, signaturer,
+                extensions, FirJvmKotlinMangler(cycleResult.session), IrFactoryImpl,
+                FirJvmVisibilityConverter,
+                Fir2IrJvmSpecialAnnotationSymbolProvider(),
+                irGenerationExtensions
+            )
+
+            val irInput = ModuleCompilerIrBackendInput(
+                targetId,
+                configuration,
+                extensions,
+                irModuleFragment,
+                symbolTable,
+                components,
+                cycleResult.session
+            )
+
+            val codegenOutput = generateCodeFromIr(irInput, compilerEnvironment)
+
+            FirDiagnosticsCompilerResultsReporter.reportToMessageCollector(
+                diagnosticsReporter,
+                collector,
+                configuration.getBoolean(CLIConfigurationKeys.RENDER_DIAGNOSTIC_INTERNAL_NAME)
+            )
+
+            writeOutputs(
+                projectEnvironment,
+                configuration,
+                listOf(codegenOutput.generationState),
+                mainClassFqName
+            )
+        } catch (e: CompilationCanceledException) {
+            collector.report(CompilerMessageSeverity.INFO, "Compilation was canceled", null)
+            return ExitCode.OK to allCompiledSources
+        } catch (e: RuntimeException) {
+            val cause = e.cause
+            if (cause is CompilationCanceledException) {
+                collector.report(CompilerMessageSeverity.INFO, "Compilation was canceled", null)
+                return ExitCode.OK to allCompiledSources
+            } else {
+                throw e
+            }
+        } finally {
+            collector.flush()
+            Disposer.dispose(rootDisposable)
+        }
+        return exitCode to allCompiledSources
+    }
+}
+
+
+fun CompilerConfiguration.configureBaseRoots(args: K2JVMCompilerArguments) {
+
+    var isJava9Module = false
+    args.javaSourceRoots?.forEach {
+        val file = File(it)
+        val packagePrefix = args.javaPackagePrefix
+        addJavaSourceRoot(file, packagePrefix)
+        if (!isJava9Module && packagePrefix == null && (file.name == PsiJavaModule.MODULE_INFO_FILE ||
+                    (file.isDirectory && file.listFiles()?.any { it.name == PsiJavaModule.MODULE_INFO_FILE } == true))
+        ) {
+            isJava9Module = true
+        }
+    }
+
+    args.classpath?.split(File.pathSeparator)?.forEach { classpathRoot ->
+        add(
+            CLIConfigurationKeys.CONTENT_ROOTS,
+            if (isJava9Module) JvmModulePathRoot(File(classpathRoot)) else JvmClasspathRoot(File(classpathRoot))
+        )
+    }
+
+    // TODO: modularJdkRoot (now seems only processed from the build file
+}
+
+fun CompilerConfiguration.configureSourceRootsFromSources(
+    allSources: Collection<File>, commonSources: Set<File>, javaPackagePrefix: String?
+) {
+    for (sourceFile in allSources) {
+        if (sourceFile.name.endsWith(JavaFileType.DOT_DEFAULT_EXTENSION)) {
+            addJavaSourceRoot(sourceFile, javaPackagePrefix)
+        } else {
+            addKotlinSourceRoot(sourceFile.path, isCommon = sourceFile in commonSources)
+
+            if (sourceFile.isDirectory) {
+                addJavaSourceRoot(sourceFile, javaPackagePrefix)
+            }
+        }
+    }
+}
@@ -192,18 +192,20 @@ class IncrementalJsCompilerRunner(
    }

    override fun runCompiler(
-        sourcesToCompile: Set<File>,
+        sourcesToCompile: List<File>,
        args: K2JSCompilerArguments,
        caches: IncrementalJsCachesManager,
        services: Services,
-        messageCollector: MessageCollector
-    ): ExitCode {
+        messageCollector: MessageCollector,
+        allSources: List<File>,
+        isIncremental: Boolean
+    ): Pair<ExitCode, Collection<File>> {
        val freeArgsBackup = args.freeArgs

        val compiler = K2JSCompiler()
        return try {
            args.freeArgs += sourcesToCompile.map { it.absolutePath }
-            compiler.exec(messageCollector, services, args)
+            compiler.exec(messageCollector, services, args) to sourcesToCompile
        } finally {
            args.freeArgs = freeArgsBackup
            reportPerformanceData(compiler.defaultPerformanceManager)
@@ -86,17 +86,24 @@ fun makeIncrementally(
    val buildReporter = BuildReporter(icReporter = reporter, buildMetricsReporter = DoNothingBuildMetricsReporter)

    withIC(args) {
-        val compiler = IncrementalJvmCompilerRunner(
-            cachesDir,
-            buildReporter,
-            // Use precise setting in case of non-Gradle build
-            usePreciseJavaTracking = !args.useFir, // TODO: add fir-based java classes tracker when available and set this to true
-            outputFiles = emptyList(),
-            buildHistoryFile = buildHistoryFile,
-            modulesApiHistory = EmptyModulesApiHistory,
-            kotlinSourceFilesExtensions = kotlinExtensions,
-            classpathChanges = ClasspathSnapshotDisabled
-        )
+        val compiler =
+            if (args.useFir)
+                IncrementalFirJvmCompilerRunner(
+                    cachesDir, buildReporter, buildHistoryFile, emptyList(), EmptyModulesApiHistory, kotlinExtensions, ClasspathSnapshotDisabled
+                )
+            else
+                IncrementalJvmCompilerRunner(
+                    cachesDir,
+                    buildReporter,
+                    // Use precise setting in case of non-Gradle build
+                    usePreciseJavaTracking = !args.useFir, // TODO: add fir-based java classes tracker when available and set this to true
+                    outputFiles = emptyList(),
+                    buildHistoryFile = buildHistoryFile,
+                    modulesApiHistory = EmptyModulesApiHistory,
+                    kotlinSourceFilesExtensions = kotlinExtensions,
+                    classpathChanges = ClasspathSnapshotDisabled
+                )
+        //TODO set properly
        compiler.compile(sourceFiles, args, messageCollector, providedChangedFiles = null)
    }
 }
@@ -116,7 +123,7 @@ inline fun <R> withIC(args: CommonCompilerArguments, enabled: Boolean = true, fn
    }
 }

-class IncrementalJvmCompilerRunner(
+open class IncrementalJvmCompilerRunner(
    workingDir: File,
    reporter: BuildReporter,
    private val usePreciseJavaTracking: Boolean,
@@ -453,12 +460,14 @@ class IncrementalJvmCompilerRunner(
        }

    override fun runCompiler(
-        sourcesToCompile: Set<File>,
+        sourcesToCompile: List<File>,
        args: K2JVMCompilerArguments,
        caches: IncrementalJvmCachesManager,
        services: Services,
-        messageCollector: MessageCollector
-    ): ExitCode {
+        messageCollector: MessageCollector,
+        allSources: List<File>,
+        isIncremental: Boolean
+    ): Pair<ExitCode, Collection<File>> {
        val compiler = K2JVMCompiler()
        val freeArgsBackup = args.freeArgs.toList()
        args.freeArgs += sourcesToCompile.map { it.absolutePath }
@@ -466,7 +475,7 @@ class IncrementalJvmCompilerRunner(
        val exitCode = compiler.exec(messageCollector, services, args)
        args.freeArgs = freeArgsBackup
        reportPerformanceData(compiler.defaultPerformanceManager)
-        return exitCode
+        return exitCode to sourcesToCompile
    }

    override fun performWorkAfterSuccessfulCompilation(caches: IncrementalJvmCachesManager) {
@@ -47,6 +47,10 @@ class InputsCache(
        }
    }

+    fun getOutputForSourceFiles(sources: Iterable<File>): List<File> = sources.flatMap {
+        sourceToOutputMap[it]
+    }
+
    // generatedFiles can contain multiple entries with the same source file
    // for example Kapt3 IC will generate a .java stub and .class stub for each source file
    fun registerOutputForSourceFiles(generatedFiles: List<GeneratedFile>) {
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental
+
+import org.jetbrains.kotlin.backend.common.pop
+import org.jetbrains.kotlin.backend.common.push
+import org.jetbrains.kotlin.backend.jvm.metadata.MetadataSerializer
+import org.jetbrains.kotlin.build.report.ICReporter
+import org.jetbrains.kotlin.cli.jvm.compiler.pipeline.ModuleCompilerAnalyzedOutput
+import org.jetbrains.kotlin.codegen.serialization.JvmSerializationBindings
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.fir.FirElement
+import org.jetbrains.kotlin.fir.backend.FirMetadataSource
+import org.jetbrains.kotlin.fir.backend.jvm.makeLocalFirMetadataSerializerForMetadataSource
+import org.jetbrains.kotlin.fir.declarations.*
+import org.jetbrains.kotlin.fir.declarations.utils.classId
+import org.jetbrains.kotlin.fir.scopes.jvm.computeJvmDescriptor
+import org.jetbrains.kotlin.fir.visitors.FirVisitor
+import org.jetbrains.kotlin.metadata.ProtoBuf
+import org.jetbrains.kotlin.modules.TargetId
+import org.jetbrains.kotlin.name.SpecialNames
+import org.jetbrains.kotlin.resolve.jvm.JvmClassName
+import org.jetbrains.org.objectweb.asm.commons.Method
+import java.io.File
+
+internal fun collectNewDirtySources(
+    analysisResults: ModuleCompilerAnalyzedOutput,
+    targetId: TargetId,
+    configuration: CompilerConfiguration,
+    caches: IncrementalJvmCachesManager,
+    alreadyCompiledSources: Set<File>,
+    reporter: ICReporter
+): LinkedHashSet<File> {
+    val changesCollector = ChangesCollector()
+    val globalSerializationBindings = JvmSerializationBindings()
+    analysisResults.fir.forEach {
+        it.accept(object : FirVisitor<Unit, MutableList<MetadataSerializer>>() {
+
+            inline fun withMetadataSerializer(
+                metadata: FirMetadataSource,
+                data: MutableList<MetadataSerializer>,
+                body: (MetadataSerializer) -> Unit
+            ) {
+                val serializer = makeLocalFirMetadataSerializerForMetadataSource(
+                    metadata,
+                    analysisResults.session,
+                    analysisResults.scopeSession,
+                    globalSerializationBindings,
+                    data.lastOrNull(),
+                    targetId,
+                    configuration
+                )
+                data.push(serializer)
+                body(serializer)
+                data.pop()
+            }
+
+            override fun visitElement(element: FirElement, data: MutableList<MetadataSerializer>) {
+                element.acceptChildren(this, data)
+            }
+
+            override fun visitRegularClass(regularClass: FirRegularClass, data: MutableList<MetadataSerializer>) {
+                visitClass(regularClass, data)
+            }
+
+            override fun visitAnonymousObject(anonymousObject: FirAnonymousObject, data: MutableList<MetadataSerializer>) {
+                visitClass(anonymousObject, data)
+            }
+
+            override fun visitFile(file: FirFile, data: MutableList<MetadataSerializer>) {
+                val metadata = FirMetadataSource.File(file)
+                withMetadataSerializer(metadata, data) {
+                    file.acceptChildren(this, data)
+                    // TODO: compare package fragments?
+                }
+            }
+
+            override fun visitSimpleFunction(simpleFunction: FirSimpleFunction, data: MutableList<MetadataSerializer>) {
+                data.firstOrNull()?.let { serializer ->
+                    super.visitFunction(simpleFunction, data)
+                    serializer.bindMethodMetadata(
+                        FirMetadataSource.Function(simpleFunction),
+                        Method(simpleFunction.name.asString(), simpleFunction.computeJvmDescriptor())
+                    )
+                }
+            }
+
+            override fun visitConstructor(constructor: FirConstructor, data: MutableList<MetadataSerializer>) {
+                super.visitConstructor(constructor, data)
+                data.first().bindMethodMetadata(
+                    FirMetadataSource.Function(constructor),
+                    Method(SpecialNames.INIT.asString(), constructor.computeJvmDescriptor(""))
+                )
+            }
+
+            override fun visitProperty(property: FirProperty, data: MutableList<MetadataSerializer>) {
+                property.acceptChildren(this, data)
+//                    data.firstOrNull()?.let {
+//                        property.acceptChildren(this, data)
+//                        it.bindPropertyMetadata(
+//                            FirMetadataSource.Property(property),
+//                            Method(property.name.asString(), ""),//property.computeJvmDescriptor())
+//                            IrDeclarationOrigin.DEFINED
+//                        )
+//                    }
+            }
+
+            override fun visitClass(klass: FirClass, data: MutableList<MetadataSerializer>) {
+                val metadata = FirMetadataSource.Class(klass)
+                withMetadataSerializer(metadata, data) { serializer ->
+                    klass.acceptChildren(this, data)
+                    serializer.serialize(metadata)?.let { (classProto, nameTable) ->
+                        caches.platformCache.collectClassChangesByFeMetadata(
+                            JvmClassName.byClassId(klass.classId),
+                            classProto as ProtoBuf.Class,
+                            nameTable,
+                            changesCollector
+                        )
+                    }
+                }
+            }
+        }, mutableListOf())
+    }
+
+    val (dirtyLookupSymbols, dirtyClassFqNames, forceRecompile) = changesCollector.getDirtyData(listOf(caches.platformCache), reporter)
+
+    val forceToRecompileFiles = mapClassesFqNamesToFiles(listOf(caches.platformCache), forceRecompile, reporter)
+
+    return linkedSetOf<File>().apply {
+        addAll(mapLookupSymbolsToFiles(caches.lookupCache, dirtyLookupSymbols, reporter, excludes = alreadyCompiledSources))
+        addAll(
+            mapClassesFqNamesToFiles(
+                listOf(caches.platformCache),
+                dirtyClassFqNames,
+                reporter,
+                excludes = alreadyCompiledSources
+            )
+        )
+        if (!alreadyCompiledSources.containsAll(forceToRecompileFiles)) {
+            addAll(forceToRecompileFiles)
+        }
+    }
+}