Clean up fall-back logic in IncrementalCompilerRunner

Make it clear that there 3 distinct cases: 1. Incremental compilation completed with an ExitCode. 2. Incremental compilation was not possible for some valid reason (e.g., for a clean build), and we will perform non-incremental compilation. 3. Incremental compilation failed with an exception. In this case, we will: - Print a warning with a stack trace - Ask the user to file a bug - Collect rebuild reason enum for analytics + TODO: Collect the stack trace too - Fall back to non-incremental compilation Test: Existing BaseIncrementalCompilationMultiProjectIT.testFailureHandling_UserError, Updated BaseIncrementalCompilationMultiProjectIT.testFailureHandling_ToolError ^KT-53015: In progress
2022-06-20 13:41:40 +01:00
parent 57bbc335f4
commit def886cd31
24 changed files with 409 additions and 341 deletions
@@ -5,8 +5,6 @@

 package org.jetbrains.kotlin.incremental

-import org.jetbrains.kotlin.build.report.BuildReporter
-import org.jetbrains.kotlin.build.report.info
 import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
 import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmNameResolver
 import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil
@@ -152,14 +150,9 @@ class AbiSnapshotImpl(override val protos: MutableMap<FqName, ProtoData>) : AbiS
            }
        }

-        fun read(file: File, reporter: BuildReporter): AbiSnapshot? {
-            if (!file.exists()) {
-                reporter.info { "jar snapshot $file is found for jar" }
-                return null
-            }
-
-            ObjectInputStream(FileInputStream(file)).use {
-                return it.readAbiSnapshot()
+        fun read(file: File): AbiSnapshot {
+            return ObjectInputStream(FileInputStream(file)).use {
+                it.readAbiSnapshot()
            }
        }
    }
@@ -20,10 +20,8 @@ import java.io.File
 import java.io.Serializable

 sealed class ChangedFiles : Serializable {
-    class Known(val modified: List<File>, val removed: List<File>) : ChangedFiles()
+    class Known(val modified: List<File>, val removed: List<File>, val forDependencies: Boolean = false) : ChangedFiles()
    class Unknown : ChangedFiles()
-    class Dependencies(val modified: List<File>, val removed: List<File>) : ChangedFiles()
-
    companion object {
        const val serialVersionUID: Long = 0
    }
@@ -16,30 +16,29 @@

 package org.jetbrains.kotlin.incremental

+import com.google.common.io.Closer
 import org.jetbrains.kotlin.build.report.ICReporter
-import org.jetbrains.kotlin.build.report.info
 import org.jetbrains.kotlin.incremental.storage.BasicMapsOwner
 import org.jetbrains.kotlin.incremental.storage.IncrementalFileToPathConverter
 import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol
+import java.io.Closeable
 import java.io.File

-
 abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache<*>>(
    cachesRootDir: File,
    rootProjectDir: File?,
    protected val reporter: ICReporter,
    storeFullFqNamesInLookupCache: Boolean = false,
    trackChangesInLookupCache: Boolean = false
-) {
+) : Closeable {
    val pathConverter = IncrementalFileToPathConverter(rootProjectDir)
    private val caches = arrayListOf<BasicMapsOwner>()

-    var isClosed = false
-    var isSuccessfulyClosed = false
+    private var isClosed = false

    @Synchronized
    protected fun <T : BasicMapsOwner> T.registerCache() {
-        assert(!isClosed) { "Attempted to add new cache into closed storage." }
+        check(!isClosed) { "This cache storage has already been closed" }
        caches.add(this)
    }

@@ -51,33 +50,29 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
        LookupStorage(lookupCacheDir, pathConverter, storeFullFqNamesInLookupCache, trackChangesInLookupCache).apply { registerCache() }
    abstract val platformCache: PlatformCache

+    @Suppress("UnstableApiUsage")
    @Synchronized
-    fun close(flush: Boolean = false): Boolean {
-        if (isClosed) {
-            return isSuccessfulyClosed
-        }
-        isSuccessfulyClosed = true
-        for (cache in caches) {
-            if (flush) {
-                try {
-                    cache.flush(false)
-                } catch (e: Throwable) {
-                    isSuccessfulyClosed = false
-                    reporter.info { "Exception when flushing cache ${cache.javaClass}: $e" }
-                }
-            }
+    override fun close() {
+        check(!isClosed) { "This cache storage has already been closed" }

-            try {
-                cache.close()
-            } catch (e: Throwable) {
-                isSuccessfulyClosed = false
-                reporter.info { "Exception when closing cache ${cache.javaClass}: $e" }
-            }
+        val closer = Closer.create()
+        caches.forEach {
+            closer.register(CacheCloser(it))
        }
+        closer.close()

        isClosed = true
-        return isSuccessfulyClosed
    }
+
+    private class CacheCloser(private val cache: BasicMapsOwner) : Closeable {
+
+        override fun close() {
+            // It's important to flush the cache when closing (see KT-53168)
+            cache.flush(memoryCachesOnly = false)
+            cache.close()
+        }
+    }
+
 }

 class IncrementalJvmCachesManager(
@@ -22,6 +22,7 @@ import org.jetbrains.kotlin.build.report.BuildReporter
 import org.jetbrains.kotlin.build.report.debug
 import org.jetbrains.kotlin.build.report.info
 import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
+import org.jetbrains.kotlin.build.report.metrics.BuildAttribute.*
 import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric
 import org.jetbrains.kotlin.build.report.metrics.BuildTime
 import org.jetbrains.kotlin.build.report.metrics.measure
@@ -40,7 +41,7 @@ import org.jetbrains.kotlin.incremental.parsing.classesFqNames
 import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.progress.CompilationCanceledStatus
-import org.jetbrains.kotlin.util.suffixIfNot
+import org.jetbrains.kotlin.util.removeSuffixIfPresent
 import java.io.File

 abstract class IncrementalCompilerRunner<
@@ -51,14 +52,24 @@ abstract class IncrementalCompilerRunner<
    cacheDirName: String,
    protected val reporter: BuildReporter,
    protected val buildHistoryFile: File,
-    // there might be some additional output directories (e.g. for generated java in kapt)
-    // to remove them correctly on rebuild, we pass them as additional argument
-    private val additionalOutputFiles: Collection<File> = emptyList(),
+
+    /**
+     * Output directories of the compilation. These include:
+     *   1. The classes output directory
+     *   2. [workingDir]
+     *   3. Any additional output directories (e.g., classpath snapshot directory or Kapt generated-stubs directory)
+     *
+     * We will clean these directories when compiling non-incrementally.
+     *
+     * If this property is not set, the directories to clean will include the first 2 directories above.
+     */
+    private val outputDirs: Collection<File>?,
+
    protected val withAbiSnapshot: Boolean = false
 ) {

    protected val cacheDirectory = File(workingDir, cacheDirName)
-    protected val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
+    private val dirtySourcesSinceLastTimeFile = File(workingDir, DIRTY_SOURCES_FILE_NAME)
    protected val lastBuildInfoFile = File(workingDir, LAST_BUILD_INFO_FILE_NAME)
    private val abiSnapshotFile = File(workingDir, ABI_SNAPSHOT_FILE_NAME)
    protected open val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
@@ -75,184 +86,204 @@ abstract class IncrementalCompilerRunner<
        providedChangedFiles: ChangedFiles?,
        projectDir: File? = null
    ): ExitCode = reporter.measure(BuildTime.INCREMENTAL_COMPILATION_DAEMON) {
-        try {
-            compileImpl(allSourceFiles, args, messageCollector, providedChangedFiles, projectDir)
-        } finally {
-            reporter.measure(BuildTime.CALCULATE_OUTPUT_SIZE) {
-                reporter.addMetric(
-                    BuildPerformanceMetric.SNAPSHOT_SIZE,
-                    buildHistoryFile.length() + lastBuildInfoFile.length() + abiSnapshotFile.length()
+        return when (val result = tryCompileIncrementally(allSourceFiles, providedChangedFiles, args, projectDir, messageCollector)) {
+            is ICResult.Completed -> {
+                reporter.debug { "Incremental compilation completed" }
+                result.exitCode
+            }
+            is ICResult.RequiresRebuild -> {
+                reporter.info { "Non-incremental compilation will be performed: ${result.reason}" }
+                reporter.addAttribute(result.reason)
+
+                compileNonIncrementally(
+                    result.reason, allSourceFiles, args, projectDir, trackChangedFiles = providedChangedFiles == null, messageCollector
                )
-                if (cacheDirectory.exists() && cacheDirectory.isDirectory()) {
-                    cacheDirectory.walkTopDown().filter { it.isFile }.map { it.length() }.sum().let {
-                        reporter.addMetric(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE, it)
-                    }
+            }
+            is ICResult.Failed -> {
+                reporter.warn {
+                    // The indentation after the first line is intentional (so that this message is distinct from next message)
+                    """
+                    |Incremental compilation was attempted but failed:
+                    |    ${result.reason.readableString}: ${result.cause.stackTraceToString().removeSuffixIfPresent("\n")}
+                    |    Falling back to non-incremental compilation (reason = ${result.reason})
+                    |    To help us fix this issue, please file a bug at https://youtrack.jetbrains.com/issues/KT with the above stack trace.
+                    |    (Be sure to search for the above exception in existing issues first to avoid filing duplicated bugs.)             
+                    """.trimMargin()
                }
+                // TODO: Collect the stack trace too
+                reporter.addAttribute(result.reason)
+
+                compileNonIncrementally(
+                    result.reason, allSourceFiles, args, projectDir, trackChangedFiles = providedChangedFiles == null, messageCollector
+                )
            }
        }
    }

-    fun rebuild(
-        reason: BuildAttribute,
-        allSourceFiles: List<File>,
-        args: Args,
-        messageCollector: MessageCollector,
-        providedChangedFiles: ChangedFiles?,
-        projectDir: File? = null,
-        classpathAbiSnapshot: Map<String, AbiSnapshot>
-    ): ExitCode {
-        reporter.info { "Non-incremental compilation will be performed: $reason" }
-        reporter.measure(BuildTime.CLEAR_OUTPUT_ON_REBUILD) {
-            cleanOutputsAndLocalStateOnRebuild(args)
-        }
-        val caches = createCacheManager(args, projectDir)
-        try {
-            if (providedChangedFiles == null) {
-                caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
-            }
-            val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) }
-            return compileIncrementally(
-                args, caches, allKotlinFiles, CompilationMode.Rebuild(reason), messageCollector, withAbiSnapshot,
-                classpathAbiSnapshot = classpathAbiSnapshot
-            ).also {
-                if (it == ExitCode.OK) {
-                    performWorkAfterSuccessfulCompilation(caches, wasIncremental = false)
-                }
-            }
-        } finally {
-            caches.close(true)
-        }
-    }
+    /** The result when attempting to compile incrementally ([tryCompileIncrementally]). */
+    private sealed interface ICResult {

-    private fun compileImpl(
-        allSourceFiles: List<File>,
-        args: Args,
-        messageCollector: MessageCollector,
-        providedChangedFiles: ChangedFiles?,
-        projectDir: File? = null
-    ): ExitCode {
-        var caches = createCacheManager(args, projectDir)
-        var rebuildReason = BuildAttribute.INTERNAL_ERROR
+        /** Incremental compilation completed with an [ExitCode]. */
+        class Completed(val exitCode: ExitCode) : ICResult

-        val classpathAbiSnapshot =
-            if (withAbiSnapshot) {
-                reporter.info { "Incremental compilation with ABI snapshot enabled" }
-                reporter.measure(BuildTime.SET_UP_ABI_SNAPSHOTS) {
-                    setupJarDependencies(args, withAbiSnapshot, reporter)
-                }
-            } else {
-                emptyMap()
-            }
+        /** Incremental compilation was not possible for some valid reason (e.g., for a clean build). */
+        class RequiresRebuild(val reason: BuildAttribute) : ICResult

-        try {
-            val changedFiles = when (providedChangedFiles) {
-                is ChangedFiles.Dependencies -> {
-                    val changedSources = caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
-                    ChangedFiles.Known(
-                        providedChangedFiles.modified + changedSources.modified,
-                        providedChangedFiles.removed + changedSources.removed
-                    )
-                }
-                null -> caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
-                else -> providedChangedFiles
-            }
-
-            var compilationMode = sourcesToCompile(caches, changedFiles, args, messageCollector, classpathAbiSnapshot)
-            val abiSnapshot = if (compilationMode is CompilationMode.Incremental && withAbiSnapshot) {
-                AbiSnapshotImpl.read(abiSnapshotFile, reporter)
-            } else {
-                if (withAbiSnapshot) {
-                    compilationMode = CompilationMode.Rebuild(BuildAttribute.NO_ABI_SNAPSHOT)
-                }
-                null
-            }
-
-            when (compilationMode) {
-                is CompilationMode.Incremental -> {
-                    try {
-                        reporter.debug { "Performing incremental compilation" }
-                        val exitCode = if (withAbiSnapshot) {
-                            compileIncrementally(
-                                args, caches, allSourceFiles, compilationMode, messageCollector,
-                                withAbiSnapshot, abiSnapshot!!, classpathAbiSnapshot
-                            )
-                        } else {
-                            compileIncrementally(args, caches, allSourceFiles, compilationMode, messageCollector, withAbiSnapshot)
-                        }
-                        if (exitCode == ExitCode.OK) {
-                            performWorkAfterSuccessfulCompilation(caches, wasIncremental = true)
-                        }
-                        return exitCode
-                    } catch (e: Throwable) {
-                        reporter.warn {
-                            "Incremental compilation failed: ${e.stackTraceToString().suffixIfNot("\n")}" +
-                                    "Falling back to non-incremental compilation"
-                        }
-                        rebuildReason = BuildAttribute.INCREMENTAL_COMPILATION_FAILED
-                    }
-                }
-                is CompilationMode.Rebuild -> rebuildReason = compilationMode.reason
-            }
-        } catch (e: Exception) {
-            reporter.warn {
-                "Incremental compilation analysis failed: ${e.stackTraceToString().suffixIfNot("\n")}" +
-                        "Falling back to non-incremental compilation"
-            }
-        } finally {
-            if (!caches.close(flush = true)) {
-                reporter.info { "Unable to close IC caches. Cleaning internal state" }
-                cleanOutputsAndLocalStateOnRebuild(args)
-            }
-        }
-        return rebuild(rebuildReason, allSourceFiles, args, messageCollector, providedChangedFiles, projectDir, classpathAbiSnapshot)
+        /** Incremental compilation failed with an exception. */
+        class Failed(val reason: BuildAttribute, val cause: Throwable) : ICResult
    }

    /**
-     * Deletes output files and contents of output directories on rebuild, including `@LocalState` files/directories.
+     * Attempts to compile incrementally and returns either [ICResult.Completed], [ICResult.RequiresRebuild], or [ICResult.Failed].
+     *
+     * Note that parts of this function may still throw exceptions that are not caught and wrapped by [ICResult.Failed] because they are not
+     * meant to be caught.
+     */
+    private fun tryCompileIncrementally(
+        allSourceFiles: List<File>,
+        providedChangedFiles: ChangedFiles?,
+        args: Args,
+        projectDir: File?,
+        messageCollector: MessageCollector
+    ): ICResult {
+        if (providedChangedFiles is ChangedFiles.Unknown) {
+            return ICResult.RequiresRebuild(UNKNOWN_CHANGES_IN_GRADLE_INPUTS)
+        }
+        providedChangedFiles as ChangedFiles.Known?
+
+        val caches = createCacheManager(args, projectDir)
+        val exitCode: ExitCode
+        try {
+            // Step 1: Get changed files
+            val changedFiles: ChangedFiles.Known = try {
+                getChangedFiles(providedChangedFiles, allSourceFiles, caches)
+            } catch (e: Throwable) {
+                return ICResult.Failed(IC_FAILED_TO_GET_CHANGED_FILES, e)
+            }
+
+            val classpathAbiSnapshot = if (withAbiSnapshot) getClasspathAbiSnapshot(args) else null
+
+            // Step 2: Compute files to recompile
+            val compilationMode = try {
+                reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
+                    calculateSourcesToCompile(caches, changedFiles, args, messageCollector, classpathAbiSnapshot ?: emptyMap())
+                }
+            } catch (e: Throwable) {
+                return ICResult.Failed(IC_FAILED_TO_COMPUTE_FILES_TO_RECOMPILE, e)
+            }
+
+            if (compilationMode is CompilationMode.Rebuild) {
+                return ICResult.RequiresRebuild(compilationMode.reason)
+            }
+
+            val abiSnapshotData = if (withAbiSnapshot) {
+                if (!abiSnapshotFile.exists()) {
+                    reporter.debug { "Jar snapshot file does not exist: ${abiSnapshotFile.path}" }
+                    return ICResult.RequiresRebuild(NO_ABI_SNAPSHOT)
+                }
+                reporter.info { "Incremental compilation with ABI snapshot enabled" }
+                AbiSnapshotData(
+                    snapshot = AbiSnapshotImpl.read(abiSnapshotFile),
+                    classpathAbiSnapshot = classpathAbiSnapshot!!
+                )
+            } else null
+
+            // Step 3: Compile incrementally
+            exitCode = try {
+                compileImpl(compilationMode as CompilationMode.Incremental, allSourceFiles, args, caches, abiSnapshotData, messageCollector)
+            } catch (e: Throwable) {
+                return ICResult.Failed(IC_FAILED_TO_COMPILE_INCREMENTALLY, e)
+            }
+        } catch (e: Throwable) {
+            // Because `caches` is a Closeable resource, it is good practice to close them in the event of an exception (in addition to
+            // closing them after a normal use).
+            try {
+                caches.close()
+            } catch (e2: Throwable) {
+                e.addSuppressed(e2)
+            }
+            throw e
+        }
+        try {
+            caches.close()
+        } catch (e: Throwable) {
+            return ICResult.Failed(IC_FAILED_TO_CLOSE_CACHES, e)
+        }
+
+        return ICResult.Completed(exitCode)
+    }
+
+    private fun compileNonIncrementally(
+        rebuildReason: BuildAttribute,
+        allSourceFiles: List<File>,
+        args: Args,
+        projectDir: File?,
+        trackChangedFiles: Boolean, // Whether we need to track changes to the source files or the build system already handles it
+        messageCollector: MessageCollector,
+    ): ExitCode {
+        reporter.measure(BuildTime.CLEAR_OUTPUT_ON_REBUILD) {
+            val mainOutputDirs = setOf(destinationDir(args), workingDir)
+            val outputDirsToClean = outputDirs?.also {
+                check(it.containsAll(mainOutputDirs)) { "outputDirs is missing classesDir and workingDir: $it" }
+            } ?: mainOutputDirs
+
+            reporter.debug { "Cleaning output directories" }
+            cleanOrCreateDirectories(outputDirsToClean)
+        }
+        return createCacheManager(args, projectDir).use { caches ->
+            if (trackChangedFiles) {
+                caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
+            }
+            val abiSnapshotData = if (withAbiSnapshot) {
+                AbiSnapshotData(snapshot = AbiSnapshotImpl(mutableMapOf()), classpathAbiSnapshot = getClasspathAbiSnapshot(args))
+            } else null
+
+            compileImpl(CompilationMode.Rebuild(rebuildReason), allSourceFiles, args, caches, abiSnapshotData, messageCollector)
+        }
+    }
+
+    private class AbiSnapshotData(val snapshot: AbiSnapshot, val classpathAbiSnapshot: Map<String, AbiSnapshot>)
+
+    private fun getClasspathAbiSnapshot(args: Args): Map<String, AbiSnapshot> {
+        return reporter.measure(BuildTime.SET_UP_ABI_SNAPSHOTS) {
+            setupJarDependencies(args, reporter)
+        }
+    }
+
+    /**
+     * Deletes the contents of the given directories (not the directories themselves).
     *
     * If the directories do not yet exist, they will be created.
     */
-    private fun cleanOutputsAndLocalStateOnRebuild(args: Args) {
-        // Use Set as additionalOutputFiles may already contain destinationDir and workingDir
-        val outputFiles = setOf(destinationDir(args), workingDir) + additionalOutputFiles
-
-        reporter.debug { "Cleaning outputs on rebuild" }
-        outputFiles.forEach {
+    private fun cleanOrCreateDirectories(outputDirs: Collection<File>) {
+        outputDirs.toSet().forEach {
            when {
-                it.isDirectory -> {
-                    reporter.debug { "  Deleting contents of directory '${it.path}'" }
-                    it.cleanDirectoryContents()
-                }
-                it.isFile -> {
-                    reporter.debug { "  Deleting file '${it.path}'" }
-                    it.deleteRecursivelyOrThrow()
-                }
+                it.isDirectory -> it.deleteDirectoryContents()
+                it.isFile -> "Expected a directory but found a regular file: ${it.path}"
+                else -> it.createDirectory()
            }
        }
    }

-    private fun sourcesToCompile(
-        caches: CacheManager,
-        changedFiles: ChangedFiles,
-        args: Args,
-        messageCollector: MessageCollector,
-        dependenciesAbiSnapshots: Map<String, AbiSnapshot>
-    ): CompilationMode =
-        when (changedFiles) {
-            is ChangedFiles.Known -> calculateSourcesToCompile(caches, changedFiles, args, messageCollector, dependenciesAbiSnapshots)
-            is ChangedFiles.Unknown -> CompilationMode.Rebuild(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS)
-            is ChangedFiles.Dependencies -> error("Unexpected ChangedFiles type (ChangedFiles.Dependencies)")
+    private fun getChangedFiles(
+        providedChangedFiles: ChangedFiles.Known?,
+        allSourceFiles: List<File>,
+        caches: CacheManager
+    ): ChangedFiles.Known {
+        return when {
+            providedChangedFiles == null -> caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
+            providedChangedFiles.forDependencies -> {
+                val moreChangedFiles = caches.inputsCache.sourceSnapshotMap.compareAndUpdate(allSourceFiles)
+                ChangedFiles.Known(
+                    modified = providedChangedFiles.modified + moreChangedFiles.modified,
+                    removed = providedChangedFiles.removed + moreChangedFiles.removed
+                )
+            }
+            else -> providedChangedFiles
        }
+    }

-    private fun calculateSourcesToCompile(
-        caches: CacheManager, changedFiles: ChangedFiles.Known, args: Args, messageCollector: MessageCollector,
-        abiSnapshots: Map<String, AbiSnapshot>
-    ): CompilationMode =
-        reporter.measure(BuildTime.IC_CALCULATE_INITIAL_DIRTY_SET) {
-            calculateSourcesToCompileImpl(caches, changedFiles, args, messageCollector, abiSnapshots)
-        }
-
-    protected abstract fun calculateSourcesToCompileImpl(
+    protected abstract fun calculateSourcesToCompile(
        caches: CacheManager,
        changedFiles: ChangedFiles.Known,
        args: Args,
@@ -260,7 +291,7 @@ abstract class IncrementalCompilerRunner<
        classpathAbiSnapshots: Map<String, AbiSnapshot>
    ): CompilationMode

-    protected open fun setupJarDependencies(args: Args, withSnapshot: Boolean, reporter: BuildReporter): Map<String, AbiSnapshot> = mapOf()
+    protected open fun setupJarDependencies(args: Args, reporter: BuildReporter): Map<String, AbiSnapshot> = emptyMap()

    protected fun initDirtyFiles(dirtyFiles: DirtyFilesContainer, changedFiles: ChangedFiles.Known) {
        dirtyFiles.add(changedFiles.modified, "was modified since last time")
@@ -284,7 +315,6 @@ abstract class IncrementalCompilerRunner<
        changesCollector: ChangesCollector
    )

-    protected open fun preBuildHook(args: Args, compilationMode: CompilationMode) {}
    protected open fun additionalDirtyFiles(caches: CacheManager, generatedFiles: List<GeneratedFile>, services: Services): Iterable<File> =
        emptyList()

@@ -315,33 +345,53 @@ abstract class IncrementalCompilerRunner<
        isIncremental: Boolean
    ): Pair<ExitCode, Collection<File>>

-    protected open fun compileIncrementally(
+    private fun compileImpl(
+        compilationMode: CompilationMode,
+        allSourceFiles: List<File>,
        args: Args,
        caches: CacheManager,
-        allKotlinSources: List<File>,
-        compilationMode: CompilationMode,
-        originalMessageCollector: MessageCollector,
-        withSnapshot: Boolean,
-        abiSnapshot: AbiSnapshot = AbiSnapshotImpl(mutableMapOf()),
-        classpathAbiSnapshot: Map<String, AbiSnapshot> = HashMap()
+        abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true
+        messageCollector: MessageCollector
    ): ExitCode {
-        if (compilationMode is CompilationMode.Rebuild) {
-            reporter.info { "Non-incremental compilation will be performed: ${compilationMode.reason}" }
+        performWorkBeforeCompilation(compilationMode, args)
+
+        val allKotlinFiles = allSourceFiles.filter { it.isKotlinFile(kotlinSourceFilesExtensions) }
+        val exitCode = doCompile(compilationMode, allKotlinFiles, args, caches, abiSnapshotData, messageCollector)
+
+        performWorkAfterCompilation(compilationMode, exitCode, caches)
+        return exitCode
+    }
+
+    protected open fun performWorkBeforeCompilation(compilationMode: CompilationMode, args: Args) {}
+
+    protected open fun performWorkAfterCompilation(compilationMode: CompilationMode, exitCode: ExitCode, caches: CacheManager) {
+        collectMetrics()
+    }
+
+    private fun collectMetrics() {
+        reporter.measure(BuildTime.CALCULATE_OUTPUT_SIZE) {
+            reporter.addMetric(
+                BuildPerformanceMetric.SNAPSHOT_SIZE,
+                buildHistoryFile.length() + lastBuildInfoFile.length() + abiSnapshotFile.length()
+            )
+            reporter.addMetric(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE, cacheDirectory.walk().sumOf { it.length() })
        }
+    }

-        preBuildHook(args, compilationMode)
-
+    private fun doCompile(
+        compilationMode: CompilationMode,
+        allKotlinSources: List<File>,
+        args: Args,
+        caches: CacheManager,
+        abiSnapshotData: AbiSnapshotData?, // Not null iff withAbiSnapshot = true
+        originalMessageCollector: MessageCollector
+    ): ExitCode {
        val dirtySources = when (compilationMode) {
-            is CompilationMode.Incremental -> {
-                compilationMode.dirtyFiles.toMutableLinkedSet()
-            }
-            is CompilationMode.Rebuild -> {
-                reporter.addAttribute(compilationMode.reason)
-                LinkedHashSet(allKotlinSources)
-            }
+            is CompilationMode.Incremental -> compilationMode.dirtyFiles.toMutableLinkedSet()
+            is CompilationMode.Rebuild -> LinkedHashSet(allKotlinSources)
        }

-        val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis(), classpathAbiSnapshot)
+        val currentBuildInfo = BuildInfo(startTS = System.currentTimeMillis(), abiSnapshotData?.classpathAbiSnapshot ?: emptyMap())
        val buildDirtyLookupSymbols = HashSet<LookupSymbol>()
        val buildDirtyFqNames = HashSet<FqName>()
        val allDirtySources = HashSet<File>()
@@ -412,8 +462,8 @@ abstract class IncrementalCompilerRunner<
                updateCaches(services, caches, generatedFiles, changesCollector)
            }
            if (compilationMode is CompilationMode.Rebuild) {
-                if (withSnapshot) {
-                    abiSnapshot.protos.putAll(changesCollector.protoDataChanges())
+                if (withAbiSnapshot) {
+                    abiSnapshotData!!.snapshot.protos.putAll(changesCollector.protoDataChanges())
                }
                break
            }
@@ -445,10 +495,10 @@ abstract class IncrementalCompilerRunner<
            buildDirtyFqNames.addAll(dirtyClassFqNames)

            //update
-            if (withSnapshot) {
+            if (withAbiSnapshot) {
                //TODO(valtman) check method/ kts class remove
-                changesCollector.protoDataRemoved().forEach { abiSnapshot.protos.remove(it) }
-                abiSnapshot.protos.putAll(changesCollector.protoDataChanges())
+                changesCollector.protoDataRemoved().forEach { abiSnapshotData!!.snapshot.protos.remove(it) }
+                abiSnapshotData!!.snapshot.protos.putAll(changesCollector.protoDataChanges())
            }
        }

@@ -457,9 +507,9 @@ abstract class IncrementalCompilerRunner<
                BuildInfo.write(currentBuildInfo, lastBuildInfoFile)

                //write abi snapshot
-                if (withSnapshot) {
+                if (withAbiSnapshot) {
                    //TODO(valtman) check method/class remove
-                    AbiSnapshotImpl.write(abiSnapshot, abiSnapshotFile)
+                    AbiSnapshotImpl.write(abiSnapshotData!!.snapshot, abiSnapshotFile)
                }
            }
        }
@@ -516,15 +566,6 @@ abstract class IncrementalCompilerRunner<
        BuildDiffsStorage.writeToFile(buildHistoryFile, BuildDiffsStorage(prevDiffs + newDiff), reporter)
    }

-    /**
-     * Performs some work after a compilation if the compilation completed successfully (no exceptions were thrown AND exit code == 0).
-     *
-     * This method MUST NOT be called when the compilation failed because the results produced by the work here would be invalid.
-     *
-     * @wasIncremental whether the compilation was incremental or non-incremental
-     */
-    protected open fun performWorkAfterSuccessfulCompilation(caches: CacheManager, wasIncremental: Boolean) {}
-
    companion object {
        const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt"
        const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin"
@@ -72,7 +72,7 @@ class IncrementalFirJvmCompilerRunner(
    workingDir: File,
    reporter: BuildReporter,
    buildHistoryFile: File,
-    outputFiles: Collection<File>,
+    outputDirs: Collection<File>?,
    modulesApiHistory: ModulesApiHistory,
    kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
    classpathChanges: ClasspathChanges
@@ -81,7 +81,7 @@ class IncrementalFirJvmCompilerRunner(
    reporter,
    false,
    buildHistoryFile,
-    outputFiles,
+    outputDirs,
    modulesApiHistory,
    kotlinSourceFilesExtensions,
    classpathChanges
@@ -93,6 +93,7 @@ class IncrementalJsCompilerRunner(
    "caches-js",
    reporter,
    buildHistoryFile = buildHistoryFile,
+    outputDirs = null,
    withAbiSnapshot = withAbiSnapshot
 ) {

@@ -115,7 +116,7 @@ class IncrementalJsCompilerRunner(
            outputFile.parentFile
    }

-    override fun calculateSourcesToCompileImpl(
+    override fun calculateSourcesToCompile(
        caches: IncrementalJsCachesManager,
        changedFiles: ChangedFiles.Known,
        args: K2JSCompilerArguments,
@@ -88,7 +88,13 @@ fun makeIncrementally(
        val compiler =
            if (args.useK2 && args.useFirIC && args.useFirLT /* TODO: move LT check into runner */ )
                IncrementalFirJvmCompilerRunner(
-                    cachesDir, buildReporter, buildHistoryFile, emptyList(), EmptyModulesApiHistory, kotlinExtensions, ClasspathSnapshotDisabled
+                    cachesDir,
+                    buildReporter,
+                    buildHistoryFile,
+                    outputDirs = null,
+                    EmptyModulesApiHistory,
+                    kotlinExtensions,
+                    ClasspathSnapshotDisabled
                )
            else
                IncrementalJvmCompilerRunner(
@@ -96,8 +102,8 @@ fun makeIncrementally(
                    buildReporter,
                    // Use precise setting in case of non-Gradle build
                    usePreciseJavaTracking = !args.useK2, // TODO: add fir-based java classes tracker when available and set this to true
-                    outputFiles = emptyList(),
                    buildHistoryFile = buildHistoryFile,
+                    outputDirs = null,
                    modulesApiHistory = EmptyModulesApiHistory,
                    kotlinSourceFilesExtensions = kotlinExtensions,
                    classpathChanges = ClasspathSnapshotDisabled
@@ -127,7 +133,7 @@ open class IncrementalJvmCompilerRunner(
    reporter: BuildReporter,
    private val usePreciseJavaTracking: Boolean,
    buildHistoryFile: File,
-    outputFiles: Collection<File>,
+    outputDirs: Collection<File>?,
    private val modulesApiHistory: ModulesApiHistory,
    override val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
    private val classpathChanges: ClasspathChanges,
@@ -136,8 +142,8 @@ open class IncrementalJvmCompilerRunner(
    workingDir,
    "caches-jvm",
    reporter,
-    additionalOutputFiles = outputFiles,
    buildHistoryFile = buildHistoryFile,
+    outputDirs = outputDirs,
    withAbiSnapshot = withAbiSnapshot
 ) {
    override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager =
@@ -181,7 +187,7 @@ open class IncrementalJvmCompilerRunner(
        else
            null

-    override fun calculateSourcesToCompileImpl(
+    override fun calculateSourcesToCompile(
        caches: IncrementalJvmCachesManager,
        changedFiles: ChangedFiles.Known,
        args: K2JVMCompilerArguments,
@@ -199,9 +205,8 @@ open class IncrementalJvmCompilerRunner(
    //TODO can't use the same way as for build-history files because abi-snapshot for all dependencies should be stored into last-build
    // and not only changed one
    // (but possibly we dont need to read it all and may be it is possible to update only those who was changed)
-    override fun setupJarDependencies(args: K2JVMCompilerArguments, withSnapshot: Boolean, reporter: BuildReporter): Map<String, AbiSnapshot> {
+    override fun setupJarDependencies(args: K2JVMCompilerArguments, reporter: BuildReporter): Map<String, AbiSnapshot> {
        //fill abiSnapshots
-        if (!withSnapshot) return emptyMap()
        val abiSnapshots = HashMap<String, AbiSnapshot>()
        args.classpathAsList
            .filter { it.extension.equals("jar", ignoreCase = true) }
@@ -209,7 +214,12 @@ open class IncrementalJvmCompilerRunner(
                modulesApiHistory.abiSnapshot(it).let { result ->
                    if (result is Either.Success<Set<File>>) {
                        result.value.forEach { file ->
-                            AbiSnapshotImpl.read(file, reporter)?.also { abiSnapshot -> abiSnapshots[it.absolutePath] = abiSnapshot }
+                            if (file.exists()) {
+                                abiSnapshots[it.absolutePath] = AbiSnapshotImpl.read(file)
+                            } else {
+                                // FIXME: We should throw an exception here
+                                reporter.warn { "Snapshot file does not exist: ${file.path}. Continue anyway." }
+                            }
                        }
                    }
                }
@@ -228,7 +238,7 @@ open class IncrementalJvmCompilerRunner(
        caches: IncrementalJvmCachesManager,
        changedFiles: ChangedFiles.Known,
        args: K2JVMCompilerArguments,
-        abiSnapshots: Map<String, AbiSnapshot> = HashMap(),
+        abiSnapshots: Map<String, AbiSnapshot>,
        withAbiSnapshot: Boolean
    ): CompilationMode {
        val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions)
@@ -370,7 +380,9 @@ open class IncrementalJvmCompilerRunner(
        return result
    }

-    override fun preBuildHook(args: K2JVMCompilerArguments, compilationMode: CompilationMode) {
+    override fun performWorkBeforeCompilation(compilationMode: CompilationMode, args: K2JVMCompilerArguments) {
+        super.performWorkBeforeCompilation(compilationMode, args)
+
        if (compilationMode is CompilationMode.Incremental) {
            val destinationDir = args.destinationAsFile
            destinationDir.mkdirs()
@@ -478,12 +490,15 @@ open class IncrementalJvmCompilerRunner(
        return exitCode to sourcesToCompile
    }

-    override fun performWorkAfterSuccessfulCompilation(caches: IncrementalJvmCachesManager, wasIncremental: Boolean) {
-        if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) {
+    override fun performWorkAfterCompilation(compilationMode: CompilationMode, exitCode: ExitCode, caches: IncrementalJvmCachesManager) {
+        super.performWorkAfterCompilation(compilationMode, exitCode, caches)
+
+        // No need to shrink and save classpath snapshot if exitCode != ExitCode.OK as the task will fail anyway
+        if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled && exitCode == ExitCode.OK) {
            reporter.measure(BuildTime.SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
                shrinkAndSaveClasspathSnapshot(
-                    wasIncremental, classpathChanges, caches.lookupCache, currentClasspathSnapshot,
-                    shrunkCurrentClasspathAgainstPreviousLookups, ClasspathSnapshotBuildReporter(reporter)
+                    compilationWasIncremental = compilationMode is CompilationMode.Incremental, classpathChanges, caches.lookupCache,
+                    currentClasspathSnapshot, shrunkCurrentClasspathAgainstPreviousLookups, ClasspathSnapshotBuildReporter(reporter)
                )
            }
        }
@@ -213,7 +213,8 @@ abstract class AbstractIncrementalMultiModuleCompilerRunnerTest<Args : CommonCom
            val moduleModifiedDependencies = modifiedLibraries.filter { it.first in moduleDependencies }.map { it.second }
            val moduleDeletedDependencies = deletedLibraries.filter { it.first in moduleDependencies }.map { it.second }

-            val changedDepsFiles = if (isInitial) null else ChangedFiles.Dependencies(moduleModifiedDependencies, moduleDeletedDependencies)
+            val changedDepsFiles =
+                if (isInitial) null else ChangedFiles.Known(moduleModifiedDependencies, moduleDeletedDependencies, forDependencies = true)

            val moduleOutDir = File(outDir, module)
            val moduleCacheDir = File(cacheDir, module)