KT-45777: Shrink classpath snapshot incrementally

Currently, we shrink classpath snapshots at 2 steps: - Classpath diffing: Shrink the current classpath snapshot against the previous lookup symbols - Classpath snapshot saving: Shrink the current classpath snapshot against the current lookup symbols With this commit, the shrinking at the second step is now incremental. The shrinking at the first step is still non-incremental.
2021-12-05 19:54:20 +00:00
parent 4f3debdec6
commit 586fa8af64
20 changed files with 366 additions and 116 deletions
@@ -1,6 +1,7 @@
 <component name="ProjectDictionaryState">
  <dictionary name="hungnv">
    <words>
+      <w>shrinker</w>
      <w>snapshotter</w>
      <w>snapshotter's</w>
    </words>
@@ -29,7 +29,11 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String)
                IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION, "Init dirty symbols set"),
                    COMPUTE_CLASSPATH_CHANGES(IC_CALCULATE_INITIAL_DIRTY_SET, "Compute classpath changes"),
                        LOAD_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load current classpath snapshot"),
+                            REMOVE_DUPLICATE_CLASSES(LOAD_CURRENT_CLASSPATH_SNAPSHOT, "Remove duplicate classes"),
                        SHRINK_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Shrink current classpath snapshot"),
+                            GET_LOOKUP_SYMBOLS(SHRINK_CURRENT_CLASSPATH_SNAPSHOT, "Get lookup symbols"),
+                            FIND_REFERENCED_CLASSES(SHRINK_CURRENT_CLASSPATH_SNAPSHOT, "Find referenced classes"),
+                            FIND_TRANSITIVELY_REFERENCED_CLASSES(SHRINK_CURRENT_CLASSPATH_SNAPSHOT, "Find transitively referenced classes"),
                        LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load shrunk previous classpath snapshot"),
                        COMPUTE_CHANGED_AND_IMPACTED_SET(COMPUTE_CLASSPATH_CHANGES, "Compute changed and impacted set"),
                            COMPUTE_CLASS_CHANGES(COMPUTE_CHANGED_AND_IMPACTED_SET, "Compute class changes"),
@@ -47,14 +51,11 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String)
                INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Incremental iteration"),
                NON_INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Non-incremental iteration"),
                IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION, "Write history file"),
-                SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION, "Save shrunk current classpath snapshot after compilation"),
-                    LOAD_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load classpath snapshot"),
-                    SHRINK_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink classpath snapshot"),
-                        GET_NON_DUPLICATE_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Get non-duplicate classes"),
-                        GET_LOOKUP_SYMBOLS(SHRINK_CLASSPATH_SNAPSHOT, "Get lookup symbols"),
-                        FIND_REFERENCED_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Find referenced classes"),
-                        FIND_TRANSITIVELY_REFERENCED_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Find transitively referenced classes"),
-                    SAVE_SHRUNK_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk classpath snapshot"),
+                SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION, "Shrink and save current classpath snapshot after compilation"),
+                    LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load shrunk previous classpath snapshot after compilation"),
+                    LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load current classpath snapshot after compilation"),
+                    SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink current classpath snapshot after compilation"),
+                    SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk classpath snapshot after compilation"),
    COMPILER_PERFORMANCE(readableString = "Compiler time"),
        COMPILER_INITIALIZATION(COMPILER_PERFORMANCE, "Compiler initialization time"),
        CODE_ANALYSIS(COMPILER_PERFORMANCE, "Compiler code analyse"),
@@ -11,7 +11,7 @@ import java.io.Serializable

 /**
 * Changes to the classpath of the `KotlinCompile` task, or information to compute them later by the Kotlin incremental compiler (see
- * [ClasspathSnapshotEnabled.ToBeComputedByIncrementalCompiler].
+ * [ClasspathSnapshotEnabled.IncrementalRun.ToBeComputedByIncrementalCompiler].
 */
 sealed class ClasspathChanges : Serializable {

@@ -19,9 +19,12 @@ sealed class ClasspathChanges : Serializable {

        abstract val classpathSnapshotFiles: ClasspathSnapshotFiles

-        class Empty(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled()
+        sealed class IncrementalRun : ClasspathSnapshotEnabled() {

-        class ToBeComputedByIncrementalCompiler(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled()
+            class NoChanges(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : IncrementalRun()
+
+            class ToBeComputedByIncrementalCompiler(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : IncrementalRun()
+        }

        class NotAvailableDueToMissingClasspathSnapshot(override val classpathSnapshotFiles: ClasspathSnapshotFiles) :
            ClasspathSnapshotEnabled()
@@ -33,7 +33,8 @@ import java.util.*
 open class LookupStorage(
    targetDataDir: File,
    pathConverter: FileToPathConverter,
-    storeFullFqNames: Boolean = false
+    storeFullFqNames: Boolean = false,
+    private val trackChanges: Boolean = false
 ) : BasicMapsOwner(targetDataDir) {
    val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder")

@@ -45,7 +46,7 @@ open class LookupStorage(
    private val countersFile = "counters".storageFile
    private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, pathConverter))
    private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, pathConverter))
-    val lookupMap = registerMap(LookupMap("lookups".storageFile, storeFullFqNames))
+    private val lookupMap = TrackedLookupMap(registerMap(LookupMap("lookups".storageFile, storeFullFqNames)), trackChanges)

    @Volatile
    private var size: Int = 0
@@ -56,7 +57,8 @@ open class LookupStorage(
            if (countersFile.exists()) {
                val lines = countersFile.readLines()
                size = lines.firstOrNull()?.toIntOrNull() ?: throw IOException("$countersFile exists, but it is empty. " +
-                                                                                       "Counters file is corrupted")
+                                                                                       "Counters file is corrupted"
+                )
                oldSize = size
            }
        } catch (e: IOException) {
@@ -66,6 +68,24 @@ open class LookupStorage(
        }
    }

+    /** Set of [LookupSymbol]s that have been added after the initialization of this [LookupStorage] instance. */
+    val addedLookupSymbols: Set<LookupSymbolKey>
+        get() = run {
+            check(trackChanges) { "trackChanges is not enabled" }
+            lookupMap.addedKeys!!
+        }
+
+    /** Set of [LookupSymbol]s that have been removed after the initialization of this [LookupStorage] instance. */
+    val removedLookupSymbols: Set<LookupSymbolKey>
+        get() = run {
+            check(trackChanges) { "trackChanges is not enabled" }
+            lookupMap.removedKeys!!
+        }
+
+    /** Returns all [LookupSymbol]s in this storage. Note that this call takes a bit of time to run. */
+    val lookupSymbols: Collection<LookupSymbolKey>
+        get() = lookupMap.keys
+
    @Synchronized
    fun get(lookupSymbol: LookupSymbol): Collection<String> {
        val key = LookupSymbolKey(lookupSymbol.name, lookupSymbol.scope)
@@ -266,3 +286,57 @@ data class LookupSymbol(val name: String, val scope: String) : Comparable<Lookup
        return name.compareTo(other.name)
    }
 }
+
+/**
+ * Wrapper of a [LookupMap] which tracks changes to the map after the initialization of this [TrackedLookupMap] instance, (unless
+ * [trackChanges] is set to `false`).
+ */
+private class TrackedLookupMap(private val lookupMap: LookupMap, private val trackChanges: Boolean) {
+
+    // Note that there may be multiple operations on the same key, and the following sets contain the *latest* differences with the original
+    // set of keys in the map. For example, if a key is added then removed, or vice versa, it will not be present in either set.
+    val addedKeys = if (trackChanges) mutableSetOf<LookupSymbolKey>() else null
+    val removedKeys = if (trackChanges) mutableSetOf<LookupSymbolKey>() else null
+
+    val keys: Collection<LookupSymbolKey>
+        get() = lookupMap.keys
+
+    operator fun get(key: LookupSymbolKey): Collection<Int>? = lookupMap[key]
+
+    operator fun set(key: LookupSymbolKey, fileIds: Set<Int>) {
+        recordSet(key)
+        lookupMap[key] = fileIds
+    }
+
+    fun append(key: LookupSymbolKey, fileIds: Collection<Int>) {
+        recordSet(key)
+        lookupMap.append(key, fileIds)
+    }
+
+    fun remove(key: LookupSymbolKey) {
+        recordRemove(key)
+        lookupMap.remove(key)
+    }
+
+    private fun recordSet(key: LookupSymbolKey) {
+        if (!trackChanges) return
+        if (lookupMap[key] == null) {
+            if (key in removedKeys!!) {
+                removedKeys.remove(key)
+            } else {
+                addedKeys!!.add(key)
+            }
+        }
+    }
+
+    private fun recordRemove(key: LookupSymbolKey) {
+        if (!trackChanges) return
+        if (lookupMap[key] != null) {
+            if (key in addedKeys!!) {
+                addedKeys.remove(key)
+            } else {
+                removedKeys!!.add(key)
+            }
+        }
+    }
+}
@@ -27,40 +27,38 @@ import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 import java.io.*

-/**
- * Storage versioning:
- * 0 - only name and value hashes are saved
- * 1 - name and scope are saved
- */
 class LookupSymbolKeyDescriptor(
    /** If `true`, original values are saved; if `false`, only hashes are saved. */
    private val storeFullFqNames: Boolean = false
 ) : KeyDescriptor<LookupSymbolKey> {

    override fun read(input: DataInput): LookupSymbolKey {
-        val version = input.readByte()
-        return when (version.toInt()) {
-            0 -> {
-                val name = input.readUTF()
-                val scope = input.readUTF()
-                LookupSymbolKey(name.hashCode(), scope.hashCode(), name, scope)
-            }
-            1 -> {
-                val first = input.readInt()
-                val second = input.readInt()
-                LookupSymbolKey(first, second, "", "")
-            }
-            else -> throw IllegalArgumentException("Unknown version of LookupSymbolKeyDescriptor=${version}")
+        // Note: The value of the storeFullFqNames variable below may or may not be the same as LookupSymbolKeyDescriptor.storeFullFqNames.
+        // Byte value `0` means storeFullFqNames == true, see `save` function below.
+        val storeFullFqNames = when (val byteValue = input.readByte().toInt()) {
+            0 -> true
+            1 -> false
+            else -> error("Unexpected byte value for storeFullFqNames: $byteValue")
+        }
+        return if (storeFullFqNames) {
+            val name = input.readUTF()
+            val scope = input.readUTF()
+            LookupSymbolKey(name.hashCode(), scope.hashCode(), name, scope)
+        } else {
+            val nameHash = input.readInt()
+            val scopeHash = input.readInt()
+            LookupSymbolKey(nameHash, scopeHash, "", "")
        }
    }

    override fun save(output: DataOutput, value: LookupSymbolKey) {
+        // Write a Byte value `0` to represent storeFullFqNames == true for historical reasons (if we switch this value to `1` or write a
+        // Boolean instead, it might impact some tests).
+        output.writeByte(if (storeFullFqNames) 0 else 1)
        if (storeFullFqNames) {
-            output.writeByte(0)
            output.writeUTF(value.name)
            output.writeUTF(value.scope)
        } else {
-            output.writeByte(1)
            output.writeInt(value.nameHash)
            output.writeInt(value.scopeHash)
        }
@@ -27,7 +27,9 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
    cachesRootDir: File,
    rootProjectDir: File?,
    protected val reporter: ICReporter,
-    storeFullFqNamesInLookupCache: Boolean = false) {
+    storeFullFqNamesInLookupCache: Boolean = false,
+    trackChangesInLookupCache: Boolean = false
+) {
    val pathConverter = IncrementalFileToPathConverter(rootProjectDir)
    private val caches = arrayListOf<BasicMapsOwner>()

@@ -43,7 +45,8 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
    private val lookupCacheDir = File(cachesRootDir, "lookups").apply { mkdirs() }

    val inputsCache: InputsCache = InputsCache(inputSnapshotsCacheDir, reporter, pathConverter).apply { registerCache() }
-    val lookupCache: LookupStorage = LookupStorage(lookupCacheDir, pathConverter, storeFullFqNamesInLookupCache).apply { registerCache() }
+    val lookupCache: LookupStorage =
+        LookupStorage(lookupCacheDir, pathConverter, storeFullFqNamesInLookupCache, trackChangesInLookupCache).apply { registerCache() }
    abstract val platformCache: PlatformCache

    @Synchronized
@@ -80,8 +83,15 @@ class IncrementalJvmCachesManager(
    rootProjectDir: File?,
    outputDir: File,
    reporter: ICReporter,
-    storeFullFqNamesInLookupCache: Boolean = false
-    ) : IncrementalCachesManager<IncrementalJvmCache>(cacheDirectory, rootProjectDir, reporter, storeFullFqNamesInLookupCache) {
+    storeFullFqNamesInLookupCache: Boolean = false,
+    trackChangesInLookupCache: Boolean = false
+) : IncrementalCachesManager<IncrementalJvmCache>(
+    cacheDirectory,
+    rootProjectDir,
+    reporter,
+    storeFullFqNamesInLookupCache,
+    trackChangesInLookupCache
+) {
    private val jvmCacheDir = File(cacheDirectory, "jvm").apply { mkdirs() }
    override val platformCache = IncrementalJvmCache(jvmCacheDir, outputDir, pathConverter).apply { registerCache() }
 }
@@ -126,7 +126,7 @@ class IncrementalJsCompilerRunner(
        val classpathChanges = getClasspathChanges(
            libs, changedFiles, lastBuildInfo, modulesApiHistory, reporter,
            mapOf(), false, caches.platformCache,
-            caches.lookupCache.lookupMap.keys.map { if (it.scope.isBlank()) it.name else it.scope }.distinct()
+            caches.lookupCache.lookupSymbols.map { if (it.scope.isBlank()) it.name else it.scope }.distinct()
        )

        @Suppress("UNUSED_VARIABLE") // for sealed when
@@ -29,7 +29,10 @@ import org.jetbrains.kotlin.build.GeneratedJvmClass
 import org.jetbrains.kotlin.build.report.BuildReporter
 import org.jetbrains.kotlin.build.report.ICReporter
 import org.jetbrains.kotlin.build.report.ICReporterBase
-import org.jetbrains.kotlin.build.report.metrics.*
+import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
+import org.jetbrains.kotlin.build.report.metrics.BuildTime
+import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
+import org.jetbrains.kotlin.build.report.metrics.measure
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
@@ -42,15 +45,16 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotDisabled
-import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.*
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.NoChanges
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.ToBeComputedByIncrementalCompiler
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableDueToMissingClasspathSnapshot
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun
 import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailableForJSCompiler
 import org.jetbrains.kotlin.incremental.classpathDiff.*
 import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory
 import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
-import org.jetbrains.kotlin.incremental.storage.ListExternalizer
-import org.jetbrains.kotlin.incremental.storage.saveToFile
 import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector
 import org.jetbrains.kotlin.incremental.util.Either
 import org.jetbrains.kotlin.load.java.JavaClassesTracker
@@ -141,8 +145,9 @@ class IncrementalJvmCompilerRunner(
            projectDir,
            File(args.destination),
            reporter,
-            storeFullFqNamesInLookupCache = withSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled
-            )
+            storeFullFqNamesInLookupCache = withSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled,
+            trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun
+        )

    override fun destinationDir(args: K2JVMCompilerArguments): File =
        args.destinationAsFile
@@ -210,9 +215,10 @@ class IncrementalJvmCompilerRunner(
        return abiSnapshots
    }

-    // Used by `calculateSourcesToCompileImpl` and `performWorkAfterCompilation` methods below.
-    // Thread safety: There is no concurrent access to this variable.
-    private var currentClasspathSnapshot: ClasspathSnapshot? = null
+    // Used by `calculateSourcesToCompileImpl` and `performWorkAfterSuccessfulCompilation` methods below.
+    // Thread safety: There is no concurrent access to these variables.
+    private var currentClasspathSnapshot: List<ClassSnapshotWithHash>? = null
+    private var shrunkCurrentClasspathAgainstPreviousLookups: List<ClassSnapshotWithHash>? = null

    private fun calculateSourcesToCompileImpl(
        caches: IncrementalJvmCachesManager,
@@ -225,15 +231,22 @@ class IncrementalJvmCompilerRunner(

        val classpathChanges = when (classpathChanges) {
            // Note: classpathChanges is deserialized, so they are no longer singleton objects and need to be compared using `is` (not `==`)
-            is Empty -> ChangesEither.Known(emptySet(), emptySet())
+            is NoChanges -> ChangesEither.Known(emptySet(), emptySet())
            is ToBeComputedByIncrementalCompiler -> reporter.measure(BuildTime.COMPUTE_CLASSPATH_CHANGES) {
                check(currentClasspathSnapshot == null)
                currentClasspathSnapshot = reporter.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT) {
-                    CachedClasspathSnapshotSerializer.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
+                    val classpathSnapshot =
+                        CachedClasspathSnapshotSerializer.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
+                    reporter.measure(BuildTime.REMOVE_DUPLICATE_CLASSES) {
+                        classpathSnapshot.removeDuplicateAndInaccessibleClasses()
+                    }
+                }
+                check(shrunkCurrentClasspathAgainstPreviousLookups == null)
+                shrunkCurrentClasspathAgainstPreviousLookups = reporter.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT) {
+                    ClasspathSnapshotShrinker.shrink(currentClasspathSnapshot!!, caches.lookupCache, reporter)
                }
                ClasspathChangesComputer.computeChangedAndImpactedSet(
-                    currentClasspathSnapshot!!,
-                    caches.lookupCache,
+                    shrunkCurrentClasspathAgainstPreviousLookups!!,
                    classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile,
                    reporter
                ).getChanges()
@@ -243,7 +256,7 @@ class IncrementalJvmCompilerRunner(
            is ClasspathSnapshotDisabled -> reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) {
                val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild(BuildAttribute.IC_IS_NOT_ENABLED)
                reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" }
-                val scopes = caches.lookupCache.lookupMap.keys.map { it.scope.ifBlank { it.name } }.distinct()
+                val scopes = caches.lookupCache.lookupSymbols.map { it.scope.ifBlank { it.name } }.distinct()

                getClasspathChanges(
                    args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter, abiSnapshots, withSnapshot,
@@ -456,35 +469,14 @@ class IncrementalJvmCompilerRunner(

    override fun performWorkAfterSuccessfulCompilation(caches: IncrementalJvmCachesManager) {
        if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) {
-            reporter.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
-                shrinkAndSaveClasspathSnapshot(classpathChanges, caches.lookupCache)
+            reporter.measure(BuildTime.SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+                shrinkAndSaveClasspathSnapshot(
+                    classpathChanges, caches.lookupCache, currentClasspathSnapshot, shrunkCurrentClasspathAgainstPreviousLookups,
+                    reporter
+                )
            }
        }
    }
-
-    private fun shrinkAndSaveClasspathSnapshot(classpathChanges: ClasspathChanges.ClasspathSnapshotEnabled, lookupStorage: LookupStorage) {
-        val classpathSnapshot = currentClasspathSnapshot ?: reporter.measure(BuildTime.LOAD_CLASSPATH_SNAPSHOT) {
-            CachedClasspathSnapshotSerializer.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
-        }
-        val shrunkClasspathSnapshot = reporter.measure(BuildTime.SHRINK_CLASSPATH_SNAPSHOT) {
-            ClasspathSnapshotShrinker.shrink(classpathSnapshot, lookupStorage, reporter)
-        }
-        reporter.measure(BuildTime.SAVE_SHRUNK_CLASSPATH_SNAPSHOT) {
-            ListExternalizer(ClassSnapshotWithHashExternalizer).saveToFile(
-                classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile,
-                shrunkClasspathSnapshot
-            )
-        }
-
-        reporter.addMetric(
-            BuildPerformanceMetric.ORIGINAL_CLASSPATH_SNAPSHOT_SIZE,
-            classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.sumOf { it.length() }
-        )
-        reporter.addMetric(
-            BuildPerformanceMetric.SHRUNK_CLASSPATH_SNAPSHOT_SIZE,
-            classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.length()
-        )
-    }
 }

 var K2JVMCompilerArguments.destinationAsFile: File
@@ -27,19 +27,16 @@ import java.util.*
 object ClasspathChangesComputer {

    /**
-     * Computes changes between the current and previous [ClasspathSnapshot]s, plus unchanged elements that are impacted by the changes.
+     * Computes changes between the current and previous shrunk [ClasspathSnapshot]s, plus unchanged elements that are impacted by the
+     * changes.
     *
     * NOTE: The original classpath may contain duplicate classes, but the shrunk classpath must not contain duplicate classes.
     */
    fun computeChangedAndImpactedSet(
-        currentClasspathSnapshot: ClasspathSnapshot,
-        lookupStorageInPreviousRun: LookupStorage,
+        shrunkCurrentClasspathSnapshot: List<ClassSnapshotWithHash>,
        shrunkPreviousClasspathSnapshotFile: File,
        metrics: BuildMetricsReporter
    ): ChangeSet {
-        val shrunkCurrentClasspathSnapshot = metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT) {
-            ClasspathSnapshotShrinker.shrink(currentClasspathSnapshot, lookupStorageInPreviousRun, metrics)
-        }
        val shrunkPreviousClasspathSnapshot = metrics.measure(BuildTime.LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT) {
            ListExternalizer(ClassSnapshotWithHashExternalizer).loadFromFile(shrunkPreviousClasspathSnapshotFile)
        }
@@ -41,17 +41,25 @@ class ClasspathEntrySnapshot(
 sealed class ClassSnapshot {

    /** Computes the hash of this [ClassSnapshot] and returns a [ClassSnapshotWithHash]. */
-    fun addHash() = ClassSnapshotWithHash(this, ClassSnapshotExternalizer.toByteArray(this).md5())
+    val withHash: ClassSnapshotWithHash by lazy {
+        ClassSnapshotWithHash(this, ClassSnapshotExternalizer.toByteArray(this).md5())
+    }
 }

 /** Contains a [ClassSnapshot] and its hash. */
-class ClassSnapshotWithHash(val classSnapshot: ClassSnapshot, val hash: Long)
+class ClassSnapshotWithHash(val classSnapshot: ClassSnapshot, val hash: Long) {
+
+    override fun toString() = classSnapshot.toString()
+}

 /** [ClassSnapshot] of a Kotlin class. */
 class KotlinClassSnapshot(
    val classInfo: KotlinClassInfo,
    val supertypes: List<JvmClassName>
-) : ClassSnapshot()
+) : ClassSnapshot() {
+
+    override fun toString() = classInfo.classId.toString()
+}

 /** [ClassSnapshot] of a Java class. */
 sealed class JavaClassSnapshot : ClassSnapshot()
@@ -81,6 +89,8 @@ class RegularJavaClassSnapshot(
            check(it == JvmClassName.byInternalName(classAbiExcludingMembers.name))
        }
    }
+
+    override fun toString() = classId.toString()
 }

 /** The ABI snapshot of a Java element (e.g., class, field, or method). */
@@ -5,11 +5,18 @@

 package org.jetbrains.kotlin.incremental.classpathDiff

-import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
-import org.jetbrains.kotlin.build.report.metrics.BuildTime
-import org.jetbrains.kotlin.build.report.metrics.measure
+import org.jetbrains.kotlin.build.report.metrics.*
+import org.jetbrains.kotlin.incremental.ClasspathChanges
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.NoChanges
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.ToBeComputedByIncrementalCompiler
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableDueToMissingClasspathSnapshot
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun
 import org.jetbrains.kotlin.incremental.LookupStorage
+import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotShrinker.shrink
+import org.jetbrains.kotlin.incremental.storage.ListExternalizer
 import org.jetbrains.kotlin.incremental.storage.LookupSymbolKey
+import org.jetbrains.kotlin.incremental.storage.loadFromFile
+import org.jetbrains.kotlin.incremental.storage.saveToFile
 import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
@@ -18,24 +25,25 @@ import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 object ClasspathSnapshotShrinker {

    /**
-     * Shrinks the given [ClasspathSnapshot] by retaining only classes that are referenced. Referencing info is stored in [LookupStorage].
-     *
-     * This method also removes duplicate classes and [InaccessibleClassSnapshot]s first.
+     * Shrinks the given classes by retaining only classes that are referenced by the lookup symbols stored in the given [LookupStorage].
     */
    fun shrink(
-        classpathSnapshot: ClasspathSnapshot,
+        allClasses: List<ClassSnapshotWithHash>,
        lookupStorage: LookupStorage,
-        metrics: BuildMetricsReporter
+        metrics: BuildMetricsReporter = DoNothingBuildMetricsReporter
    ): List<ClassSnapshotWithHash> {
-        val allClasses = metrics.measure(BuildTime.GET_NON_DUPLICATE_CLASSES) {
-            // It's important to remove duplicate classes first before removing `InaccessibleClassSnapshot`s.
-            // For example, if jar1!/com/example/A.class is empty and jar2!/com/example/A.class is non-empty, incorrect order of the actions
-            // will lead to incorrect results.
-            classpathSnapshot.getNonDuplicateClassSnapshots().filter { it.classSnapshot !is InaccessibleClassSnapshot }
-        }
        val lookupSymbols = metrics.measure(BuildTime.GET_LOOKUP_SYMBOLS) {
-            lookupStorage.lookupMap.keys
+            lookupStorage.lookupSymbols
        }
+        return shrink(allClasses, lookupSymbols, metrics)
+    }
+
+    /** Shrinks the given classes by retaining only classes that are referenced by the given lookup symbols. */
+    fun shrink(
+        allClasses: List<ClassSnapshotWithHash>,
+        lookupSymbols: Collection<LookupSymbolKey>,
+        metrics: BuildMetricsReporter = DoNothingBuildMetricsReporter
+    ): List<ClassSnapshotWithHash> {
        val referencedClasses = metrics.measure(BuildTime.FIND_REFERENCED_CLASSES) {
            findReferencedClasses(allClasses, lookupSymbols)
        }
@@ -45,7 +53,7 @@ object ClasspathSnapshotShrinker {
    }

    /**
-     * Finds classes that are referenced. Referencing info is stored in [LookupStorage].
+     * Finds classes that are referenced by the given lookup symbols.
     *
     * Note: It's okay to over-approximate referenced classes.
     */
@@ -107,6 +115,29 @@ object ClasspathSnapshotShrinker {
    }
 }

+/**
+ * Removes duplicate classes and [InaccessibleClassSnapshot]s from the given [ClasspathSnapshot].
+ *
+ * To see why removing duplicate classes is important, consider this example:
+ *   - Current classpath: (Unchanged) jar2!/com/example/A.class containing A.foo, (Added) jar3!/com/example/A.class containing A.bar
+ *   - Previous classpath: (Removed) jar1!/com/example/A.class containing A.bar, (Unchanged) jar2!/com/example/A.class containing A.foo
+ * Without removing duplicates, we might report that there are no changes (both the current classpath and previous classpath have A.foo and
+ * A.bar). However, the correct report should be that A.bar is removed and A.foo is added because the second A class on each classpath does
+ * not have any effect.
+ *
+ * It's also important to remove duplicate classes first before removing [InaccessibleClassSnapshot]s. For example, if
+ * jar1!/com/example/A.class is inaccessible and jar2!/com/example/A.class is accessible, removing inaccessible classes first would mean
+ * that jar2!/com/example/A.class would be kept whereas it shouldn't be since it is a duplicate class (keeping a duplicate class can
+ * lead to incorrect change reports as shown in the previous example).
+ *
+ * That is also why we cannot remove inaccessible classes from each classpath entry in isolation (i.e., during classpath entry
+ * snapshotting), even though it seems more efficient to do so. For correctness, we need to look at the entire classpath first, remove
+ * duplicate classes, and then remove inaccessible classes.
+ */
+internal fun ClasspathSnapshot.removeDuplicateAndInaccessibleClasses(): List<ClassSnapshotWithHash> {
+    return getNonDuplicateClassSnapshots().filter { it.classSnapshot !is InaccessibleClassSnapshot }
+}
+
 /**
 * Returns all [ClassSnapshot]s in this [ClasspathSnapshot].
 *
@@ -121,3 +152,121 @@ internal fun ClasspathSnapshot.getNonDuplicateClassSnapshots(): List<ClassSnapsh
    }
    return classSnapshots.values.toList()
 }
+
+/** Used by [shrinkAndSaveClasspathSnapshot]. */
+private sealed class ShrinkMode {
+    object NoChanges : ShrinkMode()
+
+    class IncrementalNoNewLookups(
+        val shrunkCurrentClasspathAgainstPreviousLookups: List<ClassSnapshotWithHash>,
+    ) : ShrinkMode()
+
+    class Incremental(
+        val currentClasspathSnapshot: List<ClassSnapshotWithHash>,
+        val shrunkCurrentClasspathAgainstPreviousLookups: List<ClassSnapshotWithHash>,
+        val addedLookupSymbols: Set<LookupSymbolKey>
+    ) : ShrinkMode()
+
+    object NonIncremental : ShrinkMode()
+}
+
+internal fun shrinkAndSaveClasspathSnapshot(
+    classpathChanges: ClasspathChanges.ClasspathSnapshotEnabled,
+    lookupStorage: LookupStorage,
+    currentClasspathSnapshot: List<ClassSnapshotWithHash>?, // Not null iff classpathChanges is ToBeComputedByIncrementalCompiler
+    shrunkCurrentClasspathAgainstPreviousLookups: List<ClassSnapshotWithHash>?, // Same as above
+    metrics: BuildMetricsReporter
+) {
+    // In the following, we'll try to shrink the classpath snapshot incrementally when possible.
+    // For incremental shrinking, we currently use only lookupStorage.addedLookupSymbols, not lookupStorage.removedLookupSymbols. It is
+    // because updating the shrunk classpath snapshot for removedLookupSymbols is expensive. Therefore, the shrunk classpath snapshot may be
+    // larger than necessary (and non-deterministic), but it is okay for it to be an over-approximation.
+    val shrinkMode = when (classpathChanges) {
+        is NoChanges -> {
+            val addedLookupSymbols = lookupStorage.addedLookupSymbols
+            if (addedLookupSymbols.isEmpty()) {
+                ShrinkMode.NoChanges
+            } else {
+                val shrunkPreviousClasspathAgainstPreviousLookups =
+                    metrics.measure(BuildTime.LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+                        ListExternalizer(ClassSnapshotWithHashExternalizer)
+                            .loadFromFile(classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile)
+                    }
+                ShrinkMode.Incremental(
+                    currentClasspathSnapshot = metrics.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+                        CachedClasspathSnapshotSerializer
+                            .load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
+                            .removeDuplicateAndInaccessibleClasses()
+                    },
+                    // In the current case, there are no classpath changes, so
+                    // shrunk[*Current*]ClasspathAgainstPreviousLookups == shrunk[*Previous*]ClasspathAgainstPreviousLookups
+                    shrunkCurrentClasspathAgainstPreviousLookups = shrunkPreviousClasspathAgainstPreviousLookups,
+                    addedLookupSymbols = addedLookupSymbols
+                )
+            }
+        }
+        is ToBeComputedByIncrementalCompiler -> {
+            val addedLookupSymbols = lookupStorage.addedLookupSymbols
+            if (addedLookupSymbols.isEmpty()) {
+                ShrinkMode.IncrementalNoNewLookups(shrunkCurrentClasspathAgainstPreviousLookups!!)
+            } else {
+                ShrinkMode.Incremental(
+                    currentClasspathSnapshot!!, shrunkCurrentClasspathAgainstPreviousLookups!!, addedLookupSymbols
+                )
+            }
+        }
+        is NotAvailableDueToMissingClasspathSnapshot, is NotAvailableForNonIncrementalRun -> ShrinkMode.NonIncremental
+    }
+
+    // Shrink current classpath against current lookups
+    val shrunkCurrentClasspath: List<ClassSnapshotWithHash>? = when (shrinkMode) {
+        is ShrinkMode.NoChanges -> null
+        is ShrinkMode.IncrementalNoNewLookups -> {
+            // There are no new lookups, so
+            // shrunkCurrentClasspathAgainst[*Current*]Lookups == shrunkCurrentClasspathAgainst[*Previous*]Lookups
+            shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups
+        }
+        is ShrinkMode.Incremental -> metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+            val shrunkClasses = shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups.map { it.classSnapshot.getClassId() }.toSet()
+            val notYetShrunkClasses = shrinkMode.currentClasspathSnapshot.filter { it.classSnapshot.getClassId() !in shrunkClasses }
+            // Don't provide a BuildMetricsReporter for the following call as the sub-BuildTimes in it have a different parent
+            val shrunkRemainingClassesAgainstNewLookups = shrink(notYetShrunkClasses, shrinkMode.addedLookupSymbols)
+
+            shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups + shrunkRemainingClassesAgainstNewLookups
+        }
+        is ShrinkMode.NonIncremental -> {
+            val classpathSnapshot = metrics.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+                CachedClasspathSnapshotSerializer
+                    .load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
+                    .removeDuplicateAndInaccessibleClasses()
+            }
+            metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+                // Don't provide a BuildMetricsReporter for the following call as the sub-BuildTimes in it have a different parent
+                shrink(classpathSnapshot, lookupStorage)
+            }
+        }
+    }
+
+    if (shrinkMode == ShrinkMode.NoChanges) {
+        // There are no updates to the file so just check that it exists
+        check(classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.exists()) {
+            "File '${classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.path}' does not exist"
+        }
+    } else {
+        metrics.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+            ListExternalizer(ClassSnapshotWithHashExternalizer).saveToFile(
+                classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile,
+                shrunkCurrentClasspath!!
+            )
+        }
+    }
+
+    metrics.addMetric(
+        BuildPerformanceMetric.ORIGINAL_CLASSPATH_SNAPSHOT_SIZE,
+        classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.sumOf { it.length() }
+    )
+    metrics.addMetric(
+        BuildPerformanceMetric.SHRUNK_CLASSPATH_SNAPSHOT_SIZE,
+        classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.length()
+    )
+}
@@ -34,10 +34,10 @@ object ClasspathEntrySnapshotter {
                }

        val snapshots = try {
-            ClassSnapshotter.snapshot(classes, protoBased).map { it.addHash() }
+            ClassSnapshotter.snapshot(classes, protoBased).map { it.withHash }
        } catch (e: Throwable) {
            if ((protoBased ?: protoBasedDefaultValue) && isKnownProblematicClasspathEntry(classpathEntry)) {
-                classes.map { ContentHashJavaClassSnapshot(it.contents.md5()).addHash() }
+                classes.map { ContentHashJavaClassSnapshot(it.contents.md5()).withHash }
            } else throw e
        }

@@ -293,7 +293,7 @@ private fun snapshotClasspath(classpathSourceDir: File, tmpDir: TemporaryFolder,
        classpath.addAll(listOfNotNull(classFiles.firstOrNull()?.classRoot))

        val relativePaths = classFiles.map { it.unixStyleRelativePath }
-        val classSnapshots = classFiles.snapshotAll(protoBased).map { it.addHash() }
+        val classSnapshots = classFiles.snapshotAll(protoBased).map { it.withHash }
        ClasspathEntrySnapshot(
            classSnapshots = relativePaths.zip(classSnapshots).toMap(LinkedHashMap())
        )
@@ -54,5 +54,9 @@
  "className$delegate": {
    "initializer": {},
    "_value": {}
+  },
+  "withHash$delegate": {
+    "initializer": {},
+    "_value": {}
  }
 }
@@ -44,5 +44,9 @@
  "className$delegate": {
    "initializer": {},
    "_value": {}
+  },
+  "withHash$delegate": {
+    "initializer": {},
+    "_value": {}
  }
 }
@@ -1175,5 +1175,9 @@
      "memoizedSerializedSize": -1,
      "memoizedHashCode": 0
    }
+  },
+  "withHash$delegate": {
+    "initializer": {},
+    "_value": {}
  }
 }
@@ -41,5 +41,9 @@
    {
      "internalName": "java/lang/Object"
    }
-  ]
+  ],
+  "withHash$delegate": {
+    "initializer": {},
+    "_value": {}
+  }
 }
@@ -6,7 +6,6 @@
 package org.jetbrains.kotlin.compilerRunner

 import org.gradle.api.GradleException
-import org.gradle.api.file.ConfigurableFileCollection
 import org.gradle.api.file.DirectoryProperty
 import org.gradle.api.file.FileSystemOperations
 import org.gradle.api.logging.Logging
@@ -45,7 +44,7 @@ internal class GradleCompilerRunnerWithWorkers(
        workQueue.submit(GradleKotlinCompilerWorkAction::class.java) { params ->
            params.compilerWorkArguments.set(workArgs)
            if (taskOutputsBackup != null) {
-                params.taskOutputs.from(taskOutputsBackup.outputs)
+                params.taskOutputs.set(taskOutputsBackup.outputs)
                params.buildDir.set(taskOutputsBackup.buildDirectory)
                params.snapshotsDir.set(taskOutputsBackup.snapshotsDir)
                params.metricsReporter.set(buildMetrics)
@@ -66,8 +65,8 @@ internal class GradleCompilerRunnerWithWorkers(
                    fileSystemOperations,
                    parameters.buildDir,
                    parameters.snapshotsDir,
-                    parameters.taskOutputs.files.toList(),
-                    emptyList(),
+                    parameters.taskOutputs.get(),
+                    outputsToExclude = emptyList(),
                    logger
                )
            } else {
@@ -98,7 +97,7 @@ internal class GradleCompilerRunnerWithWorkers(

    internal interface GradleKotlinCompilerWorkParameters : WorkParameters {
        val compilerWorkArguments: Property<GradleKotlinCompilerWorkArguments>
-        val taskOutputs: ConfigurableFileCollection
+        val taskOutputs: ListProperty<File>
        val snapshotsDir: DirectoryProperty
        val buildDir: DirectoryProperty
        val metricsReporter: Property<BuildMetricsReporter>
@@ -12,7 +12,6 @@ import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor
 import org.jetbrains.kotlin.gradle.report.data.TaskExecutionData
 import org.jetbrains.kotlin.gradle.utils.Printer
 import java.io.File
-import java.util.*
 import kotlin.math.max

 internal class PlainTextBuildReportWriter(
@@ -102,7 +101,7 @@ internal class PlainTextBuildReportWriter(

        p.withIndent("Build performance metrics:") {
            for (metric in BuildPerformanceMetric.values()) {
-                p.println("${metric.name}: ${allBuildMetrics[metric]}")
+                allBuildMetrics[metric]?.let { p.println("${metric.name}: $it") }
            }
        }
        p.println()
@@ -57,6 +57,7 @@ import org.jetbrains.kotlin.incremental.ChangedFiles
 import org.jetbrains.kotlin.incremental.ClasspathChanges
 import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotDisabled
 import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.*
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.*
 import org.jetbrains.kotlin.incremental.ClasspathSnapshotFiles
 import org.jetbrains.kotlin.incremental.IncrementalCompilerRunner
 import org.jetbrains.kotlin.library.impl.isKotlinLibrary
@@ -852,7 +853,7 @@ abstract class KotlinCompile @Inject constructor(
            )
            when {
                !inputChanges.isIncremental -> NotAvailableForNonIncrementalRun(classpathSnapshotFiles)
-                inputChanges.getFileChanges(classpathSnapshotProperties.classpathSnapshot).none() -> Empty(classpathSnapshotFiles)
+                inputChanges.getFileChanges(classpathSnapshotProperties.classpathSnapshot).none() -> NoChanges(classpathSnapshotFiles)
                !classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.exists() -> {
                    // When this happens, it means that the classpath snapshot in the previous run was not saved for some reason. It's
                    // likely that there were no source files to compile, so the task action was skipped (see