KT-45777: Move classpath diffing to incremental Kotlin compiler (2/2)

as we need access to the lookup tracker to compute classpath changes
more efficiently and reduce the size of the saved classpath snapshot.

The previous commit only changed the files' paths, this
commit actually updates the files' contents.

Note that classpath snapshotting still happens in Gradle artifact
transforms. (However, the previous commit also moved the code for
classpath snapshotting together with the code for classpath diffing as
they are closely related.)

build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt

@@ -18,6 +18,7 @@ enum class BuildAttributeKind : Serializable {
 enum class BuildAttribute(val kind: BuildAttributeKind, val readableString: String) : Serializable {
     NO_BUILD_HISTORY(BuildAttributeKind.REBUILD_REASON, "Build history file not found"),
     NO_ABI_SNAPSHOT(BuildAttributeKind.REBUILD_REASON, "ABI snapshot not found"),
+    CLASSPATH_SNAPSHOT_NOT_FOUND(BuildAttributeKind.REBUILD_REASON, "Classpath snapshot not found"),
     CACHE_CORRUPTION(BuildAttributeKind.REBUILD_REASON, "Cache corrupted"),
     UNKNOWN_CHANGES_IN_GRADLE_INPUTS(BuildAttributeKind.REBUILD_REASON, "Unknown Gradle changes"),
     JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED(BuildAttributeKind.REBUILD_REASON, "Untracked Java file is removed"),

build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt

@@ -10,9 +10,13 @@ import java.io.Serializable
 
 @Suppress("Reformat")
 enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, val readableString: String) : Serializable {
-    OUTPUT_SIZE(readableString = "Total output size"),
-        LOOKUP_SIZE(OUTPUT_SIZE, "Lookups size"),
-        SNAPSHOT_SIZE(OUTPUT_SIZE, "ABI snapshot size"),
+    CACHE_DIRECTORY_SIZE(readableString = "Total size of the cache directory"),
+        LOOKUP_SIZE(CACHE_DIRECTORY_SIZE, "Lookups size"),
+        SNAPSHOT_SIZE(CACHE_DIRECTORY_SIZE, "ABI snapshot size"),
+
+    // Metrics for the `kotlin.incremental.useClasspathSnapshot` feature
+    ORIGINAL_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the original classpath snapshot (before shrinking)"),
+    SHRUNK_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the shrunk classpath snapshot"),
     ;
 
     companion object {

build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt

@@ -27,6 +27,15 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String)
                     SET_UP_ABI_SNAPSHOTS(JAR_SNAPSHOT, "Set up ABI snapshot"),
                     IC_ANALYZE_JAR_FILES(JAR_SNAPSHOT, "Analyze jar files"),
                 IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION, "Init dirty symbols set"),
+                    COMPUTE_CLASSPATH_CHANGES(IC_CALCULATE_INITIAL_DIRTY_SET, "Compute classpath changes"),
+                        LOAD_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load current classpath snapshot"),
+                        SHRINK_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Shrink current classpath snapshot"),
+                        LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load shrunk previous classpath snapshot"),
+                        COMPUTE_CHANGED_AND_IMPACTED_SET(COMPUTE_CLASSPATH_CHANGES, "Compute changed and impacted set"),
+                            COMPUTE_CLASS_CHANGES(COMPUTE_CHANGED_AND_IMPACTED_SET, "Compute class changes"),
+                                COMPUTE_KOTLIN_CLASS_CHANGES(COMPUTE_CLASS_CHANGES, "Compute Kotlin class changes"),
+                                COMPUTE_JAVA_CLASS_CHANGES(COMPUTE_CLASS_CHANGES, "Compute Java class changes"),
+                            COMPUTE_IMPACTED_SET(COMPUTE_CHANGED_AND_IMPACTED_SET, "Compute impacted set"),
                     IC_ANALYZE_CHANGES_IN_DEPENDENCIES(IC_CALCULATE_INITIAL_DIRTY_SET, "Analyze dependency changes"),
                         IC_FIND_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES, "Find history files"),
                         IC_ANALYZE_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES, "Analyze history files"),
@@ -38,6 +47,14 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String)
                 INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Incremental iteration"),
                 NON_INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Non-incremental iteration"),
                 IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION, "Write history file"),
+                SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION, "Save shrunk current classpath snapshot after compilation"),
+                    LOAD_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load classpath snapshot"),
+                    SHRINK_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink classpath snapshot"),
+                        GET_NON_DUPLICATE_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Get non-duplicate classes"),
+                        GET_LOOKUP_SYMBOLS(SHRINK_CLASSPATH_SNAPSHOT, "Get lookup symbols"),
+                        FIND_REFERENCED_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Find referenced classes"),
+                        FIND_TRANSITIVELY_REFERENCED_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Find transitively referenced classes"),
+                    SAVE_SHRUNK_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk classpath snapshot"),
     COMPILER_PERFORMANCE(readableString = "Compiler time"),
         COMPILER_INITIALIZATION(COMPILER_PERFORMANCE, "Compiler initialization time"),
         CODE_ANALYSIS(COMPILER_PERFORMANCE, "Compiler code analyse"),

build-common/src/org/jetbrains/kotlin/incremental/ClasspathChanges.kt

@@ -5,70 +5,43 @@
 
 package org.jetbrains.kotlin.incremental
 
-import com.intellij.util.io.DataExternalizer
-import org.jetbrains.kotlin.incremental.storage.FqNameExternalizer
-import org.jetbrains.kotlin.incremental.storage.SetExternalizer
-import org.jetbrains.kotlin.incremental.storage.LookupSymbolExternalizer
-import org.jetbrains.kotlin.name.FqName
-import java.io.*
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled
+import java.io.File
+import java.io.Serializable
 
 /**
- * Changes to the classpath of the `KotlinCompile` task, used to compute the source files that need to be recompiled during an incremental
- * run.
+ * Changes to the classpath of the `KotlinCompile` task, or information to compute them later by the Kotlin incremental compiler (see
+ * [ClasspathSnapshotEnabled.ToBeComputedByIncrementalCompiler].
  */
 sealed class ClasspathChanges : Serializable {
 
-    class Available() : ClasspathChanges() {
+    sealed class ClasspathSnapshotEnabled : ClasspathChanges() {
 
-        companion object {
-            private const val serialVersionUID = 0L
-        }
+        abstract val classpathSnapshotFiles: ClasspathSnapshotFiles
 
-        lateinit var lookupSymbols: Set<LookupSymbol> // Preferably ordered but not required
-            private set
+        class Empty(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled()
 
-        lateinit var fqNames: Set<FqName> // Preferably ordered but not required
-            private set
+        class ToBeComputedByIncrementalCompiler(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled()
 
-        constructor(lookupSymbols: Set<LookupSymbol>, fqNames: Set<FqName>) : this() {
-            this.lookupSymbols = lookupSymbols
-            this.fqNames = fqNames
-        }
+        class NotAvailableDueToMissingClasspathSnapshot(override val classpathSnapshotFiles: ClasspathSnapshotFiles) :
+            ClasspathSnapshotEnabled()
 
-        private fun writeObject(output: ObjectOutputStream) {
-            // Can't close DataOutputStream below as it will also close the underlying ObjectOutputStream, which is still in use.
-            ClasspathChangesAvailableExternalizer.save(DataOutputStream(output), this)
-        }
-
-        private fun readObject(input: ObjectInputStream) {
-            // Can't close DataInputStream below as it will also close the underlying ObjectInputStream, which is still in use.
-            ClasspathChangesAvailableExternalizer.read(DataInputStream(input)).also {
-                lookupSymbols = it.lookupSymbols
-                fqNames = it.fqNames
-            }
-        }
+        class NotAvailableForNonIncrementalRun(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled()
     }
 
-    sealed class NotAvailable : ClasspathChanges() {
-        object UnableToCompute : NotAvailable()
-        object ForNonIncrementalRun : NotAvailable()
-        object ClasspathSnapshotIsDisabled : NotAvailable()
-        object ReservedForTestsOnly : NotAvailable()
-        object ForJSCompiler : NotAvailable()
-    }
+    object ClasspathSnapshotDisabled : ClasspathChanges()
+
+    object NotAvailableForJSCompiler : ClasspathChanges()
 }
 
-private object ClasspathChangesAvailableExternalizer : DataExternalizer<ClasspathChanges.Available> {
+class ClasspathSnapshotFiles(
+    val currentClasspathEntrySnapshotFiles: List<File>,
+    classpathSnapshotDir: File
+) : Serializable {
 
-    override fun save(output: DataOutput, classpathChanges: ClasspathChanges.Available) {
-        SetExternalizer(LookupSymbolExternalizer).save(output, classpathChanges.lookupSymbols)
-        SetExternalizer(FqNameExternalizer).save(output, classpathChanges.fqNames)
-    }
+    val shrunkPreviousClasspathSnapshotFile: File = File(classpathSnapshotDir, "shrunk-classpath-snapshot.bin")
 
-    override fun read(input: DataInput): ClasspathChanges.Available {
-        return ClasspathChanges.Available(
-            lookupSymbols = SetExternalizer(LookupSymbolExternalizer).read(input),
-            fqNames = SetExternalizer(FqNameExternalizer).read(input)
-        )
+    companion object {
+        private const val serialVersionUID = 0L
     }
 }

build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt

@@ -32,7 +32,8 @@ import java.util.*
 
 open class LookupStorage(
     targetDataDir: File,
-    pathConverter: FileToPathConverter
+    pathConverter: FileToPathConverter,
+    storeFullFqNames: Boolean = false
 ) : BasicMapsOwner(targetDataDir) {
     val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder")
 
@@ -44,7 +45,7 @@ open class LookupStorage(
     private val countersFile = "counters".storageFile
     private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, pathConverter))
     private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, pathConverter))
-    val lookupMap = registerMap(LookupMap("lookups".storageFile))
+    val lookupMap = registerMap(LookupMap("lookups".storageFile, storeFullFqNames))
 
     @Volatile
     private var size: Int = 0

build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt

@@ -18,7 +18,9 @@ package org.jetbrains.kotlin.incremental.storage
 
 import java.io.File
 
-class LookupMap(storage: File) : BasicMap<LookupSymbolKey, Collection<Int>>(storage, LookupSymbolKeyDescriptor, IntCollectionExternalizer) {
+class LookupMap(storage: File, storeFullFqNames: Boolean) :
+    BasicMap<LookupSymbolKey, Collection<Int>>(storage, LookupSymbolKeyDescriptor(storeFullFqNames), IntCollectionExternalizer) {
+
     override fun dumpKey(key: LookupSymbolKey): String = key.toString()
 
     override fun dumpValue(value: Collection<Int>): String = value.toString()

build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt

@@ -22,22 +22,21 @@ import com.intellij.util.io.DataExternalizer
 import com.intellij.util.io.EnumeratorStringDescriptor
 import com.intellij.util.io.IOUtil
 import com.intellij.util.io.KeyDescriptor
-import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
-import org.jetbrains.kotlin.cli.common.toBooleanLenient
-import org.jetbrains.kotlin.incremental.LookupSymbol
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
-import java.io.DataInput
-import java.io.DataInputStream
-import java.io.DataOutput
+import java.io.*
 
 /**
  * Storage versioning:
  * 0 - only name and value hashes are saved
  * 1 - name and scope are saved
  */
-object LookupSymbolKeyDescriptor : KeyDescriptor<LookupSymbolKey> {
+class LookupSymbolKeyDescriptor(
+    /** If `true`, original values are saved; if `false`, only hashes are saved. */
+    private val storeFullFqNames: Boolean = false
+) : KeyDescriptor<LookupSymbolKey> {
+
     override fun read(input: DataInput): LookupSymbolKey {
         val version = input.readByte()
         return when (version.toInt()) {
@@ -51,14 +50,12 @@ object LookupSymbolKeyDescriptor : KeyDescriptor<LookupSymbolKey> {
                 val second = input.readInt()
                 LookupSymbolKey(first, second, "", "")
             }
-            else -> throw RuntimeException("Unknown version of LookupSymbolKeyDescriptor=${version}")
+            else -> throw IllegalArgumentException("Unknown version of LookupSymbolKeyDescriptor=${version}")
         }
     }
 
-    private val storeFullFqName = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient() ?: false
-
     override fun save(output: DataOutput, value: LookupSymbolKey) {
-        if (storeFullFqName) {
+        if (storeFullFqNames) {
             output.writeByte(0)
             output.writeUTF(value.name)
             output.writeUTF(value.scope)
@@ -74,18 +71,6 @@ object LookupSymbolKeyDescriptor : KeyDescriptor<LookupSymbolKey> {
     override fun isEqual(val1: LookupSymbolKey, val2: LookupSymbolKey): Boolean = val1 == val2
 }
 
-object LookupSymbolExternalizer : DataExternalizer<LookupSymbol> {
-
-    override fun save(output: DataOutput, lookupSymbol: LookupSymbol) {
-        output.writeString(lookupSymbol.name)
-        output.writeString(lookupSymbol.scope)
-    }
-
-    override fun read(input: DataInput): LookupSymbol {
-        return LookupSymbol(name = input.readString(), scope = input.readString())
-    }
-}
-
 object FqNameExternalizer : DataExternalizer<FqName> {
 
     override fun save(output: DataOutput, fqName: FqName) {
@@ -226,6 +211,32 @@ object ConstantExternalizer : DataExternalizer<Any> {
     }
 }
 
+fun <T> DataExternalizer<T>.saveToFile(file: File, value: T) {
+    return DataOutputStream(FileOutputStream(file).buffered()).use {
+        save(it, value)
+    }
+}
+
+fun <T> DataExternalizer<T>.loadFromFile(file: File): T {
+    return DataInputStream(FileInputStream(file).buffered()).use {
+        read(it)
+    }
+}
+
+fun <T> DataExternalizer<T>.toByteArray(value: T): ByteArray {
+    val byteArrayOutputStream = ByteArrayOutputStream()
+    DataOutputStream(byteArrayOutputStream.buffered()).use {
+        save(it, value)
+    }
+    return byteArrayOutputStream.toByteArray()
+}
+
+fun <T> DataExternalizer<T>.fromByteArray(byteArray: ByteArray): T {
+    return DataInputStream(ByteArrayInputStream(byteArray).buffered()).use {
+        read(it)
+    }
+}
+
 object IntExternalizer : DataExternalizer<Int> {
     override fun save(output: DataOutput, value: Int) = output.writeInt(value)
     override fun read(input: DataInput): Int = input.readInt()

build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt

@@ -7,6 +7,8 @@ package org.jetbrains.kotlin.incremental.storage
 
 import com.intellij.util.containers.MultiMap
 import org.jetbrains.kotlin.TestWithWorkingDir
+import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
+import org.jetbrains.kotlin.cli.common.toBooleanLenient
 import org.jetbrains.kotlin.incremental.LookupStorage
 import org.jetbrains.kotlin.incremental.LookupSymbol
 import org.jetbrains.kotlin.incremental.testingUtils.assertEqualDirectories
@@ -48,7 +50,11 @@ class RelocatableCachesTest : TestWithWorkingDir() {
     private fun fillLookupStorage(projectRoot: File, reverseFiles: Boolean, reverseLookups: Boolean) {
         val storageRoot = projectRoot.storageRoot
         val fileToPathConverter = RelativeFileToPathConverter(projectRoot)
-        val lookupStorage = LookupStorage(storageRoot, fileToPathConverter)
+        val lookupStorage = LookupStorage(
+            storageRoot,
+            fileToPathConverter,
+            storeFullFqNames = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient() ?: false
+        )
         val files = LinkedHashSet<String>()
         val symbols = LinkedHashSet<LookupSymbol>()
         val lookups = MultiMap.createOrderedSet<LookupSymbol, String>()

compiler/incremental-compilation-impl/build.gradle.kts

@@ -15,6 +15,7 @@ dependencies {
     api(project(":compiler:cli-js"))
     api(project(":kotlin-build-common"))
     api(project(":daemon-common"))
+    implementation("com.google.code.gson:gson:${rootProject.extra["versions.jar.gson"]}")
     compileOnly(intellijCoreDep()) { includeJars("intellij-core") }
 
     testApi(commonDep("junit:junit"))

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt

@@ -26,8 +26,8 @@ import java.io.File
 abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache<*>>(
     cachesRootDir: File,
     rootProjectDir: File?,
-    protected val reporter: ICReporter
-) {
+    protected val reporter: ICReporter,
+    storeFullFqNamesInLookupCache: Boolean = false) {
     val pathConverter = IncrementalFileToPathConverter(rootProjectDir)
     private val caches = arrayListOf<BasicMapsOwner>()
 
@@ -43,7 +43,7 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
     private val lookupCacheDir = File(cachesRootDir, "lookups").apply { mkdirs() }
 
     val inputsCache: InputsCache = InputsCache(inputSnapshotsCacheDir, reporter, pathConverter).apply { registerCache() }
-    val lookupCache: LookupStorage = LookupStorage(lookupCacheDir, pathConverter).apply { registerCache() }
+    val lookupCache: LookupStorage = LookupStorage(lookupCacheDir, pathConverter, storeFullFqNamesInLookupCache).apply { registerCache() }
     abstract val platformCache: PlatformCache
 
     @Synchronized
@@ -79,8 +79,9 @@ class IncrementalJvmCachesManager(
     cacheDirectory: File,
     rootProjectDir: File?,
     outputDir: File,
-    reporter: ICReporter
-) : IncrementalCachesManager<IncrementalJvmCache>(cacheDirectory, rootProjectDir, reporter) {
+    reporter: ICReporter,
+    storeFullFqNamesInLookupCache: Boolean = false
+    ) : IncrementalCachesManager<IncrementalJvmCache>(cacheDirectory, rootProjectDir, reporter, storeFullFqNamesInLookupCache) {
     private val jvmCacheDir = File(cacheDirectory, "jvm").apply { mkdirs() }
     override val platformCache = IncrementalJvmCache(jvmCacheDir, outputDir, pathConverter).apply { registerCache() }
 }
@@ -89,8 +90,9 @@ class IncrementalJsCachesManager(
     cachesRootDir: File,
     rootProjectDir: File?,
     reporter: ICReporter,
-    serializerProtocol: SerializerExtensionProtocol
-) : IncrementalCachesManager<IncrementalJsCache>(cachesRootDir, rootProjectDir, reporter) {
+    serializerProtocol: SerializerExtensionProtocol,
+    storeFullFqNamesInLookupCache: Boolean
+) : IncrementalCachesManager<IncrementalJsCache>(cachesRootDir, rootProjectDir, reporter, storeFullFqNamesInLookupCache) {
     private val jsCacheFile = File(cachesRootDir, "js").apply { mkdirs() }
     override val platformCache = IncrementalJsCache(jsCacheFile, pathConverter, serializerProtocol).apply { registerCache() }
 }

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt

@@ -19,9 +19,9 @@ package org.jetbrains.kotlin.incremental
 import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS
 import org.jetbrains.kotlin.build.GeneratedFile
 import org.jetbrains.kotlin.build.report.BuildReporter
-import org.jetbrains.kotlin.build.report.metrics.BuildTime
 import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
 import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric
+import org.jetbrains.kotlin.build.report.metrics.BuildTime
 import org.jetbrains.kotlin.build.report.metrics.measure
 import org.jetbrains.kotlin.cli.common.*
 import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
@@ -39,8 +39,6 @@ import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.progress.CompilationCanceledStatus
 import java.io.File
 import java.io.IOException
-import java.util.*
-import kotlin.collections.HashMap
 
 abstract class IncrementalCompilerRunner<
         Args : CommonCompilerArguments,
@@ -108,7 +106,7 @@ abstract class IncrementalCompilerRunner<
             caches.close(false)
             // todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate
             reporter.measure(BuildTime.CLEAR_OUTPUT_ON_REBUILD) {
-                clearLocalStateOnRebuild(args)
+                clearOutputsOnRebuild(args)
             }
             caches = createCacheManager(args, projectDir)
             if (providedChangedFiles == null) {
@@ -134,7 +132,6 @@ abstract class IncrementalCompilerRunner<
                 else -> providedChangedFiles
             }
 
-
             val compilationMode = sourcesToCompile(caches, changedFiles, args, messageCollector, classpathAbiSnapshot)
 
             val exitCode = when (compilationMode) {
@@ -162,7 +159,8 @@ abstract class IncrementalCompilerRunner<
                             allSourceFiles,
                             compilationMode,
                             messageCollector,
-                            withSnapshot)
+                            withSnapshot
+                        )
                     }
                 }
                 is CompilationMode.Rebuild -> {
@@ -170,6 +168,8 @@ abstract class IncrementalCompilerRunner<
                 }
             }
 
+            performWorkAfterCompilation(caches)
+
             if (!caches.close(flush = true)) throw RuntimeException("Could not flush caches")
             // Here we should analyze exit code of compiler. E.g. compiler failure should lead to caches rebuild,
             // but now JsKlib compiler reports invalid exit code.
@@ -182,7 +182,7 @@ abstract class IncrementalCompilerRunner<
                 )
                 if (cacheDirectory.exists() && cacheDirectory.isDirectory()) {
                     cacheDirectory.walkTopDown().filter { it.isFile }.map { it.length() }.sum().let {
-                        reporter.addMetric(BuildPerformanceMetric.OUTPUT_SIZE, it)
+                        reporter.addMetric(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE, it)
                     }
                 }
             }
@@ -193,37 +193,42 @@ abstract class IncrementalCompilerRunner<
             rebuild(BuildAttribute.CACHE_CORRUPTION)
         } finally {
             if (cachesMayBeCorrupted) {
-                clearLocalStateOnRebuild(args)
+                clearOutputsOnRebuild(args)
             }
         }
     }
 
-    private fun clearLocalStateOnRebuild(args: Args) {
+    /**
+     * Deletes output files and contents of output directories on rebuild (including `@LocalState` files/directories).
+     *
+     * If the output directories do not yet exist, they will be created.
+     */
+    private fun clearOutputsOnRebuild(args: Args) {
         val destinationDir = destinationDir(args)
+        val (outputDirsThatExist, regularOrNonExistentOutputFiles) = outputFiles.partition { it.isDirectory }
+        val regularOutputFiles = regularOrNonExistentOutputFiles.filter { it.exists() }
 
-        reporter.reportVerbose { "Clearing output on rebuild" }
-        for (file in sequenceOf(destinationDir, workingDir) + outputFiles.asSequence()) {
-            val deleted: Boolean? = when {
-                file.isDirectory -> {
-                    reporter.reportVerbose { "  Deleting directory $file" }
-                    file.deleteRecursively()
-                }
-                file.isFile -> {
-                    reporter.reportVerbose { "  Deleting $file" }
-                    file.delete()
-                }
-                else -> null
+        // outputDirsThatExist may or may not contain destinationDir and workingDir.
+        // Collect all of them so that we don't miss any output directories.
+        // Use Set to avoid duplication.
+        val allOutputDirs = setOf(destinationDir, workingDir) + outputDirsThatExist
+
+        reporter.reportVerbose { "Clearing outputs on rebuild" }
+        allOutputDirs.forEach { dir ->
+            reporter.reportVerbose { "  Deleting contents of directory '${dir.path}'" }
+            dir.listFiles()?.forEach {
+                it.deleteRecursively()
+                if (it.exists()) throw IOException("Could not delete '${it.path}'")
             }
 
-            if (deleted == false) {
-                reporter.reportVerbose { "  Could not delete $file" }
-            }
+            dir.mkdirs()
+            if (!dir.exists()) throw IOException("Could not create directory '${dir.path}'")
+        }
+        regularOutputFiles.forEach { file ->
+            reporter.reportVerbose { "  Deleting file '${file.path}'" }
+            file.delete()
+            if (file.exists()) throw IOException("Could not delete file '${file.path}'")
         }
-
-        if (destinationDir.exists()) throw IOException("Could not delete directory $destinationDir.")
-        if (workingDir.exists()) throw IOException("Could not delete internal caches in folder $workingDir")
-        destinationDir.mkdirs()
-        workingDir.mkdirs()
     }
 
     private fun sourcesToCompile(
@@ -393,9 +398,8 @@ abstract class IncrementalCompilerRunner<
                 caches.platformCache.updateComplementaryFiles(dirtySources, expectActualTracker)
                 caches.inputsCache.registerOutputForSourceFiles(generatedFiles)
                 caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources)
-            updateCaches(services, caches, generatedFiles, changesCollector)
-
-}
+                updateCaches(services, caches, generatedFiles, changesCollector)
+            }
             if (compilationMode is CompilationMode.Rebuild) {
                 if (withSnapshot) {
                     abiSnapshot.protos.putAll(changesCollector.protoDataChanges())
@@ -498,6 +502,8 @@ abstract class IncrementalCompilerRunner<
         BuildDiffsStorage.writeToFile(buildHistoryFile, BuildDiffsStorage(prevDiffs + newDiff), reporter)
     }
 
+    protected open fun performWorkAfterCompilation(caches: CacheManager) {}
+
     companion object {
         const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt"
         const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin"

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt

@@ -20,9 +20,8 @@ import org.jetbrains.kotlin.build.GeneratedFile
 import org.jetbrains.kotlin.build.report.BuildReporter
 import org.jetbrains.kotlin.build.report.ICReporter
 import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
-import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric
 import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
-import org.jetbrains.kotlin.cli.common.*
+import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
 import org.jetbrains.kotlin.cli.common.arguments.isIrBackendEnabled
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
@@ -92,7 +91,13 @@ class IncrementalJsCompilerRunner(
 
     override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?): IncrementalJsCachesManager {
         val serializerProtocol = if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol
-        return IncrementalJsCachesManager(cacheDirectory, projectDir, reporter, serializerProtocol)
+        return IncrementalJsCachesManager(
+            cacheDirectory,
+            projectDir,
+            reporter,
+            serializerProtocol,
+            storeFullFqNamesInLookupCache = withSnapshot
+            )
     }
 
     override fun destinationDir(args: K2JSCompilerArguments): File {

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt

@@ -41,17 +41,18 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.IncrementalCompilation
 import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotDisabled
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.*
+import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailableForJSCompiler
+import org.jetbrains.kotlin.incremental.classpathDiff.*
 import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory
 import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory
+import org.jetbrains.kotlin.incremental.storage.ListExternalizer
+import org.jetbrains.kotlin.incremental.storage.saveToFile
 import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector
 import org.jetbrains.kotlin.incremental.util.Either
-import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.UnableToCompute
-import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ForJSCompiler
-import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ReservedForTestsOnly
-import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ForNonIncrementalRun
-import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ClasspathSnapshotIsDisabled
 import org.jetbrains.kotlin.load.java.JavaClassesTracker
 import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents
@@ -88,7 +89,7 @@ fun makeIncrementally(
             buildHistoryFile = buildHistoryFile,
             modulesApiHistory = EmptyModulesApiHistory,
             kotlinSourceFilesExtensions = kotlinExtensions,
-            classpathChanges = ReservedForTestsOnly
+            classpathChanges = ClasspathSnapshotDisabled
         )
         //TODO set properly
         compiler.compile(sourceFiles, args, messageCollector, providedChangedFiles = null)
@@ -123,7 +124,7 @@ class IncrementalJvmCompilerRunner(
     outputFiles: Collection<File>,
     private val modulesApiHistory: ModulesApiHistory,
     override val kotlinSourceFilesExtensions: List<String> = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS,
-    private val classpathChanges: ClasspathChanges,
+    private val classpathChanges: ClasspathChanges
 ) : IncrementalCompilerRunner<K2JVMCompilerArguments, IncrementalJvmCachesManager>(
     workingDir,
     "caches-jvm",
@@ -135,7 +136,13 @@ class IncrementalJvmCompilerRunner(
         IncrementalCompilation.isEnabledForJvm()
 
     override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager =
-        IncrementalJvmCachesManager(cacheDirectory, projectDir, File(args.destination), reporter)
+        IncrementalJvmCachesManager(
+            cacheDirectory,
+            projectDir,
+            File(args.destination),
+            reporter,
+            storeFullFqNamesInLookupCache = withSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled
+            )
 
     override fun destinationDir(args: K2JVMCompilerArguments): File =
         args.destinationAsFile
@@ -203,6 +210,10 @@ class IncrementalJvmCompilerRunner(
         return abiSnapshots
     }
 
+    // Used by `calculateSourcesToCompileImpl` and `performWorkAfterCompilation` methods below.
+    // Thread safety: There is no concurrent access to this variable.
+    private var currentClasspathSnapshot: ClasspathSnapshot? = null
+
     private fun calculateSourcesToCompileImpl(
         caches: IncrementalJvmCachesManager,
         changedFiles: ChangedFiles.Known,
@@ -212,26 +223,34 @@ class IncrementalJvmCompilerRunner(
         val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions)
         initDirtyFiles(dirtyFiles, changedFiles)
 
-        val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY)
-        reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" }
-
         val classpathChanges = when (classpathChanges) {
             // Note: classpathChanges is deserialized, so they are no longer singleton objects and need to be compared using `is` (not `==`)
-            is ClasspathChanges.Available -> ChangesEither.Known(classpathChanges.lookupSymbols, classpathChanges.fqNames)
-            is ClasspathChanges.NotAvailable -> when (classpathChanges) {
-                is UnableToCompute, is ClasspathSnapshotIsDisabled, is ReservedForTestsOnly -> {
-                    reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) {
-                        val scopes = caches.lookupCache.lookupMap.keys.map { it.scope.ifBlank { it.name } }.distinct()
-                        getClasspathChanges(
-                            args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter, abiSnapshots, withSnapshot,
-                            caches.platformCache, scopes
-                        )
-                    }
-                }
-                is ForNonIncrementalRun, is ForJSCompiler -> {
-                    error("Unexpected type for this code path: ${classpathChanges.javaClass.name}.")
+            is Empty -> ChangesEither.Known(emptySet(), emptySet())
+            is ToBeComputedByIncrementalCompiler -> reporter.measure(BuildTime.COMPUTE_CLASSPATH_CHANGES) {
+                check(currentClasspathSnapshot == null)
+                currentClasspathSnapshot = reporter.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT) {
+                    CachedClasspathSnapshotSerializer.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
                 }
+                ClasspathChangesComputer.computeChangedAndImpactedSet(
+                    currentClasspathSnapshot!!,
+                    caches.lookupCache,
+                    classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile,
+                    reporter
+                ).getChanges()
             }
+            is NotAvailableDueToMissingClasspathSnapshot -> ChangesEither.Unknown(BuildAttribute.CLASSPATH_SNAPSHOT_NOT_FOUND)
+            is NotAvailableForNonIncrementalRun -> ChangesEither.Unknown(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS)
+            is ClasspathSnapshotDisabled -> reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) {
+                val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild(BuildAttribute.IC_IS_NOT_ENABLED)
+                reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" }
+                val scopes = caches.lookupCache.lookupMap.keys.map { it.scope.ifBlank { it.name } }.distinct()
+
+                getClasspathChanges(
+                    args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter, abiSnapshots, withSnapshot,
+                    caches.platformCache, scopes
+                )
+            }
+            is NotAvailableForJSCompiler -> error("Unexpected type for this code path: ${classpathChanges.javaClass.name}.")
         }
 
         @Suppress("UNUSED_VARIABLE") // for sealed when
@@ -434,6 +453,38 @@ class IncrementalJvmCompilerRunner(
         reportPerformanceData(compiler.defaultPerformanceManager)
         return exitCode
     }
+
+    override fun performWorkAfterCompilation(caches: IncrementalJvmCachesManager) {
+        if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) {
+            reporter.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+                shrinkAndSaveClasspathSnapshot(classpathChanges, caches.lookupCache)
+            }
+        }
+    }
+
+    private fun shrinkAndSaveClasspathSnapshot(classpathChanges: ClasspathChanges.ClasspathSnapshotEnabled, lookupStorage: LookupStorage) {
+        val classpathSnapshot = currentClasspathSnapshot ?: reporter.measure(BuildTime.LOAD_CLASSPATH_SNAPSHOT) {
+            CachedClasspathSnapshotSerializer.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
+        }
+        val shrunkClasspathSnapshot = reporter.measure(BuildTime.SHRINK_CLASSPATH_SNAPSHOT) {
+            ClasspathSnapshotShrinker.shrink(classpathSnapshot, lookupStorage, reporter)
+        }
+        reporter.measure(BuildTime.SAVE_SHRUNK_CLASSPATH_SNAPSHOT) {
+            ListExternalizer(ClassSnapshotWithHashExternalizer).saveToFile(
+                classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile,
+                shrunkClasspathSnapshot
+            )
+        }
+
+        reporter.addMetric(
+            BuildPerformanceMetric.ORIGINAL_CLASSPATH_SNAPSHOT_SIZE,
+            classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.sumOf { it.length() }
+        )
+        reporter.addMetric(
+            BuildPerformanceMetric.SHRUNK_CLASSPATH_SNAPSHOT_SIZE,
+            classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.length()
+        )
+    }
 }
 
 var K2JVMCompilerArguments.destinationAsFile: File

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ChangeSet.kt

@@ -5,12 +5,12 @@
 
 package org.jetbrains.kotlin.incremental.classpathDiff
 
-import org.jetbrains.kotlin.incremental.ClasspathChanges
+import org.jetbrains.kotlin.incremental.ChangesEither
 import org.jetbrains.kotlin.incremental.LookupSymbol
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 
-/** Intermediate data to compute [ClasspathChanges] (see [toClasspathChanges]). */
+/** Set of classes, class members, and top-level members that are changed (or impacted by a change). */
 class ChangeSet(
 
     /** Set of changed classes, preferably ordered by not required. */
@@ -73,7 +73,7 @@ class ChangeSet(
         changedTopLevelMembers + other.changedTopLevelMembers
     )
 
-    fun toClasspathChanges(): ClasspathChanges.Available {
+    internal fun getChanges(): ChangesEither.Known {
         val lookupSymbols = mutableSetOf<LookupSymbol>()
         val fqNames = mutableSetOf<FqName>()
 
@@ -98,6 +98,6 @@ class ChangeSet(
             fqNames.add(changedPackage)
         }
 
-        return ClasspathChanges.Available(lookupSymbols, fqNames)
+        return ChangesEither.Known(lookupSymbols, fqNames)
     }
 }

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt

@@ -6,9 +6,14 @@
 package org.jetbrains.kotlin.incremental.classpathDiff
 
 import com.intellij.openapi.util.io.FileUtil
-import org.jetbrains.kotlin.incremental.classpathDiff.ImpactAnalysis.computeImpactedSet
+import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
+import org.jetbrains.kotlin.build.report.metrics.BuildTime
+import org.jetbrains.kotlin.build.report.metrics.measure
 import org.jetbrains.kotlin.incremental.*
+import org.jetbrains.kotlin.incremental.classpathDiff.ImpactAnalysis.computeImpactedSet
 import org.jetbrains.kotlin.incremental.storage.FileToCanonicalPathConverter
+import org.jetbrains.kotlin.incremental.storage.ListExternalizer
+import org.jetbrains.kotlin.incremental.storage.loadFromFile
 import org.jetbrains.kotlin.metadata.deserialization.TypeTable
 import org.jetbrains.kotlin.metadata.deserialization.supertypes
 import org.jetbrains.kotlin.name.ClassId
@@ -18,156 +23,101 @@ import org.jetbrains.kotlin.serialization.deserialization.getClassId
 import java.io.File
 import java.util.*
 
-/** Computes [ClasspathChanges] between two [ClasspathSnapshot]s .*/
+/** Computes changes between two [ClasspathSnapshot]s .*/
 object ClasspathChangesComputer {
 
-    fun compute(
-        currentClasspathEntrySnapshotFiles: List<File>,
-        previousClasspathEntrySnapshotFiles: List<File>,
-        unchangedCurrentClasspathEntrySnapshotFiles: List<File>
-    ): ClasspathChanges {
-        // To improve performance, we will compute changes for the changed snapshot files only, ignoring unchanged ones. (Duplicate classes
-        // will make this a bit tricky, but it will be dealt with below.)
-        // First, align unchanged snapshot files in the current classpath with unchanged snapshot files in the previous classpath. Gradle
-        // has this information, but doesn't expose it, so we have to reconstruct it here.
-        val unchangedCurrentToPreviousAlignment: Map<File, File> =
-            alignUnchangedSnapshotFiles(unchangedCurrentClasspathEntrySnapshotFiles, previousClasspathEntrySnapshotFiles)
-
-        // Use sets to make presence checks faster
-        val unchangedCurrentFiles: Set<File> = unchangedCurrentToPreviousAlignment.keys
-        val unchangedPreviousFiles: Set<File> = unchangedCurrentToPreviousAlignment.values.toSet()
-
-        // We will split the current files into 2 groups:
-        //    1a) Unchanged current files
-        //    1b) Added files
-        // We will split the previous files into 2 groups:
-        //    2a) Unchanged previous files
-        //    2b) Removed files
-        // If the classpath doesn't contain duplicate classes, comparing (1b) with (2b) would be enough.
-        // However, if the classpath contains duplicate classes, comparing (1b) with (2b) would not be enough.
-        // Therefore, to deal with duplicate classes while still being able to compare (1b) with (2b), we will find snapshot files in groups
-        // (1a) and (2a) that have duplicate classes with groups (1b) or (2b) and add them to groups (1b) and (2b). Duplicate classes in
-        // groups (1b) and (2b) will then be handled in a separate step (see ClasspathChangesComputer.getNonDuplicateClassSnapshots).
-        val addedFiles: List<File> = currentClasspathEntrySnapshotFiles.filter { it !in unchangedCurrentFiles }
-        val removedFiles: List<File> = previousClasspathEntrySnapshotFiles.filter { it !in unchangedPreviousFiles }
-
-        val adjustedAddedFiles = addedFiles.toMutableSet()
-        val adjustedRemovedFiles = removedFiles.toMutableSet()
-        unchangedCurrentToPreviousAlignment.forEach { (unchangedCurrentFile, unchangedPreviousFile) ->
-            if (unchangedCurrentFile.containsDuplicatesWith(addedFiles) || unchangedPreviousFile.containsDuplicatesWith(removedFiles)) {
-                adjustedAddedFiles.add(unchangedCurrentFile)
-                adjustedRemovedFiles.add(unchangedPreviousFile)
-            }
+    /**
+     * Computes changes between the current and previous [ClasspathSnapshot]s, plus unchanged elements that are impacted by the changes.
+     *
+     * NOTE: The original classpath may contain duplicate classes, but the shrunk classpath must not contain duplicate classes.
+     */
+    fun computeChangedAndImpactedSet(
+        currentClasspathSnapshot: ClasspathSnapshot,
+        lookupStorageInPreviousRun: LookupStorage,
+        shrunkPreviousClasspathSnapshotFile: File,
+        metrics: BuildMetricsReporter
+    ): ChangeSet {
+        val shrunkCurrentClasspathSnapshot = metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT) {
+            ClasspathSnapshotShrinker.shrink(currentClasspathSnapshot, lookupStorageInPreviousRun, metrics)
+        }
+        val shrunkPreviousClasspathSnapshot = metrics.measure(BuildTime.LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT) {
+            ListExternalizer(ClassSnapshotWithHashExternalizer).loadFromFile(shrunkPreviousClasspathSnapshotFile)
+        }
+        return metrics.measure(BuildTime.COMPUTE_CHANGED_AND_IMPACTED_SET) {
+            computeChangedAndImpactedSet(shrunkCurrentClasspathSnapshot, shrunkPreviousClasspathSnapshot, metrics)
         }
-
-        // Keep the original order of added/removed files as it is important for the handling of duplicate classes.
-        val finalAddedFiles: List<File> = currentClasspathEntrySnapshotFiles.filter { it in adjustedAddedFiles }
-        val finalRemovedFiles: List<File> = previousClasspathEntrySnapshotFiles.filter { it in adjustedRemovedFiles }
-
-        val changedCurrentSnapshot = ClasspathSnapshotSerializer.load(finalAddedFiles)
-        val changedPreviousSnapshot = ClasspathSnapshotSerializer.load(finalRemovedFiles)
-
-        return compute(changedCurrentSnapshot, changedPreviousSnapshot)
     }
 
     /**
-     * Maps the unchanged snapshot files of the current build to the unchanged snapshot files of the previous build (selected from all the
-     * snapshot files of the previous build).
+     * Computes changes between the current and previous lists of classes, plus unchanged elements that are impacted by the changes.
      *
-     * Note that the unchanged files of the current build were detected by Gradle, so a mapping must exist for each of them, we only have to
-     * find it.
-     *
-     * IMPORTANT: The alignment algorithm must use the same input normalization that is used for snapshot files in the Gradle task.
-     * Currently, snapshot files are annotated with `@Classpath` and are regular files (not jars), so (only) their contents and order
-     * matter.
+     * NOTE: Each list of classes must not contain duplicates.
      */
-    private fun alignUnchangedSnapshotFiles(unchangedCurrentSnapshotFiles: List<File>, previousSnapshotFiles: List<File>): Map<File, File> {
-        val sizeToPreviousFiles: Map<Long, List<IndexedValue<File>>> = previousSnapshotFiles.withIndex().groupBy { it.value.length() }
+    fun computeChangedAndImpactedSet(
+        currentClassSnapshots: List<ClassSnapshotWithHash>,
+        previousClassSnapshots: List<ClassSnapshotWithHash>,
+        metrics: BuildMetricsReporter
+    ): ChangeSet {
+        val currentClasses: Map<ClassId, ClassSnapshotWithHash> = currentClassSnapshots.associateBy { it.classSnapshot.getClassId() }
+        val previousClasses: Map<ClassId, ClassSnapshotWithHash> = previousClassSnapshots.associateBy { it.classSnapshot.getClassId() }
 
-        var startIndexToSearch = 0
-        return unchangedCurrentSnapshotFiles.associateWith { unchangedCurrentFile ->
-            val candidates = (sizeToPreviousFiles[unchangedCurrentFile.length()] ?: emptyList()).filter { it.index >= startIndexToSearch }
-            val unchangedPreviousFileWithIndex: IndexedValue<File> = if (candidates.size == 1) {
-                // A matching file must exist, so if there is only one candidate, it is the one.
-                candidates.single()
-            } else {
-                // If there are multiple matching files, select the first one. (Even if it doesn't match Gradle's alignment, it is still a
-                // correct alignment.)
-                val unchangedContents = unchangedCurrentFile.readBytes()
-                candidates.firstOrNull { candidate ->
-                    unchangedContents.contentEquals(candidate.value.readBytes())
-                } ?: error("Can't find previous snapshot file of unchanged current snapshot file '${unchangedCurrentFile.path}'")
-            }
-            startIndexToSearch = unchangedPreviousFileWithIndex.index + 1
-            unchangedPreviousFileWithIndex.value
+        val changedCurrentClasses: List<ClassSnapshot> = currentClasses.filter { (classId, currentClass) ->
+            val previousClass = previousClasses[classId]
+            previousClass == null || currentClass.hash != previousClass.hash
+        }.map { it.value.classSnapshot }
+
+        val changedPreviousClasses: List<ClassSnapshot> = previousClasses.filter { (classId, previousClass) ->
+            val currentClass = currentClasses[classId]
+            currentClass == null || currentClass.hash != previousClass.hash
+        }.map { it.value.classSnapshot }
+
+        val classChanges = metrics.measure(BuildTime.COMPUTE_CLASS_CHANGES) {
+            computeClassChanges(changedCurrentClasses, changedPreviousClasses, metrics)
         }
-    }
 
-    /** Returns `true` if this snapshot file contains a duplicate class with another snapshot file in the given list. */
-    @Suppress("unused", "UNUSED_PARAMETER")
-    private fun File.containsDuplicatesWith(otherSnapshotFiles: List<File>): Boolean {
-        // FIXME: Implement and optimize this method
-        return false
-    }
+        if (classChanges.isEmpty()) {
+            return classChanges
+        }
 
-    fun compute(currentClasspathSnapshot: ClasspathSnapshot, previousClasspathSnapshot: ClasspathSnapshot): ClasspathChanges {
-        val currentClassSnapshots = currentClasspathSnapshot.getNonDuplicateClassSnapshots()
-        val previousClassSnapshots = previousClasspathSnapshot.getNonDuplicateClassSnapshots()
-
-        return computeClassChanges(currentClassSnapshots, previousClassSnapshots)
+        return metrics.measure(BuildTime.COMPUTE_IMPACTED_SET) {
+            computeImpactedSet(classChanges, previousClasses.map { it.value.classSnapshot })
+        }
     }
 
     /**
-     * Returns all [ClassSnapshot]s in this [ClasspathSnapshot].
+     * Computes changes between the current and previous lists of classes. The returned result does not need to include elements that are
+     * impacted by the changes.
      *
-     * If there are duplicate classes on the classpath, retain only the first one to match the compiler's behavior.
+     * NOTE: Each list of classes must not contain duplicates.
      */
-    private fun ClasspathSnapshot.getNonDuplicateClassSnapshots(): List<ClassSnapshot> {
-        val classSnapshots = LinkedHashMap<String, ClassSnapshot>(classpathEntrySnapshots.sumOf { it.classSnapshots.size })
-        for (classpathEntrySnapshot in classpathEntrySnapshots) {
-            for ((unixStyleRelativePath, classSnapshot) in classpathEntrySnapshot.classSnapshots) {
-                classSnapshots.putIfAbsent(unixStyleRelativePath, classSnapshot)
+    fun computeClassChanges(
+        currentClassSnapshots: List<ClassSnapshot>,
+        previousClassSnapshots: List<ClassSnapshot>,
+        metrics: BuildMetricsReporter
+    ): ChangeSet {
+        val asmBasedSnapshotPredicate: (ClassSnapshot) -> Boolean = {
+            when (it) {
+                is RegularJavaClassSnapshot -> true
+                is KotlinClassSnapshot, is ProtoBasedJavaClassSnapshot -> false
+                else -> error("Unexpected type (it should have been handled earlier): ${it.javaClass.name}")
             }
         }
-        return classSnapshots.values.toList()
-    }
+        val (currentAsmBasedSnapshots, currentProtoBasedSnapshots) = currentClassSnapshots.partition(asmBasedSnapshotPredicate)
+        val (previousAsmBasedSnapshots, previousProtoBasedSnapshots) = previousClassSnapshots.partition(asmBasedSnapshotPredicate)
 
-    /**
-     * Computes changes between two lists of [ClassSnapshot]s.
-     *
-     * Each list must not contain duplicate classes.
-     */
-    fun computeClassChanges(currentClassSnapshots: List<ClassSnapshot>, previousClassSnapshots: List<ClassSnapshot>): ClasspathChanges {
-        if (currentClassSnapshots.any { it is ContentHashJavaClassSnapshot }
-            || previousClassSnapshots.any { it is ContentHashJavaClassSnapshot }) {
-            return ClasspathChanges.NotAvailable.UnableToCompute
+        val kotlinClassChanges = metrics.measure(BuildTime.COMPUTE_KOTLIN_CLASS_CHANGES) {
+            computeChangesForProtoBasedSnapshots(currentProtoBasedSnapshots, previousProtoBasedSnapshots)
         }
 
-        // Ignore `EmptyJavaClassSnapshot`s as they don't impact the result
-        val currentNonEmptyClassSnapshots = currentClassSnapshots.filter { it !is EmptyJavaClassSnapshot }
-        val previousNonEmptyClassSnapshots = previousClassSnapshots.filter { it !is EmptyJavaClassSnapshot }
-
-        val (currentAsmBasedSnapshots, currentProtoBasedSnapshots) =
-            currentNonEmptyClassSnapshots.partition { it is RegularJavaClassSnapshot }
-        val (previousAsmBasedSnapshots, previousProtoBasedSnapshots) =
-            previousNonEmptyClassSnapshots.partition { it is RegularJavaClassSnapshot }
-
-        val changeSet1 = computeChangesForProtoBasedSnapshots(currentProtoBasedSnapshots, previousProtoBasedSnapshots)
-
         @Suppress("UNCHECKED_CAST")
-        val changeSet2 = JavaClassChangesComputer.compute(
-            currentAsmBasedSnapshots as List<RegularJavaClassSnapshot>,
-            previousAsmBasedSnapshots as List<RegularJavaClassSnapshot>
-        )
-
-        val allChanges = changeSet1 + changeSet2
-        if (allChanges.isEmpty()) {
-            return allChanges.toClasspathChanges()
+        val javaClassChanges = metrics.measure(BuildTime.COMPUTE_JAVA_CLASS_CHANGES) {
+            JavaClassChangesComputer.compute(
+                currentAsmBasedSnapshots as List<RegularJavaClassSnapshot>,
+                previousAsmBasedSnapshots as List<RegularJavaClassSnapshot>
+            )
         }
 
-        val impactedSet = computeImpactedSet(allChanges, previousNonEmptyClassSnapshots)
-
-        return impactedSet.toClasspathChanges()
+        return kotlinClassChanges + javaClassChanges
     }
 
     private fun computeChangesForProtoBasedSnapshots(
@@ -203,7 +153,7 @@ object ClasspathChangesComputer {
                     )
                     incrementalJvmCache.markDirty(JvmClassName.byClassId(previousSnapshot.serializedJavaClass.classId))
                 }
-                is RegularJavaClassSnapshot, is ContentHashJavaClassSnapshot, is EmptyJavaClassSnapshot -> {
+                is RegularJavaClassSnapshot, is EmptyJavaClassSnapshot, is ContentHashJavaClassSnapshot -> {
                     error("Unexpected type (it should have been handled earlier): ${previousSnapshot.javaClass.name}")
                 }
             }
@@ -233,7 +183,7 @@ object ClasspathChangesComputer {
                         collector = changesCollector
                     )
                 }
-                is RegularJavaClassSnapshot, is ContentHashJavaClassSnapshot, is EmptyJavaClassSnapshot -> {
+                is RegularJavaClassSnapshot, is EmptyJavaClassSnapshot, is ContentHashJavaClassSnapshot -> {
                     error("Unexpected type (it should have been handled earlier): ${currentSnapshot.javaClass.name}")
                 }
             }
@@ -255,7 +205,7 @@ object ClasspathChangesComputer {
 
     private fun DirtyData.normalize(currentClassSnapshots: List<ClassSnapshot>, previousClassSnapshots: List<ClassSnapshot>): ChangeSet {
         val allClassIds = currentClassSnapshots.map { it.getClassId() }.toSet() + previousClassSnapshots.map { it.getClassId() }
-        val fqNameToClassId = LinkedHashMap<FqName, ClassId>(allClassIds.size)
+        val fqNameToClassId = HashMap<FqName, ClassId>(allClassIds.size)
         allClassIds.forEach { classId ->
             val fqName = classId.asSingleFqName()
             check(!fqNameToClassId.contains(fqName)) {
@@ -264,68 +214,70 @@ object ClasspathChangesComputer {
             fqNameToClassId[fqName] = classId
         }
 
-        return ChangeSet.Collector().run {
+        val changes = ChangeSet.Collector().run {
             dirtyLookupSymbols.forEach {
-                fqNameToClassId[FqName(it.scope)]?.let { classIdOfScope ->
-                    // If scope is a class, lookup symbol is a class member and maybe inner class
-                    fqNameToClassId[FqName("${it.scope}.${it.name}")]?.let { innerClass ->
-                        addChangedClass(innerClass)
-                    } ?: addChangedClassMember(classIdOfScope, it.name)
-                    return@forEach
+                val lookupSymbolFqName = if (it.scope.isEmpty()) FqName(it.name) else FqName("${it.scope}.${it.name}")
+                val lookupSymbolClassId: ClassId? = fqNameToClassId[lookupSymbolFqName]
+                if (lookupSymbolClassId != null) {
+                    addChangedClass(lookupSymbolClassId)
+                } else {
+                    // When lookupSymbolClassId == null, it means that either (1) the LookupSymbol does not refer to a class (it refers to
+                    // a class member or a package-level member), or (2) it refers to a class outside allClassIds. In the following, we
+                    // assume that (1) is the case.
+                    // (2) should typically never happen. In the case that it happens, we will collect incorrect changes, but it's
+                    // acceptable to collect more changes than necessary; also, we do not need to collect changes outside allClassIds, so
+                    // we're not collecting fewer changes than required.
+                    val scopeClassId: ClassId? = fqNameToClassId[FqName(it.scope)]
+                    if (scopeClassId != null) {
+                        addChangedClassMember(scopeClassId, it.name)
+                    } else {
+                        // Similarly, when scopeClassId == null, we assume that LookupSymbol.scope does not refer to a class outside
+                        // allClassIds. It means that the LookupSymbol does not refer to a class member. Therefore, it must refer to a
+                        // package-level member.
+                        addChangedTopLevelMember(FqName(it.scope), it.name)
+                    }
                 }
-
-                // scope is a package, so changed symbol is a top-level member and maybe a class
-                val potentialClassFqName = if (it.scope.isEmpty()) FqName(it.name) else FqName("${it.scope}.${it.name}")
-                fqNameToClassId[potentialClassFqName]?.let { classId ->
-                    // Lookup symbol is a class
-                    addChangedClass(classId)
-                } ?: addChangedTopLevelMember(FqName(it.scope), it.name)
             }
-            val changes = getChanges()
-
-            // dirtyClassesFqNames should be derived from dirtyLookupSymbols. Double-check that this is the case.
-            val changedFqNames: Set<FqName> =
-                changes.changedClasses.map { it.asSingleFqName() }.toSet() +
-                        changes.changedClassMembers.keys.map { it.asSingleFqName() } +
-                        changes.changedTopLevelMembers.keys
-            check(dirtyClassesFqNames.toSet() == changedFqNames) {
-                "Two sets differ:\n" +
-                        "dirtyClassesFqNames: $dirtyClassesFqNames\n" +
-                        "changedFqNames: $changedFqNames"
-            }
-            changes
+            getChanges()
         }
+
+        // DirtyData contains:
+        //   1. dirtyLookupSymbols => This contains all info we need (extracted above).
+        //   2. dirtyClassesFqNames => This should be derived from dirtyLookupSymbols.
+        //   3. dirtyClassesFqNamesForceRecompile => Should be irrelevant.
+        // Double-check that the assumption at bullet 2 above is correct.
+        val changedFqNames: Set<FqName> =
+            changes.changedClasses.map { it.asSingleFqName() }.toSet() +
+                    changes.changedClassMembers.keys.map { it.asSingleFqName() } +
+                    changes.changedTopLevelMembers.keys
+        check(dirtyClassesFqNames.toSet() == changedFqNames) {
+            "Two sets differ:\n" +
+                    "dirtyClassesFqNames: $dirtyClassesFqNames\n" +
+                    "changedFqNames: $changedFqNames"
+        }
+
+        return changes
     }
 }
 
-private fun ClassSnapshot.getClassId(): ClassId {
-    return when (this) {
-        is KotlinClassSnapshot -> classInfo.classId
-        is RegularJavaClassSnapshot -> classId
-        is ProtoBasedJavaClassSnapshot -> serializedJavaClass.classId
-        is EmptyJavaClassSnapshot, is ContentHashJavaClassSnapshot -> {
-            error("Unexpected type (it should have been handled earlier): ${javaClass.name}")
-        }
-    }
-}
-
-private object ImpactAnalysis {
+internal object ImpactAnalysis {
 
     /**
-     * Computes the set of classes/class members that are impacted by the given changes.
+     * Computes the set of classes, class members, and top-level members that are impacted by the given changes.
      *
-     * For example, if a superclass has changed, any of its subclasses will be impacted even if it has not changed, and unchanged source
+     * For example, if a superclass has changed, any of its subclasses will be impacted even if it has not changed because unchanged source
      * files in the previous compilation that depended on the subclasses will need to be recompiled.
      *
      * The returned set is also a [ChangeSet], which includes the given changes plus the impacted ones.
      */
     fun computeImpactedSet(changes: ChangeSet, previousClassSnapshots: List<ClassSnapshot>): ChangeSet {
         val classIdToSubclasses = getClassIdToSubclassesMap(previousClassSnapshots)
+        val impactedClassesResolver = { classId: ClassId -> classIdToSubclasses[classId] ?: emptySet() }
 
         return ChangeSet.Collector().run {
-            addChangedClasses(findSubclassesInclusive(changes.changedClasses, classIdToSubclasses))
+            addChangedClasses(findImpactedClassesInclusive(changes.changedClasses, impactedClassesResolver))
             for ((changedClass, changedClassMembers) in changes.changedClassMembers) {
-                findSubclassesInclusive(setOf(changedClass), classIdToSubclasses).forEach {
+                findImpactedClassesInclusive(setOf(changedClass), impactedClassesResolver).forEach {
                     addChangedClassMembers(it, changedClassMembers)
                 }
             }
@@ -337,7 +289,7 @@ private object ImpactAnalysis {
     }
 
     private fun getClassIdToSubclassesMap(classSnapshots: List<ClassSnapshot>): Map<ClassId, Set<ClassId>> {
-        val classIds = classSnapshots.map { it.getClassId() }
+        val classIds: Set<ClassId> = classSnapshots.map { it.getClassId() }.toSet() // Use Set for presence check
         val classNameToClassId = classIds.associateBy { JvmClassName.byClassId(it) }
         val classNameToClassIdResolver = { className: JvmClassName -> classNameToClassId[className] }
 
@@ -345,7 +297,7 @@ private object ImpactAnalysis {
         classSnapshots.forEach { classSnapshot ->
             val classId = classSnapshot.getClassId()
             classSnapshot.getSupertypes(classNameToClassIdResolver).forEach { supertype ->
-                // No need to collect supertypes outside the considered class snapshots (e.g., "java/lang/Object")
+                // No need to collect supertypes outside the given set of classes (e.g., "java/lang/Object")
                 if (supertype in classIds) {
                     classIdToSubclasses.computeIfAbsent(supertype) { mutableSetOf() }.add(classId)
                 }
@@ -354,38 +306,17 @@ private object ImpactAnalysis {
         return classIdToSubclasses
     }
 
-    private fun ClassSnapshot.getSupertypes(classIdResolver: (JvmClassName) -> ClassId?): List<ClassId> {
-        return when (this) {
-            is RegularJavaClassSnapshot -> supertypes.mapNotNull {
-                // The following call returns null if supertype is outside the considered class snapshots (e.g., "java/lang/Object").
-                // Use `mapNotNull` as we don't need to collect those supertypes (see getClassIdToSubclassesMap).
-                classIdResolver.invoke(it)
-            }
-            is KotlinClassSnapshot -> supertypes.mapNotNull {
-                // Same as above
-                classIdResolver.invoke(it)
-            }
-            is ProtoBasedJavaClassSnapshot -> {
-                val (proto, nameResolver) = serializedJavaClass.toProtoData()
-                proto.supertypes(TypeTable(proto.typeTable)).map { nameResolver.getClassId(it.className) }
-            }
-            is EmptyJavaClassSnapshot, is ContentHashJavaClassSnapshot -> {
-                error("Unexpected type (it should have been handled earlier): ${javaClass.name}")
-            }
-        }
-    }
-
     /**
-     * Finds direct and indirect subclasses of the given classes. The return set includes both the given classes and their direct and
-     * indirect subclasses.
+     * Finds directly and transitively impacted classes of the given classes. The return set includes both the given classes and the
+     * impacted classes.
      */
-    private fun findSubclassesInclusive(classIds: Set<ClassId>, classIdsToSubclasses: Map<ClassId, Set<ClassId>>): Set<ClassId> {
+    fun findImpactedClassesInclusive(classIds: Set<ClassId>, impactedClassesResolver: (ClassId) -> Set<ClassId>): Set<ClassId> {
         val visitedClasses = mutableSetOf<ClassId>()
         val toVisitClasses = classIds.toMutableSet()
         while (toVisitClasses.isNotEmpty()) {
             val nextToVisit = mutableSetOf<ClassId>()
             toVisitClasses.forEach {
-                nextToVisit.addAll(classIdsToSubclasses[it] ?: emptyList())
+                nextToVisit.addAll(impactedClassesResolver.invoke(it))
             }
             visitedClasses.addAll(toVisitClasses)
             toVisitClasses.clear()
@@ -394,3 +325,34 @@ private object ImpactAnalysis {
         return visitedClasses
     }
 }
+
+internal fun ClassSnapshot.getClassId(): ClassId {
+    return when (this) {
+        is KotlinClassSnapshot -> classInfo.classId
+        is RegularJavaClassSnapshot -> classId
+        is ProtoBasedJavaClassSnapshot -> serializedJavaClass.classId
+        is EmptyJavaClassSnapshot, is ContentHashJavaClassSnapshot -> {
+            error("Unexpected type (it should have been handled earlier): ${javaClass.name}")
+        }
+    }
+}
+
+/**
+ * Returns the [ClassId]s of the supertypes of this class.
+ *
+ * @param classIdResolver Resolves the [ClassId] from the [JvmClassName] of a supertype. It may return null if the supertype is outside the
+ *     considered set of classes (e.g., "java/lang/Object"). Those supertypes do not need to be included in the returned result.
+ */
+internal fun ClassSnapshot.getSupertypes(classIdResolver: (JvmClassName) -> ClassId?): List<ClassId> {
+    return when (this) {
+        is KotlinClassSnapshot -> supertypes.mapNotNull { classIdResolver.invoke(it) }
+        is RegularJavaClassSnapshot -> supertypes.mapNotNull { classIdResolver.invoke(it) }
+        is ProtoBasedJavaClassSnapshot -> {
+            val (proto, nameResolver) = serializedJavaClass.toProtoData()
+            proto.supertypes(TypeTable(proto.typeTable)).map { nameResolver.getClassId(it.className) }
+        }
+        is EmptyJavaClassSnapshot, is ContentHashJavaClassSnapshot -> {
+            error("Unexpected type (it should have been handled earlier): ${javaClass.name}")
+        }
+    }
+}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt

@@ -7,6 +7,8 @@ package org.jetbrains.kotlin.incremental.classpathDiff
 
 import org.jetbrains.kotlin.incremental.KotlinClassInfo
 import org.jetbrains.kotlin.incremental.SerializedJavaClass
+import org.jetbrains.kotlin.incremental.md5
+import org.jetbrains.kotlin.incremental.storage.toByteArray
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 
@@ -25,7 +27,7 @@ class ClasspathEntrySnapshot(
      * Maps (Unix-style) relative paths of classes to their snapshots. The paths are relative to the containing classpath entry (directory
      * or jar).
      */
-    val classSnapshots: LinkedHashMap<String, ClassSnapshot>
+    val classSnapshots: LinkedHashMap<String, ClassSnapshotWithHash>
 )
 
 /**
@@ -36,7 +38,14 @@ class ClasspathEntrySnapshot(
  * `KotlinCompile` task and the task needs to support compile avoidance. For example, this class should contain public method signatures,
  * and should not contain private method signatures, or method implementations.
  */
-sealed class ClassSnapshot
+sealed class ClassSnapshot {
+
+    /** Computes the hash of this [ClassSnapshot] and returns a [ClassSnapshotWithHash]. */
+    fun addHash() = ClassSnapshotWithHash(this, ClassSnapshotExternalizer.toByteArray(this).md5())
+}
+
+/** Contains a [ClassSnapshot] and its hash. */
+class ClassSnapshotWithHash(val classSnapshot: ClassSnapshot, val hash: Long)
 
 /** [ClassSnapshot] of a Kotlin class. */
 class KotlinClassSnapshot(

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotSerializer.kt

@@ -10,32 +10,59 @@ import org.jetbrains.kotlin.incremental.JavaClassProtoMapValueExternalizer
 import org.jetbrains.kotlin.incremental.KotlinClassInfo
 import org.jetbrains.kotlin.incremental.storage.*
 import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
-import java.io.*
+import java.io.DataInput
+import java.io.DataOutput
+import java.io.File
+import java.util.concurrent.ConcurrentHashMap
 
 /** Utility to serialize a [ClasspathSnapshot]. */
-object ClasspathSnapshotSerializer {
+object CachedClasspathSnapshotSerializer {
+    private val cache = ConcurrentHashMap<File, ClasspathEntrySnapshot>()
+    private const val RECOMMENDED_MAX_CACHE_SIZE = 100
 
     fun load(classpathEntrySnapshotFiles: List<File>): ClasspathSnapshot {
-        return ClasspathSnapshot(classpathEntrySnapshotFiles.map {
-            ClasspathEntrySnapshotSerializer.load(it)
-        })
+        return ClasspathSnapshot(classpathEntrySnapshotFiles.map { snapshotFile ->
+            cache.computeIfAbsent(snapshotFile) {
+                ClasspathEntrySnapshotExternalizer.loadFromFile(it)
+            }
+        }).also {
+            handleCacheEviction(recentlyReferencedKeys = classpathEntrySnapshotFiles)
+        }
+    }
+
+    private fun handleCacheEviction(recentlyReferencedKeys: List<File>) {
+        if (cache.size > RECOMMENDED_MAX_CACHE_SIZE) {
+            // Remove old entries.
+            // Note:
+            //   - The cache entries after eviction = recently-referenced entries + some other entries (so that
+            //     size = RECOMMENDED_MAX_CACHE_SIZE)
+            //       + Removed entries don't have to be the oldest (for simplicity).
+            //       + If recentlyReferencedKeys.size > RECOMMENDED_MAX_CACHE_SIZE, all of them will be kept. The reason is that
+            //         recently-referenced entries will likely be used again, so we keep them even if the cache is larger than recommended.
+            //   - It's okay to have race condition in this method.
+            val oldKeys = cache.keys - recentlyReferencedKeys.toSet()
+            for (oldKey in oldKeys) {
+                cache.remove(oldKey)
+                if (cache.size <= RECOMMENDED_MAX_CACHE_SIZE) break
+            }
+        }
     }
 }
 
-object ClasspathEntrySnapshotSerializer : DataSerializer<ClasspathEntrySnapshot> {
+object ClasspathEntrySnapshotExternalizer : DataExternalizer<ClasspathEntrySnapshot> {
 
     override fun save(output: DataOutput, snapshot: ClasspathEntrySnapshot) {
-        LinkedHashMapExternalizer(StringExternalizer, ClassSnapshotDataSerializer).save(output, snapshot.classSnapshots)
+        LinkedHashMapExternalizer(StringExternalizer, ClassSnapshotWithHashExternalizer).save(output, snapshot.classSnapshots)
     }
 
     override fun read(input: DataInput): ClasspathEntrySnapshot {
         return ClasspathEntrySnapshot(
-            classSnapshots = LinkedHashMapExternalizer(StringExternalizer, ClassSnapshotDataSerializer).read(input)
+            classSnapshots = LinkedHashMapExternalizer(StringExternalizer, ClassSnapshotWithHashExternalizer).read(input)
         )
     }
 }
 
-object ClassSnapshotDataSerializer : DataSerializer<ClassSnapshot> {
+object ClassSnapshotExternalizer : DataExternalizer<ClassSnapshot> {
 
     override fun save(output: DataOutput, snapshot: ClassSnapshot) {
         output.writeBoolean(snapshot is KotlinClassSnapshot)
@@ -55,6 +82,21 @@ object ClassSnapshotDataSerializer : DataSerializer<ClassSnapshot> {
     }
 }
 
+object ClassSnapshotWithHashExternalizer : DataExternalizer<ClassSnapshotWithHash> {
+
+    override fun save(output: DataOutput, snapshot: ClassSnapshotWithHash) {
+        ClassSnapshotExternalizer.save(output, snapshot.classSnapshot)
+        LongExternalizer.save(output, snapshot.hash)
+    }
+
+    override fun read(input: DataInput): ClassSnapshotWithHash {
+        return ClassSnapshotWithHash(
+            classSnapshot = ClassSnapshotExternalizer.read(input),
+            hash = LongExternalizer.read(input)
+        )
+    }
+}
+
 object KotlinClassSnapshotExternalizer : DataExternalizer<KotlinClassSnapshot> {
 
     override fun save(output: DataOutput, snapshot: KotlinClassSnapshot) {
@@ -183,32 +225,3 @@ object ContentHashJavaClassSnapshotExternalizer : DataExternalizer<ContentHashJa
         return ContentHashJavaClassSnapshot(contentHash = LongExternalizer.read(input))
     }
 }
-
-interface DataSerializer<T> : DataExternalizer<T> {
-
-    fun save(file: File, value: T) {
-        return DataOutputStream(FileOutputStream(file).buffered()).use {
-            save(it, value)
-        }
-    }
-
-    fun load(file: File): T {
-        return DataInputStream(FileInputStream(file).buffered()).use {
-            read(it)
-        }
-    }
-
-    fun toByteArray(value: T): ByteArray {
-        val byteArrayOutputStream = ByteArrayOutputStream()
-        DataOutputStream(byteArrayOutputStream.buffered()).use {
-            save(it, value)
-        }
-        return byteArrayOutputStream.toByteArray()
-    }
-
-    fun fromByteArray(byteArray: ByteArray): T {
-        return DataInputStream(ByteArrayInputStream(byteArray).buffered()).use {
-            read(it)
-        }
-    }
-}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt

@@ -0,0 +1,123 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental.classpathDiff
+
+import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
+import org.jetbrains.kotlin.build.report.metrics.BuildTime
+import org.jetbrains.kotlin.build.report.metrics.measure
+import org.jetbrains.kotlin.incremental.LookupStorage
+import org.jetbrains.kotlin.incremental.storage.LookupSymbolKey
+import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.resolve.jvm.JvmClassName
+
+object ClasspathSnapshotShrinker {
+
+    /**
+     * Shrinks the given [ClasspathSnapshot] by retaining only classes that are referenced. Referencing info is stored in [LookupStorage].
+     *
+     * This method also removes duplicate classes and [EmptyJavaClassSnapshot]s first.
+     */
+    fun shrink(
+        classpathSnapshot: ClasspathSnapshot,
+        lookupStorage: LookupStorage,
+        metrics: BuildMetricsReporter
+    ): List<ClassSnapshotWithHash> {
+        val allClasses = metrics.measure(BuildTime.GET_NON_DUPLICATE_CLASSES) {
+            // It's important to remove duplicate classes first before removing `EmptyJavaClassSnapshot`s.
+            // For example, if jar1!/com/example/A.class is empty and jar2!/com/example/A.class is non-empty, incorrect order of the actions
+            // will lead to incorrect results.
+            classpathSnapshot.getNonDuplicateClassSnapshots().filter { it.classSnapshot !is EmptyJavaClassSnapshot }
+        }
+        val lookupSymbols = metrics.measure(BuildTime.GET_LOOKUP_SYMBOLS) {
+            lookupStorage.lookupMap.keys
+        }
+        val referencedClasses = metrics.measure(BuildTime.FIND_REFERENCED_CLASSES) {
+            findReferencedClasses(allClasses, lookupSymbols)
+        }
+        return metrics.measure(BuildTime.FIND_TRANSITIVELY_REFERENCED_CLASSES) {
+            findTransitivelyReferencedClasses(allClasses, referencedClasses)
+        }
+    }
+
+    /**
+     * Finds classes that are referenced. Referencing info is stored in [LookupStorage].
+     *
+     * Note: It's okay to over-approximate referenced classes.
+     */
+    private fun findReferencedClasses(
+        allClasses: List<ClassSnapshotWithHash>,
+        lookupSymbols: Collection<LookupSymbolKey>
+    ): List<ClassSnapshotWithHash> {
+        val potentialClassNamesOfReferencedClasses =
+            lookupSymbols.flatMap {
+                val lookupSymbolFqName = if (it.scope.isEmpty()) FqName(it.name) else FqName("${it.scope}.${it.name}")
+                listOf(
+                    lookupSymbolFqName, // If LookupSymbol refers to a class, the class's FqName will be captured here.
+                    FqName(it.scope) // If LookupSymbol refers to a class member, the class's FqName will be captured here.
+                )
+            }.toSet() // Use Set for presence check
+        val potentialPackageNamesOfReferencedPackageLevelMembers =
+            lookupSymbols.map {
+                FqName(it.scope) // If LookupSymbol refers to a package-level member, the package's FqName will be captured here.
+            }.toSet() // Use Set for presence check
+
+        return allClasses.filter {
+            val classId = it.classSnapshot.getClassId()
+
+            (classId.asSingleFqName() in potentialClassNamesOfReferencedClasses) ||
+                    (it.classSnapshot is KotlinClassSnapshot
+                            && it.classSnapshot.classInfo.classKind != KotlinClassHeader.Kind.CLASS
+                            && classId.packageFqName in potentialPackageNamesOfReferencedPackageLevelMembers)
+        }
+    }
+
+    /**
+     * Finds classes that are transitively referenced. For example, if a subclass is referenced, its supertypes will potentially be
+     * referenced.
+     *
+     * The returned list includes the given referenced classes plus the transitively referenced ones.
+     */
+    private fun findTransitivelyReferencedClasses(
+        allClasses: List<ClassSnapshotWithHash>,
+        referencedClasses: List<ClassSnapshotWithHash>
+    ): List<ClassSnapshotWithHash> {
+        val classIdToClassSnapshot = allClasses.associateBy { it.classSnapshot.getClassId() }
+        val classIds: Set<ClassId> = classIdToClassSnapshot.keys // Use Set for presence check
+        val classNameToClassId = classIds.associateBy { JvmClassName.byClassId(it) }
+        val classNameToClassIdResolver = { className: JvmClassName -> classNameToClassId[className] }
+
+        val supertypesResolver = { classId: ClassId ->
+            // No need to collect supertypes outside the given set of classes (e.g., "java/lang/Object")
+            @Suppress("SimpleRedundantLet")
+            classIdToClassSnapshot[classId]?.let {
+                it.classSnapshot.getSupertypes(classNameToClassIdResolver).filter { supertype -> supertype in classIds }.toSet()
+            } ?: emptySet()
+        }
+
+        val referencedClassIds = referencedClasses.map { it.classSnapshot.getClassId() }.toSet()
+        val transitivelyReferencedClassIds: Set<ClassId> =
+            ImpactAnalysis.findImpactedClassesInclusive(referencedClassIds, supertypesResolver) // Use Set for presence check
+
+        return allClasses.filter { it.classSnapshot.getClassId() in transitivelyReferencedClassIds }
+    }
+}
+
+/**
+ * Returns all [ClassSnapshot]s in this [ClasspathSnapshot].
+ *
+ * If there are duplicate classes on the classpath, retain only the first one to match the compiler's behavior.
+ */
+internal fun ClasspathSnapshot.getNonDuplicateClassSnapshots(): List<ClassSnapshotWithHash> {
+    val classSnapshots = LinkedHashMap<String, ClassSnapshotWithHash>(classpathEntrySnapshots.sumOf { it.classSnapshots.size })
+    for (classpathEntrySnapshot in classpathEntrySnapshots) {
+        for ((unixStyleRelativePath, classSnapshot) in classpathEntrySnapshot.classSnapshots) {
+            classSnapshots.putIfAbsent(unixStyleRelativePath, classSnapshot)
+        }
+    }
+    return classSnapshots.values.toList()
+}

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt

@@ -38,10 +38,10 @@ object ClasspathEntrySnapshotter {
                 }
 
         val snapshots = try {
-            ClassSnapshotter.snapshot(classes, protoBased)
+            ClassSnapshotter.snapshot(classes, protoBased).map { it.addHash() }
         } catch (e: Throwable) {
             if (isKnownProblematicClasspathEntry(classpathEntry)) {
-                classes.map { ContentHashJavaClassSnapshot(it.contents.md5()) }
+                classes.map { ContentHashJavaClassSnapshot(it.contents.md5()).addHash() }
             } else throw e
         }
 

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt

@@ -5,11 +5,12 @@
 
 package org.jetbrains.kotlin.incremental.classpathDiff
 
+import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
+import org.jetbrains.kotlin.incremental.ChangesEither
+import org.jetbrains.kotlin.incremental.LookupSymbol
 import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotTestCommon.Util.compileAll
 import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotTestCommon.Util.snapshot
 import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotTestCommon.Util.snapshotAll
-import org.jetbrains.kotlin.incremental.ClasspathChanges
-import org.jetbrains.kotlin.incremental.LookupSymbol
 import org.jetbrains.kotlin.resolve.sam.SAM_LOOKUP_NAME
 import org.junit.Test
 import org.junit.rules.TemporaryFolder
@@ -46,7 +47,7 @@ class KotlinOnlyClasspathChangesComputerTest : ClasspathChangesComputerTest() {
         val sourceFile = SimpleKotlinClass(tmpDir)
         val previousSnapshot = sourceFile.compileAndSnapshot()
         val currentSnapshot = sourceFile.changePublicMethodSignature().compileAndSnapshot()
-        val changes = ClasspathChangesComputer.computeClassChanges(listOf(currentSnapshot), listOf(previousSnapshot)).normalize()
+        val changes = computeClassChanges(currentSnapshot, previousSnapshot)
 
         Changes(
             lookupSymbols = setOf(
@@ -62,7 +63,7 @@ class KotlinOnlyClasspathChangesComputerTest : ClasspathChangesComputerTest() {
         val sourceFile = SimpleKotlinClass(tmpDir)
         val previousSnapshot = sourceFile.compileAndSnapshot()
         val currentSnapshot = sourceFile.changeMethodImplementation().compileAndSnapshot()
-        val changes = ClasspathChangesComputer.computeClassChanges(listOf(currentSnapshot), listOf(previousSnapshot)).normalize()
+        val changes = computeClassChanges(currentSnapshot, previousSnapshot)
 
         Changes(emptySet(), emptySet()).assertEquals(changes)
     }
@@ -72,7 +73,7 @@ class KotlinOnlyClasspathChangesComputerTest : ClasspathChangesComputerTest() {
         val classpathSourceDir = File(testDataDir, "../ClasspathChangesComputerTest/testVariousAbiChanges/src/kotlin").canonicalFile
         val currentSnapshot = snapshotClasspath(File(classpathSourceDir, "current-classpath"), tmpDir)
         val previousSnapshot = snapshotClasspath(File(classpathSourceDir, "previous-classpath"), tmpDir)
-        val changes = ClasspathChangesComputer.compute(currentSnapshot, previousSnapshot).normalize()
+        val changes = computeClasspathChanges(currentSnapshot, previousSnapshot)
 
         Changes(
             lookupSymbols = setOf(
@@ -121,7 +122,7 @@ class KotlinOnlyClasspathChangesComputerTest : ClasspathChangesComputerTest() {
             File(testDataDir, "../ClasspathChangesComputerTest/testImpactAnalysis_KotlinOrJava/src/kotlin").canonicalFile
         val currentSnapshot = snapshotClasspath(File(classpathSourceDir, "current-classpath"), tmpDir)
         val previousSnapshot = snapshotClasspath(File(classpathSourceDir, "previous-classpath"), tmpDir)
-        val changes = ClasspathChangesComputer.compute(currentSnapshot, previousSnapshot).normalize()
+        val changes = computeClasspathChanges(currentSnapshot, previousSnapshot)
 
         Changes(
             lookupSymbols = setOf(
@@ -158,7 +159,7 @@ class JavaOnlyClasspathChangesComputerTest(private val protoBased: Boolean) : Cl
         val sourceFile = SimpleJavaClass(tmpDir)
         val previousSnapshot = sourceFile.compile().snapshot(protoBased)
         val currentSnapshot = sourceFile.changePublicMethodSignature().compile().snapshot(protoBased)
-        val changes = ClasspathChangesComputer.computeClassChanges(listOf(currentSnapshot), listOf(previousSnapshot)).normalize()
+        val changes = computeClassChanges(currentSnapshot, previousSnapshot)
 
         Changes(
             lookupSymbols = setOf(
@@ -174,7 +175,7 @@ class JavaOnlyClasspathChangesComputerTest(private val protoBased: Boolean) : Cl
         val sourceFile = SimpleJavaClass(tmpDir)
         val previousSnapshot = sourceFile.compile().snapshot(protoBased)
         val currentSnapshot = sourceFile.changeMethodImplementation().compile().snapshot(protoBased)
-        val changes = ClasspathChangesComputer.computeClassChanges(listOf(currentSnapshot), listOf(previousSnapshot)).normalize()
+        val changes = computeClassChanges(currentSnapshot, previousSnapshot)
 
         Changes(emptySet(), emptySet()).assertEquals(changes)
     }
@@ -184,7 +185,7 @@ class JavaOnlyClasspathChangesComputerTest(private val protoBased: Boolean) : Cl
         val classpathSourceDir = File(testDataDir, "../ClasspathChangesComputerTest/testVariousAbiChanges/src/java").canonicalFile
         val currentSnapshot = snapshotClasspath(File(classpathSourceDir, "current-classpath"), tmpDir, protoBased)
         val previousSnapshot = snapshotClasspath(File(classpathSourceDir, "previous-classpath"), tmpDir, protoBased)
-        val changes = ClasspathChangesComputer.compute(currentSnapshot, previousSnapshot).normalize()
+        val changes = computeClasspathChanges(currentSnapshot, previousSnapshot)
 
         Changes(
             lookupSymbols = setOf(
@@ -229,7 +230,7 @@ class JavaOnlyClasspathChangesComputerTest(private val protoBased: Boolean) : Cl
         val classpathSourceDir = File(testDataDir, "../ClasspathChangesComputerTest/testImpactAnalysis_KotlinOrJava/src/java").canonicalFile
         val currentSnapshot = snapshotClasspath(File(classpathSourceDir, "current-classpath"), tmpDir, protoBased)
         val previousSnapshot = snapshotClasspath(File(classpathSourceDir, "previous-classpath"), tmpDir, protoBased)
-        val changes = ClasspathChangesComputer.compute(currentSnapshot, previousSnapshot).normalize()
+        val changes = computeClasspathChanges(currentSnapshot, previousSnapshot)
 
         Changes(
             lookupSymbols = setOf(
@@ -259,7 +260,7 @@ class KotlinAndJavaClasspathChangesComputerTest : ClasspathSnapshotTestCommon()
         val classpathSourceDir = File(testDataDir, "../ClasspathChangesComputerTest/testImpactAnalysis_KotlinAndJava/src").canonicalFile
         val currentSnapshot = snapshotClasspath(File(classpathSourceDir, "current-classpath"), tmpDir)
         val previousSnapshot = snapshotClasspath(File(classpathSourceDir, "previous-classpath"), tmpDir)
-        val changes = ClasspathChangesComputer.compute(currentSnapshot, previousSnapshot).normalize()
+        val changes = computeClasspathChanges(currentSnapshot, previousSnapshot)
 
         Changes(
             lookupSymbols = setOf(
@@ -301,7 +302,7 @@ private fun snapshotClasspath(classpathSourceDir: File, tmpDir: TemporaryFolder,
         classpath.addAll(listOfNotNull(classFiles.firstOrNull()?.classRoot))
 
         val relativePaths = classFiles.map { it.unixStyleRelativePath }
-        val classSnapshots = classFiles.snapshotAll(protoBased)
+        val classSnapshots = classFiles.snapshotAll(protoBased).map { it.addHash() }
         ClasspathEntrySnapshot(
             classSnapshots = relativePaths.zip(classSnapshots).toMap(LinkedHashMap())
         )
@@ -309,12 +310,31 @@ private fun snapshotClasspath(classpathSourceDir: File, tmpDir: TemporaryFolder,
     return ClasspathSnapshot(classpathEntrySnapshots)
 }
 
-/** Adapted version of [ClasspathChanges.Available] for readability in this test. */
+private fun computeClasspathChanges(
+    currentClasspathSnapshot: ClasspathSnapshot,
+    previousClasspathSnapshot: ClasspathSnapshot
+): Changes {
+    return ClasspathChangesComputer.computeChangedAndImpactedSet(
+        currentClasspathSnapshot.getNonDuplicateClassSnapshots(),
+        previousClasspathSnapshot.getNonDuplicateClassSnapshots(),
+        DoNothingBuildMetricsReporter
+    ).normalize()
+}
+
+private fun computeClassChanges(currentClassSnapshot: ClassSnapshot, previousClassSnapshot: ClassSnapshot): Changes {
+    return ClasspathChangesComputer.computeClassChanges(
+        listOf(currentClassSnapshot),
+        listOf(previousClassSnapshot),
+        DoNothingBuildMetricsReporter
+    ).normalize()
+}
+
+/** Adapted version of [ChangesEither.Known] for readability in this test. */
 private data class Changes(val lookupSymbols: Set<LookupSymbol>, val fqNames: Set<String>)
 
-private fun ClasspathChanges.normalize(): Changes {
-    this as ClasspathChanges.Available
-    return Changes(lookupSymbols, fqNames.map { it.asString() }.toSet())
+private fun ChangeSet.normalize(): Changes {
+    val changes: ChangesEither.Known = getChanges()
+    return Changes(changes.lookupSymbols.toSet(), changes.fqNames.map { it.asString() }.toSet())
 }
 
 private fun Changes.assertEquals(actual: Changes) {

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotSerializerTest.kt

@@ -6,6 +6,8 @@
 package org.jetbrains.kotlin.incremental.classpathDiff
 
 import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotTestCommon.Util.readBytes
+import org.jetbrains.kotlin.incremental.storage.fromByteArray
+import org.jetbrains.kotlin.incremental.storage.toByteArray
 import org.junit.Assert.assertEquals
 import org.junit.Test
 
@@ -16,8 +18,8 @@ abstract class ClasspathSnapshotSerializerTest : ClasspathSnapshotTestCommon() {
     @Test
     open fun `test ClassSnapshotDataSerializer`() {
         val originalSnapshot = testSourceFile.compileAndSnapshot()
-        val serializedSnapshot = ClassSnapshotDataSerializer.toByteArray(originalSnapshot)
-        val deserializedSnapshot = ClassSnapshotDataSerializer.fromByteArray(serializedSnapshot)
+        val serializedSnapshot = ClassSnapshotExternalizer.toByteArray(originalSnapshot)
+        val deserializedSnapshot = ClassSnapshotExternalizer.fromByteArray(serializedSnapshot)
 
         assertEquals(originalSnapshot.toGson(), deserializedSnapshot.toGson())
     }
@@ -36,8 +38,8 @@ class JavaClassesClasspathSnapshotSerializerTest : ClasspathSnapshotSerializerTe
         val originalSnapshot = testSourceFile.compile().let {
             ClassSnapshotter.snapshot(listOf(ClassFileWithContents(it, it.readBytes())), includeDebugInfoInSnapshot = false)
         }.single()
-        val serializedSnapshot = ClassSnapshotDataSerializer.toByteArray(originalSnapshot)
-        val deserializedSnapshot = ClassSnapshotDataSerializer.fromByteArray(serializedSnapshot)
+        val serializedSnapshot = ClassSnapshotExternalizer.toByteArray(originalSnapshot)
+        val deserializedSnapshot = ClassSnapshotExternalizer.fromByteArray(serializedSnapshot)
 
         assertEquals(originalSnapshot.toGson(), deserializedSnapshot.toGson())
     }

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinkerTest.kt

@@ -0,0 +1,9 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental.classpathDiff
+
+// TODO: Write this test
+abstract class ClasspathSnapshotShrinkerTest

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon.kt

@@ -21,7 +21,7 @@ abstract class ClasspathSnapshotTestCommon {
 
     companion object {
         val testDataDir =
-            File("libraries/tools/kotlin-gradle-plugin/src/testData/org/jetbrains/kotlin/gradle/incremental/ClasspathSnapshotTestCommon")
+            File("compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon")
     }
 
     @get:Rule

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt

@@ -9,8 +9,9 @@ import org.gradle.api.artifacts.transform.*
 import org.gradle.api.file.FileSystemLocation
 import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.Classpath
-import org.jetbrains.kotlin.gradle.incremental.ClasspathEntrySnapshotSerializer
-import org.jetbrains.kotlin.gradle.incremental.ClasspathEntrySnapshotter
+import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotExternalizer
+import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotter
+import org.jetbrains.kotlin.incremental.storage.saveToFile
 
 /** Transform to create a snapshot of a classpath entry (directory or jar). */
 @CacheableTransform
@@ -25,6 +26,6 @@ abstract class ClasspathEntrySnapshotTransform : TransformAction<TransformParame
         val snapshotFile = outputs.file(classpathEntry.name.replace('.', '_') + "-snapshot.bin")
 
         val snapshot = ClasspathEntrySnapshotter.snapshot(classpathEntry)
-        ClasspathEntrySnapshotSerializer.save(snapshotFile, snapshot)
+        ClasspathEntrySnapshotExternalizer.saveToFile(snapshotFile, snapshot)
     }
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt

@@ -55,6 +55,9 @@ import org.jetbrains.kotlin.gradle.targets.js.ir.isProduceUnzippedKlib
 import org.jetbrains.kotlin.gradle.utils.*
 import org.jetbrains.kotlin.incremental.ChangedFiles
 import org.jetbrains.kotlin.incremental.ClasspathChanges
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotDisabled
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.*
+import org.jetbrains.kotlin.incremental.ClasspathSnapshotFiles
 import org.jetbrains.kotlin.incremental.IncrementalCompilerRunner
 import org.jetbrains.kotlin.library.impl.isKotlinLibrary
 import org.jetbrains.kotlin.statistics.BuildSessionLogger
@@ -584,7 +587,6 @@ abstract class KotlinCompile @Inject constructor(
                 )
                 val classpathSnapshotDir = getClasspathSnapshotDir(task)
                 task.classpathSnapshotProperties.classpathSnapshotDir.value(classpathSnapshotDir).disallowChanges()
-                task.classpathSnapshotProperties.classpathSnapshotDirFileCollection.from(classpathSnapshotDir)
             } else {
                 task.classpathSnapshotProperties.classpath.from(task.project.provider { task.classpath })
             }
@@ -644,14 +646,6 @@ abstract class KotlinCompile @Inject constructor(
         @get:OutputDirectory
         @get:Optional // Set if useClasspathSnapshot == true
         abstract val classpathSnapshotDir: DirectoryProperty
-
-        /**
-         * [FileCollection] containing a single file which is [classpathSnapshotDir], used when a [FileCollection] is required instead of a
-         * [DirectoryProperty].
-         */
-        // Set if useClasspathSnapshot == true
-        @get:Internal
-        abstract val classpathSnapshotDirFileCollection: ConfigurableFileCollection
     }
 
     @get:Internal
@@ -740,15 +734,10 @@ abstract class KotlinCompile @Inject constructor(
         val compilerRunner = compilerRunner.get()
 
         val icEnv = if (isIncrementalCompilationEnabled()) {
-            val classpathChanges = when {
-                !classpathSnapshotProperties.useClasspathSnapshot.get() -> ClasspathChanges.NotAvailable.ClasspathSnapshotIsDisabled
-                inputChanges.isIncremental -> getClasspathChanges(inputChanges)
-                else -> ClasspathChanges.NotAvailable.ForNonIncrementalRun
-            }
             logger.info(USING_JVM_INCREMENTAL_COMPILATION_MESSAGE)
             IncrementalCompilationEnvironment(
                 changedFiles = getChangedFiles(inputChanges, incrementalProps),
-                classpathChanges = classpathChanges,
+                classpathChanges = getClasspathChanges(inputChanges),
                 workingDir = taskBuildDirectory.get().asFile,
                 usePreciseJavaTracking = usePreciseJavaTracking,
                 disableMultiModuleIC = disableMultiModuleIC,
@@ -756,16 +745,9 @@ abstract class KotlinCompile @Inject constructor(
             )
         } else null
 
-        with(classpathSnapshotProperties) {
-            if (isIncrementalCompilationEnabled() && useClasspathSnapshot.get()) {
-                copyCurrentClasspathEntrySnapshotFiles()
-            }
-        }
-
         val environment = GradleCompilerEnvironment(
             defaultCompilerClasspath, messageCollector, outputItemCollector,
-            // The compiler runner should not manage (read, modify, or delete) classpathSnapshotDir
-            outputFiles = allOutputFiles().minus(classpathSnapshotProperties.classpathSnapshotDirFileCollection),
+            outputFiles = allOutputFiles(),
             reportingSettings = reportingSettings,
             incrementalCompilationEnvironment = icEnv,
             kotlinScriptExtensions = sourceFilesExtensions.get().toTypedArray()
@@ -848,62 +830,29 @@ abstract class KotlinCompile @Inject constructor(
         return super.source(*sources)
     }
 
-    private fun getClasspathChanges(inputChanges: InputChanges): ClasspathChanges {
-        val fileChanges = inputChanges.getFileChanges(classpathSnapshotProperties.classpathSnapshot).toList()
-        return if (fileChanges.isEmpty()) {
-            ClasspathChanges.Available(LinkedHashSet(), LinkedHashSet())
-        } else {
-            val previousClasspathEntrySnapshotFiles = getPreviousClasspathEntrySnapshotFiles()
-            if (previousClasspathEntrySnapshotFiles.isEmpty()) {
-                // When this happens, it means that either the previous classpath was empty or there were no source files to compile in the
-                // previous non-incremental run so the task action was skipped and the classpath snapshot directory was not populated (see
-                // AbstractKotlinCompile.executeImpl).
-                // We could improve this handling, but it's fine to return `UnableToCompute` here as it's likely that there are also no
-                // source files to compile/recompile in this incremental run.
-                ClasspathChanges.NotAvailable.UnableToCompute
-            } else {
-                val currentClasspathEntrySnapshotFiles = classpathSnapshotProperties.classpathSnapshot.files.toList()
-                val changedCurrentFiles = fileChanges
-                    .filter { it.changeType == ChangeType.ADDED || it.changeType == ChangeType.MODIFIED }
-                    .map { it.file }.toSet()
-                ClasspathChangesComputer.compute(
-                    currentClasspathEntrySnapshotFiles = currentClasspathEntrySnapshotFiles,
-                    previousClasspathEntrySnapshotFiles = previousClasspathEntrySnapshotFiles,
-                    unchangedCurrentClasspathEntrySnapshotFiles = currentClasspathEntrySnapshotFiles.filter { it !in changedCurrentFiles }
-                )
+    private fun getClasspathChanges(inputChanges: InputChanges): ClasspathChanges = when {
+        !classpathSnapshotProperties.useClasspathSnapshot.get() -> ClasspathSnapshotDisabled
+        else -> {
+            val classpathSnapshotFiles = ClasspathSnapshotFiles(
+                classpathSnapshotProperties.classpathSnapshot.files.toList(),
+                classpathSnapshotProperties.classpathSnapshotDir.get().asFile
+            )
+            when {
+                !inputChanges.isIncremental -> NotAvailableForNonIncrementalRun(classpathSnapshotFiles)
+                inputChanges.getFileChanges(classpathSnapshotProperties.classpathSnapshot).none() -> Empty(classpathSnapshotFiles)
+                !classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.exists() -> {
+                    // When this happens, it means that the classpath snapshot in the previous run was not saved for some reason. It's
+                    // likely that there were no source files to compile, so the task action was skipped (see
+                    // AbstractKotlinCompile.executeImpl), and therefore the classpath snapshot was not saved.
+                    // Missing classpath snapshot will make this run non-incremental, but because there were no source files to compile in
+                    // the previous run, *all* source files in this run (if there are any) need to be compiled anyway, so being
+                    // non-incremental is actually okay.
+                    NotAvailableDueToMissingClasspathSnapshot(classpathSnapshotFiles)
+                }
+                else -> ToBeComputedByIncrementalCompiler(classpathSnapshotFiles)
             }
         }
     }
-
-    /**
-     * Copies classpath entry snapshot files of the current build to `classpathSnapshotDir`. They will be used in the next build (see
-     * [getPreviousClasspathEntrySnapshotFiles]).
-     *
-     * To preserve their order, we put them in subdirectories with names being their indices, as shown below:
-     *     classpathSnapshotDir/0/a-snapshot.bin
-     *     classpathSnapshotDir/1/b-snapshot.bin
-     *     ...
-     *     classpathSnapshotDir/N-1/z-snapshot.bin
-     */
-    private fun copyCurrentClasspathEntrySnapshotFiles() {
-        val snapshotFiles = classpathSnapshotProperties.classpathSnapshot.files.toList()
-        val classpathSnapshotDir = classpathSnapshotProperties.classpathSnapshotDir.get().asFile
-        classpathSnapshotDir.deleteRecursively()
-        classpathSnapshotDir.mkdirs()
-        snapshotFiles.forEachIndexed { index, snapshotFile ->
-            snapshotFile.copyTo(File("$classpathSnapshotDir/$index/${snapshotFile.name}"))
-        }
-    }
-
-    /**
-     * Returns classpath entry snapshot files of the previous build, stored in `classpathSnapshotDir` (see
-     * [copyCurrentClasspathEntrySnapshotFiles]).
-     */
-    private fun getPreviousClasspathEntrySnapshotFiles(): List<File> {
-        val classpathSnapshotDir = classpathSnapshotProperties.classpathSnapshotDir.get().asFile
-        val subDirs = classpathSnapshotDir.listFiles()!!.sortedBy { it.name.toInt() }
-        return subDirs.map { it.listFiles()!!.single() }
-    }
 }
 
 @CacheableTask
@@ -1152,7 +1101,7 @@ abstract class Kotlin2JsCompile @Inject constructor(
             logger.info(USING_JS_INCREMENTAL_COMPILATION_MESSAGE)
             IncrementalCompilationEnvironment(
                 getChangedFiles(inputChanges, incrementalProps),
-                ClasspathChanges.NotAvailable.ForJSCompiler,
+                ClasspathChanges.NotAvailableForJSCompiler,
                 taskBuildDirectory.get().asFile,
                 multiModuleICSettings = multiModuleICSettings
             )