KT-45777: Move classpath diffing to incremental Kotlin compiler (2/2)

as we need access to the lookup tracker to compute classpath changes more efficiently and reduce the size of the saved classpath snapshot. The previous commit only changed the files' paths, this commit actually updates the files' contents. Note that classpath snapshotting still happens in Gradle artifact transforms. (However, the previous commit also moved the code for classpath snapshotting together with the code for classpath diffing as they are closely related.)
2021-11-10 10:57:02 +00:00
parent dfaf195e1d
commit f52be5f471
25 changed files with 659 additions and 491 deletions
@@ -18,6 +18,7 @@ enum class BuildAttributeKind : Serializable {
 enum class BuildAttribute(val kind: BuildAttributeKind, val readableString: String) : Serializable {
    NO_BUILD_HISTORY(BuildAttributeKind.REBUILD_REASON, "Build history file not found"),
    NO_ABI_SNAPSHOT(BuildAttributeKind.REBUILD_REASON, "ABI snapshot not found"),
+    CLASSPATH_SNAPSHOT_NOT_FOUND(BuildAttributeKind.REBUILD_REASON, "Classpath snapshot not found"),
    CACHE_CORRUPTION(BuildAttributeKind.REBUILD_REASON, "Cache corrupted"),
    UNKNOWN_CHANGES_IN_GRADLE_INPUTS(BuildAttributeKind.REBUILD_REASON, "Unknown Gradle changes"),
    JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED(BuildAttributeKind.REBUILD_REASON, "Untracked Java file is removed"),
@@ -10,9 +10,13 @@ import java.io.Serializable

@Suppress("Reformat")
 enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, val readableString: String) : Serializable {
-    OUTPUT_SIZE(readableString = "Total output size"),
-        LOOKUP_SIZE(OUTPUT_SIZE, "Lookups size"),
-        SNAPSHOT_SIZE(OUTPUT_SIZE, "ABI snapshot size"),
+    CACHE_DIRECTORY_SIZE(readableString = "Total size of the cache directory"),
+        LOOKUP_SIZE(CACHE_DIRECTORY_SIZE, "Lookups size"),
+        SNAPSHOT_SIZE(CACHE_DIRECTORY_SIZE, "ABI snapshot size"),
+
+    // Metrics for the `kotlin.incremental.useClasspathSnapshot` feature
+    ORIGINAL_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the original classpath snapshot (before shrinking)"),
+    SHRUNK_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the shrunk classpath snapshot"),
    ;

    companion object {
@@ -27,6 +27,15 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String)
                    SET_UP_ABI_SNAPSHOTS(JAR_SNAPSHOT, "Set up ABI snapshot"),
                    IC_ANALYZE_JAR_FILES(JAR_SNAPSHOT, "Analyze jar files"),
                IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION, "Init dirty symbols set"),
+                    COMPUTE_CLASSPATH_CHANGES(IC_CALCULATE_INITIAL_DIRTY_SET, "Compute classpath changes"),
+                        LOAD_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load current classpath snapshot"),
+                        SHRINK_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Shrink current classpath snapshot"),
+                        LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load shrunk previous classpath snapshot"),
+                        COMPUTE_CHANGED_AND_IMPACTED_SET(COMPUTE_CLASSPATH_CHANGES, "Compute changed and impacted set"),
+                            COMPUTE_CLASS_CHANGES(COMPUTE_CHANGED_AND_IMPACTED_SET, "Compute class changes"),
+                                COMPUTE_KOTLIN_CLASS_CHANGES(COMPUTE_CLASS_CHANGES, "Compute Kotlin class changes"),
+                                COMPUTE_JAVA_CLASS_CHANGES(COMPUTE_CLASS_CHANGES, "Compute Java class changes"),
+                            COMPUTE_IMPACTED_SET(COMPUTE_CHANGED_AND_IMPACTED_SET, "Compute impacted set"),
                    IC_ANALYZE_CHANGES_IN_DEPENDENCIES(IC_CALCULATE_INITIAL_DIRTY_SET, "Analyze dependency changes"),
                        IC_FIND_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES, "Find history files"),
                        IC_ANALYZE_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES, "Analyze history files"),
@@ -38,6 +47,14 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String)
                INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Incremental iteration"),
                NON_INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Non-incremental iteration"),
                IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION, "Write history file"),
+                SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION, "Save shrunk current classpath snapshot after compilation"),
+                    LOAD_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load classpath snapshot"),
+                    SHRINK_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink classpath snapshot"),
+                        GET_NON_DUPLICATE_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Get non-duplicate classes"),
+                        GET_LOOKUP_SYMBOLS(SHRINK_CLASSPATH_SNAPSHOT, "Get lookup symbols"),
+                        FIND_REFERENCED_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Find referenced classes"),
+                        FIND_TRANSITIVELY_REFERENCED_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Find transitively referenced classes"),
+                    SAVE_SHRUNK_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk classpath snapshot"),
    COMPILER_PERFORMANCE(readableString = "Compiler time"),
        COMPILER_INITIALIZATION(COMPILER_PERFORMANCE, "Compiler initialization time"),
        CODE_ANALYSIS(COMPILER_PERFORMANCE, "Compiler code analyse"),
@@ -5,70 +5,43 @@

 package org.jetbrains.kotlin.incremental

-import com.intellij.util.io.DataExternalizer
-import org.jetbrains.kotlin.incremental.storage.FqNameExternalizer
-import org.jetbrains.kotlin.incremental.storage.SetExternalizer
-import org.jetbrains.kotlin.incremental.storage.LookupSymbolExternalizer
-import org.jetbrains.kotlin.name.FqName
-import java.io.*
+import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled
+import java.io.File
+import java.io.Serializable

 /**
- * Changes to the classpath of the `KotlinCompile` task, used to compute the source files that need to be recompiled during an incremental
- * run.
+ * Changes to the classpath of the `KotlinCompile` task, or information to compute them later by the Kotlin incremental compiler (see
+ * [ClasspathSnapshotEnabled.ToBeComputedByIncrementalCompiler].
 */
 sealed class ClasspathChanges : Serializable {

-    class Available() : ClasspathChanges() {
+    sealed class ClasspathSnapshotEnabled : ClasspathChanges() {

-        companion object {
-            private const val serialVersionUID = 0L
-        }
+        abstract val classpathSnapshotFiles: ClasspathSnapshotFiles

-        lateinit var lookupSymbols: Set<LookupSymbol> // Preferably ordered but not required
-            private set
+        class Empty(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled()

-        lateinit var fqNames: Set<FqName> // Preferably ordered but not required
-            private set
+        class ToBeComputedByIncrementalCompiler(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled()

-        constructor(lookupSymbols: Set<LookupSymbol>, fqNames: Set<FqName>) : this() {
-            this.lookupSymbols = lookupSymbols
-            this.fqNames = fqNames
-        }
+        class NotAvailableDueToMissingClasspathSnapshot(override val classpathSnapshotFiles: ClasspathSnapshotFiles) :
+            ClasspathSnapshotEnabled()

-        private fun writeObject(output: ObjectOutputStream) {
-            // Can't close DataOutputStream below as it will also close the underlying ObjectOutputStream, which is still in use.
-            ClasspathChangesAvailableExternalizer.save(DataOutputStream(output), this)
-        }
-
-        private fun readObject(input: ObjectInputStream) {
-            // Can't close DataInputStream below as it will also close the underlying ObjectInputStream, which is still in use.
-            ClasspathChangesAvailableExternalizer.read(DataInputStream(input)).also {
-                lookupSymbols = it.lookupSymbols
-                fqNames = it.fqNames
-            }
-        }
+        class NotAvailableForNonIncrementalRun(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled()
    }

-    sealed class NotAvailable : ClasspathChanges() {
-        object UnableToCompute : NotAvailable()
-        object ForNonIncrementalRun : NotAvailable()
-        object ClasspathSnapshotIsDisabled : NotAvailable()
-        object ReservedForTestsOnly : NotAvailable()
-        object ForJSCompiler : NotAvailable()
-    }
+    object ClasspathSnapshotDisabled : ClasspathChanges()
+
+    object NotAvailableForJSCompiler : ClasspathChanges()
 }

-private object ClasspathChangesAvailableExternalizer : DataExternalizer<ClasspathChanges.Available> {
+class ClasspathSnapshotFiles(
+    val currentClasspathEntrySnapshotFiles: List<File>,
+    classpathSnapshotDir: File
+) : Serializable {

-    override fun save(output: DataOutput, classpathChanges: ClasspathChanges.Available) {
-        SetExternalizer(LookupSymbolExternalizer).save(output, classpathChanges.lookupSymbols)
-        SetExternalizer(FqNameExternalizer).save(output, classpathChanges.fqNames)
-    }
+    val shrunkPreviousClasspathSnapshotFile: File = File(classpathSnapshotDir, "shrunk-classpath-snapshot.bin")

-    override fun read(input: DataInput): ClasspathChanges.Available {
-        return ClasspathChanges.Available(
-            lookupSymbols = SetExternalizer(LookupSymbolExternalizer).read(input),
-            fqNames = SetExternalizer(FqNameExternalizer).read(input)
-        )
+    companion object {
+        private const val serialVersionUID = 0L
    }
 }
@@ -32,7 +32,8 @@ import java.util.*

 open class LookupStorage(
    targetDataDir: File,
-    pathConverter: FileToPathConverter
+    pathConverter: FileToPathConverter,
+    storeFullFqNames: Boolean = false
 ) : BasicMapsOwner(targetDataDir) {
    val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder")

@@ -44,7 +45,7 @@ open class LookupStorage(
    private val countersFile = "counters".storageFile
    private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, pathConverter))
    private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, pathConverter))
-    val lookupMap = registerMap(LookupMap("lookups".storageFile))
+    val lookupMap = registerMap(LookupMap("lookups".storageFile, storeFullFqNames))

    @Volatile
    private var size: Int = 0
@@ -18,7 +18,9 @@ package org.jetbrains.kotlin.incremental.storage

 import java.io.File

-class LookupMap(storage: File) : BasicMap<LookupSymbolKey, Collection<Int>>(storage, LookupSymbolKeyDescriptor, IntCollectionExternalizer) {
+class LookupMap(storage: File, storeFullFqNames: Boolean) :
+    BasicMap<LookupSymbolKey, Collection<Int>>(storage, LookupSymbolKeyDescriptor(storeFullFqNames), IntCollectionExternalizer) {
+
    override fun dumpKey(key: LookupSymbolKey): String = key.toString()

    override fun dumpValue(value: Collection<Int>): String = value.toString()
@@ -22,22 +22,21 @@ import com.intellij.util.io.DataExternalizer
 import com.intellij.util.io.EnumeratorStringDescriptor
 import com.intellij.util.io.IOUtil
 import com.intellij.util.io.KeyDescriptor
-import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
-import org.jetbrains.kotlin.cli.common.toBooleanLenient
-import org.jetbrains.kotlin.incremental.LookupSymbol
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
-import java.io.DataInput
-import java.io.DataInputStream
-import java.io.DataOutput
+import java.io.*

 /**
 * Storage versioning:
 * 0 - only name and value hashes are saved
 * 1 - name and scope are saved
 */
-object LookupSymbolKeyDescriptor : KeyDescriptor<LookupSymbolKey> {
+class LookupSymbolKeyDescriptor(
+    /** If `true`, original values are saved; if `false`, only hashes are saved. */
+    private val storeFullFqNames: Boolean = false
+) : KeyDescriptor<LookupSymbolKey> {
+
    override fun read(input: DataInput): LookupSymbolKey {
        val version = input.readByte()
        return when (version.toInt()) {
@@ -51,14 +50,12 @@ object LookupSymbolKeyDescriptor : KeyDescriptor<LookupSymbolKey> {
                val second = input.readInt()
                LookupSymbolKey(first, second, "", "")
            }
-            else -> throw RuntimeException("Unknown version of LookupSymbolKeyDescriptor=${version}")
+            else -> throw IllegalArgumentException("Unknown version of LookupSymbolKeyDescriptor=${version}")
        }
    }

-    private val storeFullFqName = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient() ?: false
-
    override fun save(output: DataOutput, value: LookupSymbolKey) {
-        if (storeFullFqName) {
+        if (storeFullFqNames) {
            output.writeByte(0)
            output.writeUTF(value.name)
            output.writeUTF(value.scope)
@@ -74,18 +71,6 @@ object LookupSymbolKeyDescriptor : KeyDescriptor<LookupSymbolKey> {
    override fun isEqual(val1: LookupSymbolKey, val2: LookupSymbolKey): Boolean = val1 == val2
 }

-object LookupSymbolExternalizer : DataExternalizer<LookupSymbol> {
-
-    override fun save(output: DataOutput, lookupSymbol: LookupSymbol) {
-        output.writeString(lookupSymbol.name)
-        output.writeString(lookupSymbol.scope)
-    }
-
-    override fun read(input: DataInput): LookupSymbol {
-        return LookupSymbol(name = input.readString(), scope = input.readString())
-    }
-}
-
 object FqNameExternalizer : DataExternalizer<FqName> {

    override fun save(output: DataOutput, fqName: FqName) {
@@ -226,6 +211,32 @@ object ConstantExternalizer : DataExternalizer<Any> {
    }
 }

+fun <T> DataExternalizer<T>.saveToFile(file: File, value: T) {
+    return DataOutputStream(FileOutputStream(file).buffered()).use {
+        save(it, value)
+    }
+}
+
+fun <T> DataExternalizer<T>.loadFromFile(file: File): T {
+    return DataInputStream(FileInputStream(file).buffered()).use {
+        read(it)
+    }
+}
+
+fun <T> DataExternalizer<T>.toByteArray(value: T): ByteArray {
+    val byteArrayOutputStream = ByteArrayOutputStream()
+    DataOutputStream(byteArrayOutputStream.buffered()).use {
+        save(it, value)
+    }
+    return byteArrayOutputStream.toByteArray()
+}
+
+fun <T> DataExternalizer<T>.fromByteArray(byteArray: ByteArray): T {
+    return DataInputStream(ByteArrayInputStream(byteArray).buffered()).use {
+        read(it)
+    }
+}
+
 object IntExternalizer : DataExternalizer<Int> {
    override fun save(output: DataOutput, value: Int) = output.writeInt(value)
    override fun read(input: DataInput): Int = input.readInt()
@@ -7,6 +7,8 @@ package org.jetbrains.kotlin.incremental.storage

 import com.intellij.util.containers.MultiMap
 import org.jetbrains.kotlin.TestWithWorkingDir
+import org.jetbrains.kotlin.cli.common.CompilerSystemProperties
+import org.jetbrains.kotlin.cli.common.toBooleanLenient
 import org.jetbrains.kotlin.incremental.LookupStorage
 import org.jetbrains.kotlin.incremental.LookupSymbol
 import org.jetbrains.kotlin.incremental.testingUtils.assertEqualDirectories
@@ -48,7 +50,11 @@ class RelocatableCachesTest : TestWithWorkingDir() {
    private fun fillLookupStorage(projectRoot: File, reverseFiles: Boolean, reverseLookups: Boolean) {
        val storageRoot = projectRoot.storageRoot
        val fileToPathConverter = RelativeFileToPathConverter(projectRoot)
-        val lookupStorage = LookupStorage(storageRoot, fileToPathConverter)
+        val lookupStorage = LookupStorage(
+            storageRoot,
+            fileToPathConverter,
+            storeFullFqNames = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient() ?: false
+        )
        val files = LinkedHashSet<String>()
        val symbols = LinkedHashSet<LookupSymbol>()
        val lookups = MultiMap.createOrderedSet<LookupSymbol, String>()