diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt index 7a17eecc12d..9115e6daa35 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildAttribute.kt @@ -18,6 +18,7 @@ enum class BuildAttributeKind : Serializable { enum class BuildAttribute(val kind: BuildAttributeKind, val readableString: String) : Serializable { NO_BUILD_HISTORY(BuildAttributeKind.REBUILD_REASON, "Build history file not found"), NO_ABI_SNAPSHOT(BuildAttributeKind.REBUILD_REASON, "ABI snapshot not found"), + CLASSPATH_SNAPSHOT_NOT_FOUND(BuildAttributeKind.REBUILD_REASON, "Classpath snapshot not found"), CACHE_CORRUPTION(BuildAttributeKind.REBUILD_REASON, "Cache corrupted"), UNKNOWN_CHANGES_IN_GRADLE_INPUTS(BuildAttributeKind.REBUILD_REASON, "Unknown Gradle changes"), JAVA_CHANGE_UNTRACKED_FILE_IS_REMOVED(BuildAttributeKind.REBUILD_REASON, "Untracked Java file is removed"), diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt index 909b2586398..3f72a048ac4 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt @@ -10,9 +10,13 @@ import java.io.Serializable @Suppress("Reformat") enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, val readableString: String) : Serializable { - OUTPUT_SIZE(readableString = "Total output size"), - LOOKUP_SIZE(OUTPUT_SIZE, "Lookups size"), - SNAPSHOT_SIZE(OUTPUT_SIZE, "ABI snapshot size"), + CACHE_DIRECTORY_SIZE(readableString = "Total size of the cache directory"), + LOOKUP_SIZE(CACHE_DIRECTORY_SIZE, "Lookups size"), + SNAPSHOT_SIZE(CACHE_DIRECTORY_SIZE, "ABI snapshot size"), + + // Metrics for the `kotlin.incremental.useClasspathSnapshot` feature + ORIGINAL_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the original classpath snapshot (before shrinking)"), + SHRUNK_CLASSPATH_SNAPSHOT_SIZE(parent = null, "Size of the shrunk classpath snapshot"), ; companion object { diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt index 23f4ff420aa..89aa290e2bc 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt @@ -27,6 +27,15 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String) SET_UP_ABI_SNAPSHOTS(JAR_SNAPSHOT, "Set up ABI snapshot"), IC_ANALYZE_JAR_FILES(JAR_SNAPSHOT, "Analyze jar files"), IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION, "Init dirty symbols set"), + COMPUTE_CLASSPATH_CHANGES(IC_CALCULATE_INITIAL_DIRTY_SET, "Compute classpath changes"), + LOAD_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load current classpath snapshot"), + SHRINK_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Shrink current classpath snapshot"), + LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load shrunk previous classpath snapshot"), + COMPUTE_CHANGED_AND_IMPACTED_SET(COMPUTE_CLASSPATH_CHANGES, "Compute changed and impacted set"), + COMPUTE_CLASS_CHANGES(COMPUTE_CHANGED_AND_IMPACTED_SET, "Compute class changes"), + COMPUTE_KOTLIN_CLASS_CHANGES(COMPUTE_CLASS_CHANGES, "Compute Kotlin class changes"), + COMPUTE_JAVA_CLASS_CHANGES(COMPUTE_CLASS_CHANGES, "Compute Java class changes"), + COMPUTE_IMPACTED_SET(COMPUTE_CHANGED_AND_IMPACTED_SET, "Compute impacted set"), IC_ANALYZE_CHANGES_IN_DEPENDENCIES(IC_CALCULATE_INITIAL_DIRTY_SET, "Analyze dependency changes"), IC_FIND_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES, "Find history files"), IC_ANALYZE_HISTORY_FILES(IC_ANALYZE_CHANGES_IN_DEPENDENCIES, "Analyze history files"), @@ -38,6 +47,14 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String) INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Incremental iteration"), NON_INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Non-incremental iteration"), IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION, "Write history file"), + SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION, "Save shrunk current classpath snapshot after compilation"), + LOAD_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load classpath snapshot"), + SHRINK_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink classpath snapshot"), + GET_NON_DUPLICATE_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Get non-duplicate classes"), + GET_LOOKUP_SYMBOLS(SHRINK_CLASSPATH_SNAPSHOT, "Get lookup symbols"), + FIND_REFERENCED_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Find referenced classes"), + FIND_TRANSITIVELY_REFERENCED_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Find transitively referenced classes"), + SAVE_SHRUNK_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk classpath snapshot"), COMPILER_PERFORMANCE(readableString = "Compiler time"), COMPILER_INITIALIZATION(COMPILER_PERFORMANCE, "Compiler initialization time"), CODE_ANALYSIS(COMPILER_PERFORMANCE, "Compiler code analyse"), diff --git a/build-common/src/org/jetbrains/kotlin/incremental/ClasspathChanges.kt b/build-common/src/org/jetbrains/kotlin/incremental/ClasspathChanges.kt index bac6c23fb43..5a78b8034f6 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/ClasspathChanges.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/ClasspathChanges.kt @@ -5,70 +5,43 @@ package org.jetbrains.kotlin.incremental -import com.intellij.util.io.DataExternalizer -import org.jetbrains.kotlin.incremental.storage.FqNameExternalizer -import org.jetbrains.kotlin.incremental.storage.SetExternalizer -import org.jetbrains.kotlin.incremental.storage.LookupSymbolExternalizer -import org.jetbrains.kotlin.name.FqName -import java.io.* +import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled +import java.io.File +import java.io.Serializable /** - * Changes to the classpath of the `KotlinCompile` task, used to compute the source files that need to be recompiled during an incremental - * run. + * Changes to the classpath of the `KotlinCompile` task, or information to compute them later by the Kotlin incremental compiler (see + * [ClasspathSnapshotEnabled.ToBeComputedByIncrementalCompiler]. */ sealed class ClasspathChanges : Serializable { - class Available() : ClasspathChanges() { + sealed class ClasspathSnapshotEnabled : ClasspathChanges() { - companion object { - private const val serialVersionUID = 0L - } + abstract val classpathSnapshotFiles: ClasspathSnapshotFiles - lateinit var lookupSymbols: Set // Preferably ordered but not required - private set + class Empty(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled() - lateinit var fqNames: Set // Preferably ordered but not required - private set + class ToBeComputedByIncrementalCompiler(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled() - constructor(lookupSymbols: Set, fqNames: Set) : this() { - this.lookupSymbols = lookupSymbols - this.fqNames = fqNames - } + class NotAvailableDueToMissingClasspathSnapshot(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : + ClasspathSnapshotEnabled() - private fun writeObject(output: ObjectOutputStream) { - // Can't close DataOutputStream below as it will also close the underlying ObjectOutputStream, which is still in use. - ClasspathChangesAvailableExternalizer.save(DataOutputStream(output), this) - } - - private fun readObject(input: ObjectInputStream) { - // Can't close DataInputStream below as it will also close the underlying ObjectInputStream, which is still in use. - ClasspathChangesAvailableExternalizer.read(DataInputStream(input)).also { - lookupSymbols = it.lookupSymbols - fqNames = it.fqNames - } - } + class NotAvailableForNonIncrementalRun(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled() } - sealed class NotAvailable : ClasspathChanges() { - object UnableToCompute : NotAvailable() - object ForNonIncrementalRun : NotAvailable() - object ClasspathSnapshotIsDisabled : NotAvailable() - object ReservedForTestsOnly : NotAvailable() - object ForJSCompiler : NotAvailable() - } + object ClasspathSnapshotDisabled : ClasspathChanges() + + object NotAvailableForJSCompiler : ClasspathChanges() } -private object ClasspathChangesAvailableExternalizer : DataExternalizer { +class ClasspathSnapshotFiles( + val currentClasspathEntrySnapshotFiles: List, + classpathSnapshotDir: File +) : Serializable { - override fun save(output: DataOutput, classpathChanges: ClasspathChanges.Available) { - SetExternalizer(LookupSymbolExternalizer).save(output, classpathChanges.lookupSymbols) - SetExternalizer(FqNameExternalizer).save(output, classpathChanges.fqNames) - } + val shrunkPreviousClasspathSnapshotFile: File = File(classpathSnapshotDir, "shrunk-classpath-snapshot.bin") - override fun read(input: DataInput): ClasspathChanges.Available { - return ClasspathChanges.Available( - lookupSymbols = SetExternalizer(LookupSymbolExternalizer).read(input), - fqNames = SetExternalizer(FqNameExternalizer).read(input) - ) + companion object { + private const val serialVersionUID = 0L } } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt index c8b8c8a3dba..ff6f37b4fb6 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt @@ -32,7 +32,8 @@ import java.util.* open class LookupStorage( targetDataDir: File, - pathConverter: FileToPathConverter + pathConverter: FileToPathConverter, + storeFullFqNames: Boolean = false ) : BasicMapsOwner(targetDataDir) { val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder") @@ -44,7 +45,7 @@ open class LookupStorage( private val countersFile = "counters".storageFile private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, pathConverter)) private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, pathConverter)) - val lookupMap = registerMap(LookupMap("lookups".storageFile)) + val lookupMap = registerMap(LookupMap("lookups".storageFile, storeFullFqNames)) @Volatile private var size: Int = 0 diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt index bf612acb13b..9ac0720ba73 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt @@ -18,7 +18,9 @@ package org.jetbrains.kotlin.incremental.storage import java.io.File -class LookupMap(storage: File) : BasicMap>(storage, LookupSymbolKeyDescriptor, IntCollectionExternalizer) { +class LookupMap(storage: File, storeFullFqNames: Boolean) : + BasicMap>(storage, LookupSymbolKeyDescriptor(storeFullFqNames), IntCollectionExternalizer) { + override fun dumpKey(key: LookupSymbolKey): String = key.toString() override fun dumpValue(value: Collection): String = value.toString() diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt index 4c4c637d7a6..6b33e124bc6 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt @@ -22,22 +22,21 @@ import com.intellij.util.io.DataExternalizer import com.intellij.util.io.EnumeratorStringDescriptor import com.intellij.util.io.IOUtil import com.intellij.util.io.KeyDescriptor -import org.jetbrains.kotlin.cli.common.CompilerSystemProperties -import org.jetbrains.kotlin.cli.common.toBooleanLenient -import org.jetbrains.kotlin.incremental.LookupSymbol import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.resolve.jvm.JvmClassName -import java.io.DataInput -import java.io.DataInputStream -import java.io.DataOutput +import java.io.* /** * Storage versioning: * 0 - only name and value hashes are saved * 1 - name and scope are saved */ -object LookupSymbolKeyDescriptor : KeyDescriptor { +class LookupSymbolKeyDescriptor( + /** If `true`, original values are saved; if `false`, only hashes are saved. */ + private val storeFullFqNames: Boolean = false +) : KeyDescriptor { + override fun read(input: DataInput): LookupSymbolKey { val version = input.readByte() return when (version.toInt()) { @@ -51,14 +50,12 @@ object LookupSymbolKeyDescriptor : KeyDescriptor { val second = input.readInt() LookupSymbolKey(first, second, "", "") } - else -> throw RuntimeException("Unknown version of LookupSymbolKeyDescriptor=${version}") + else -> throw IllegalArgumentException("Unknown version of LookupSymbolKeyDescriptor=${version}") } } - private val storeFullFqName = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient() ?: false - override fun save(output: DataOutput, value: LookupSymbolKey) { - if (storeFullFqName) { + if (storeFullFqNames) { output.writeByte(0) output.writeUTF(value.name) output.writeUTF(value.scope) @@ -74,18 +71,6 @@ object LookupSymbolKeyDescriptor : KeyDescriptor { override fun isEqual(val1: LookupSymbolKey, val2: LookupSymbolKey): Boolean = val1 == val2 } -object LookupSymbolExternalizer : DataExternalizer { - - override fun save(output: DataOutput, lookupSymbol: LookupSymbol) { - output.writeString(lookupSymbol.name) - output.writeString(lookupSymbol.scope) - } - - override fun read(input: DataInput): LookupSymbol { - return LookupSymbol(name = input.readString(), scope = input.readString()) - } -} - object FqNameExternalizer : DataExternalizer { override fun save(output: DataOutput, fqName: FqName) { @@ -226,6 +211,32 @@ object ConstantExternalizer : DataExternalizer { } } +fun DataExternalizer.saveToFile(file: File, value: T) { + return DataOutputStream(FileOutputStream(file).buffered()).use { + save(it, value) + } +} + +fun DataExternalizer.loadFromFile(file: File): T { + return DataInputStream(FileInputStream(file).buffered()).use { + read(it) + } +} + +fun DataExternalizer.toByteArray(value: T): ByteArray { + val byteArrayOutputStream = ByteArrayOutputStream() + DataOutputStream(byteArrayOutputStream.buffered()).use { + save(it, value) + } + return byteArrayOutputStream.toByteArray() +} + +fun DataExternalizer.fromByteArray(byteArray: ByteArray): T { + return DataInputStream(ByteArrayInputStream(byteArray).buffered()).use { + read(it) + } +} + object IntExternalizer : DataExternalizer { override fun save(output: DataOutput, value: Int) = output.writeInt(value) override fun read(input: DataInput): Int = input.readInt() diff --git a/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt b/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt index 30a72275506..bb52e8933ef 100644 --- a/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt +++ b/build-common/test/org/jetbrains/kotlin/incremental/storage/RelocatableCachesTest.kt @@ -7,6 +7,8 @@ package org.jetbrains.kotlin.incremental.storage import com.intellij.util.containers.MultiMap import org.jetbrains.kotlin.TestWithWorkingDir +import org.jetbrains.kotlin.cli.common.CompilerSystemProperties +import org.jetbrains.kotlin.cli.common.toBooleanLenient import org.jetbrains.kotlin.incremental.LookupStorage import org.jetbrains.kotlin.incremental.LookupSymbol import org.jetbrains.kotlin.incremental.testingUtils.assertEqualDirectories @@ -48,7 +50,11 @@ class RelocatableCachesTest : TestWithWorkingDir() { private fun fillLookupStorage(projectRoot: File, reverseFiles: Boolean, reverseLookups: Boolean) { val storageRoot = projectRoot.storageRoot val fileToPathConverter = RelativeFileToPathConverter(projectRoot) - val lookupStorage = LookupStorage(storageRoot, fileToPathConverter) + val lookupStorage = LookupStorage( + storageRoot, + fileToPathConverter, + storeFullFqNames = CompilerSystemProperties.COMPILE_INCREMENTAL_WITH_CLASSPATH_SNAPSHOTS.value.toBooleanLenient() ?: false + ) val files = LinkedHashSet() val symbols = LinkedHashSet() val lookups = MultiMap.createOrderedSet() diff --git a/compiler/incremental-compilation-impl/build.gradle.kts b/compiler/incremental-compilation-impl/build.gradle.kts index db2680b86be..04f8ac84655 100644 --- a/compiler/incremental-compilation-impl/build.gradle.kts +++ b/compiler/incremental-compilation-impl/build.gradle.kts @@ -15,6 +15,7 @@ dependencies { api(project(":compiler:cli-js")) api(project(":kotlin-build-common")) api(project(":daemon-common")) + implementation("com.google.code.gson:gson:${rootProject.extra["versions.jar.gson"]}") compileOnly(intellijCoreDep()) { includeJars("intellij-core") } testApi(commonDep("junit:junit")) diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt index 73f5f8c56f9..c5097e348fe 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt @@ -26,8 +26,8 @@ import java.io.File abstract class IncrementalCachesManager>( cachesRootDir: File, rootProjectDir: File?, - protected val reporter: ICReporter -) { + protected val reporter: ICReporter, + storeFullFqNamesInLookupCache: Boolean = false) { val pathConverter = IncrementalFileToPathConverter(rootProjectDir) private val caches = arrayListOf() @@ -43,7 +43,7 @@ abstract class IncrementalCachesManager(cacheDirectory, rootProjectDir, reporter) { + reporter: ICReporter, + storeFullFqNamesInLookupCache: Boolean = false + ) : IncrementalCachesManager(cacheDirectory, rootProjectDir, reporter, storeFullFqNamesInLookupCache) { private val jvmCacheDir = File(cacheDirectory, "jvm").apply { mkdirs() } override val platformCache = IncrementalJvmCache(jvmCacheDir, outputDir, pathConverter).apply { registerCache() } } @@ -89,8 +90,9 @@ class IncrementalJsCachesManager( cachesRootDir: File, rootProjectDir: File?, reporter: ICReporter, - serializerProtocol: SerializerExtensionProtocol -) : IncrementalCachesManager(cachesRootDir, rootProjectDir, reporter) { + serializerProtocol: SerializerExtensionProtocol, + storeFullFqNamesInLookupCache: Boolean +) : IncrementalCachesManager(cachesRootDir, rootProjectDir, reporter, storeFullFqNamesInLookupCache) { private val jsCacheFile = File(cachesRootDir, "js").apply { mkdirs() } override val platformCache = IncrementalJsCache(jsCacheFile, pathConverter, serializerProtocol).apply { registerCache() } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index 8bfa97c3e24..c071a5cc5e6 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -19,9 +19,9 @@ package org.jetbrains.kotlin.incremental import org.jetbrains.kotlin.build.DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS import org.jetbrains.kotlin.build.GeneratedFile import org.jetbrains.kotlin.build.report.BuildReporter -import org.jetbrains.kotlin.build.report.metrics.BuildTime import org.jetbrains.kotlin.build.report.metrics.BuildAttribute import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric +import org.jetbrains.kotlin.build.report.metrics.BuildTime import org.jetbrains.kotlin.build.report.metrics.measure import org.jetbrains.kotlin.cli.common.* import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments @@ -39,8 +39,6 @@ import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.progress.CompilationCanceledStatus import java.io.File import java.io.IOException -import java.util.* -import kotlin.collections.HashMap abstract class IncrementalCompilerRunner< Args : CommonCompilerArguments, @@ -108,7 +106,7 @@ abstract class IncrementalCompilerRunner< caches.close(false) // todo: we can recompile all files incrementally (not cleaning caches), so rebuild won't propagate reporter.measure(BuildTime.CLEAR_OUTPUT_ON_REBUILD) { - clearLocalStateOnRebuild(args) + clearOutputsOnRebuild(args) } caches = createCacheManager(args, projectDir) if (providedChangedFiles == null) { @@ -134,7 +132,6 @@ abstract class IncrementalCompilerRunner< else -> providedChangedFiles } - val compilationMode = sourcesToCompile(caches, changedFiles, args, messageCollector, classpathAbiSnapshot) val exitCode = when (compilationMode) { @@ -162,7 +159,8 @@ abstract class IncrementalCompilerRunner< allSourceFiles, compilationMode, messageCollector, - withSnapshot) + withSnapshot + ) } } is CompilationMode.Rebuild -> { @@ -170,6 +168,8 @@ abstract class IncrementalCompilerRunner< } } + performWorkAfterCompilation(caches) + if (!caches.close(flush = true)) throw RuntimeException("Could not flush caches") // Here we should analyze exit code of compiler. E.g. compiler failure should lead to caches rebuild, // but now JsKlib compiler reports invalid exit code. @@ -182,7 +182,7 @@ abstract class IncrementalCompilerRunner< ) if (cacheDirectory.exists() && cacheDirectory.isDirectory()) { cacheDirectory.walkTopDown().filter { it.isFile }.map { it.length() }.sum().let { - reporter.addMetric(BuildPerformanceMetric.OUTPUT_SIZE, it) + reporter.addMetric(BuildPerformanceMetric.CACHE_DIRECTORY_SIZE, it) } } } @@ -193,37 +193,42 @@ abstract class IncrementalCompilerRunner< rebuild(BuildAttribute.CACHE_CORRUPTION) } finally { if (cachesMayBeCorrupted) { - clearLocalStateOnRebuild(args) + clearOutputsOnRebuild(args) } } } - private fun clearLocalStateOnRebuild(args: Args) { + /** + * Deletes output files and contents of output directories on rebuild (including `@LocalState` files/directories). + * + * If the output directories do not yet exist, they will be created. + */ + private fun clearOutputsOnRebuild(args: Args) { val destinationDir = destinationDir(args) + val (outputDirsThatExist, regularOrNonExistentOutputFiles) = outputFiles.partition { it.isDirectory } + val regularOutputFiles = regularOrNonExistentOutputFiles.filter { it.exists() } - reporter.reportVerbose { "Clearing output on rebuild" } - for (file in sequenceOf(destinationDir, workingDir) + outputFiles.asSequence()) { - val deleted: Boolean? = when { - file.isDirectory -> { - reporter.reportVerbose { " Deleting directory $file" } - file.deleteRecursively() - } - file.isFile -> { - reporter.reportVerbose { " Deleting $file" } - file.delete() - } - else -> null + // outputDirsThatExist may or may not contain destinationDir and workingDir. + // Collect all of them so that we don't miss any output directories. + // Use Set to avoid duplication. + val allOutputDirs = setOf(destinationDir, workingDir) + outputDirsThatExist + + reporter.reportVerbose { "Clearing outputs on rebuild" } + allOutputDirs.forEach { dir -> + reporter.reportVerbose { " Deleting contents of directory '${dir.path}'" } + dir.listFiles()?.forEach { + it.deleteRecursively() + if (it.exists()) throw IOException("Could not delete '${it.path}'") } - if (deleted == false) { - reporter.reportVerbose { " Could not delete $file" } - } + dir.mkdirs() + if (!dir.exists()) throw IOException("Could not create directory '${dir.path}'") + } + regularOutputFiles.forEach { file -> + reporter.reportVerbose { " Deleting file '${file.path}'" } + file.delete() + if (file.exists()) throw IOException("Could not delete file '${file.path}'") } - - if (destinationDir.exists()) throw IOException("Could not delete directory $destinationDir.") - if (workingDir.exists()) throw IOException("Could not delete internal caches in folder $workingDir") - destinationDir.mkdirs() - workingDir.mkdirs() } private fun sourcesToCompile( @@ -393,9 +398,8 @@ abstract class IncrementalCompilerRunner< caches.platformCache.updateComplementaryFiles(dirtySources, expectActualTracker) caches.inputsCache.registerOutputForSourceFiles(generatedFiles) caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources) - updateCaches(services, caches, generatedFiles, changesCollector) - -} + updateCaches(services, caches, generatedFiles, changesCollector) + } if (compilationMode is CompilationMode.Rebuild) { if (withSnapshot) { abiSnapshot.protos.putAll(changesCollector.protoDataChanges()) @@ -498,6 +502,8 @@ abstract class IncrementalCompilerRunner< BuildDiffsStorage.writeToFile(buildHistoryFile, BuildDiffsStorage(prevDiffs + newDiff), reporter) } + protected open fun performWorkAfterCompilation(caches: CacheManager) {} + companion object { const val DIRTY_SOURCES_FILE_NAME = "dirty-sources.txt" const val LAST_BUILD_INFO_FILE_NAME = "last-build.bin" diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt index e70d620a036..e014983a068 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -20,9 +20,8 @@ import org.jetbrains.kotlin.build.GeneratedFile import org.jetbrains.kotlin.build.report.BuildReporter import org.jetbrains.kotlin.build.report.ICReporter import org.jetbrains.kotlin.build.report.metrics.BuildAttribute -import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter -import org.jetbrains.kotlin.cli.common.* +import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments import org.jetbrains.kotlin.cli.common.arguments.isIrBackendEnabled import org.jetbrains.kotlin.cli.common.messages.MessageCollector @@ -92,7 +91,13 @@ class IncrementalJsCompilerRunner( override fun createCacheManager(args: K2JSCompilerArguments, projectDir: File?): IncrementalJsCachesManager { val serializerProtocol = if (!args.isIrBackendEnabled()) JsSerializerProtocol else KlibMetadataSerializerProtocol - return IncrementalJsCachesManager(cacheDirectory, projectDir, reporter, serializerProtocol) + return IncrementalJsCachesManager( + cacheDirectory, + projectDir, + reporter, + serializerProtocol, + storeFullFqNamesInLookupCache = withSnapshot + ) } override fun destinationDir(args: K2JSCompilerArguments): File { diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index 6e6872c58a4..074f4500711 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -41,17 +41,18 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.IncrementalCompilation import org.jetbrains.kotlin.config.Services +import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotDisabled +import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.* +import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailableForJSCompiler +import org.jetbrains.kotlin.incremental.classpathDiff.* import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory +import org.jetbrains.kotlin.incremental.storage.ListExternalizer +import org.jetbrains.kotlin.incremental.storage.saveToFile import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector import org.jetbrains.kotlin.incremental.util.Either -import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.UnableToCompute -import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ForJSCompiler -import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ReservedForTestsOnly -import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ForNonIncrementalRun -import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailable.ClasspathSnapshotIsDisabled import org.jetbrains.kotlin.load.java.JavaClassesTracker import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCompilationComponents @@ -88,7 +89,7 @@ fun makeIncrementally( buildHistoryFile = buildHistoryFile, modulesApiHistory = EmptyModulesApiHistory, kotlinSourceFilesExtensions = kotlinExtensions, - classpathChanges = ReservedForTestsOnly + classpathChanges = ClasspathSnapshotDisabled ) //TODO set properly compiler.compile(sourceFiles, args, messageCollector, providedChangedFiles = null) @@ -123,7 +124,7 @@ class IncrementalJvmCompilerRunner( outputFiles: Collection, private val modulesApiHistory: ModulesApiHistory, override val kotlinSourceFilesExtensions: List = DEFAULT_KOTLIN_SOURCE_FILES_EXTENSIONS, - private val classpathChanges: ClasspathChanges, + private val classpathChanges: ClasspathChanges ) : IncrementalCompilerRunner( workingDir, "caches-jvm", @@ -135,7 +136,13 @@ class IncrementalJvmCompilerRunner( IncrementalCompilation.isEnabledForJvm() override fun createCacheManager(args: K2JVMCompilerArguments, projectDir: File?): IncrementalJvmCachesManager = - IncrementalJvmCachesManager(cacheDirectory, projectDir, File(args.destination), reporter) + IncrementalJvmCachesManager( + cacheDirectory, + projectDir, + File(args.destination), + reporter, + storeFullFqNamesInLookupCache = withSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled + ) override fun destinationDir(args: K2JVMCompilerArguments): File = args.destinationAsFile @@ -203,6 +210,10 @@ class IncrementalJvmCompilerRunner( return abiSnapshots } + // Used by `calculateSourcesToCompileImpl` and `performWorkAfterCompilation` methods below. + // Thread safety: There is no concurrent access to this variable. + private var currentClasspathSnapshot: ClasspathSnapshot? = null + private fun calculateSourcesToCompileImpl( caches: IncrementalJvmCachesManager, changedFiles: ChangedFiles.Known, @@ -212,26 +223,34 @@ class IncrementalJvmCompilerRunner( val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions) initDirtyFiles(dirtyFiles, changedFiles) - val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild(BuildAttribute.NO_BUILD_HISTORY) - reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" } - val classpathChanges = when (classpathChanges) { // Note: classpathChanges is deserialized, so they are no longer singleton objects and need to be compared using `is` (not `==`) - is ClasspathChanges.Available -> ChangesEither.Known(classpathChanges.lookupSymbols, classpathChanges.fqNames) - is ClasspathChanges.NotAvailable -> when (classpathChanges) { - is UnableToCompute, is ClasspathSnapshotIsDisabled, is ReservedForTestsOnly -> { - reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) { - val scopes = caches.lookupCache.lookupMap.keys.map { it.scope.ifBlank { it.name } }.distinct() - getClasspathChanges( - args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter, abiSnapshots, withSnapshot, - caches.platformCache, scopes - ) - } - } - is ForNonIncrementalRun, is ForJSCompiler -> { - error("Unexpected type for this code path: ${classpathChanges.javaClass.name}.") + is Empty -> ChangesEither.Known(emptySet(), emptySet()) + is ToBeComputedByIncrementalCompiler -> reporter.measure(BuildTime.COMPUTE_CLASSPATH_CHANGES) { + check(currentClasspathSnapshot == null) + currentClasspathSnapshot = reporter.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT) { + CachedClasspathSnapshotSerializer.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) } + ClasspathChangesComputer.computeChangedAndImpactedSet( + currentClasspathSnapshot!!, + caches.lookupCache, + classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile, + reporter + ).getChanges() } + is NotAvailableDueToMissingClasspathSnapshot -> ChangesEither.Unknown(BuildAttribute.CLASSPATH_SNAPSHOT_NOT_FOUND) + is NotAvailableForNonIncrementalRun -> ChangesEither.Unknown(BuildAttribute.UNKNOWN_CHANGES_IN_GRADLE_INPUTS) + is ClasspathSnapshotDisabled -> reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) { + val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild(BuildAttribute.IC_IS_NOT_ENABLED) + reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" } + val scopes = caches.lookupCache.lookupMap.keys.map { it.scope.ifBlank { it.name } }.distinct() + + getClasspathChanges( + args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter, abiSnapshots, withSnapshot, + caches.platformCache, scopes + ) + } + is NotAvailableForJSCompiler -> error("Unexpected type for this code path: ${classpathChanges.javaClass.name}.") } @Suppress("UNUSED_VARIABLE") // for sealed when @@ -434,6 +453,38 @@ class IncrementalJvmCompilerRunner( reportPerformanceData(compiler.defaultPerformanceManager) return exitCode } + + override fun performWorkAfterCompilation(caches: IncrementalJvmCachesManager) { + if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) { + reporter.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + shrinkAndSaveClasspathSnapshot(classpathChanges, caches.lookupCache) + } + } + } + + private fun shrinkAndSaveClasspathSnapshot(classpathChanges: ClasspathChanges.ClasspathSnapshotEnabled, lookupStorage: LookupStorage) { + val classpathSnapshot = currentClasspathSnapshot ?: reporter.measure(BuildTime.LOAD_CLASSPATH_SNAPSHOT) { + CachedClasspathSnapshotSerializer.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) + } + val shrunkClasspathSnapshot = reporter.measure(BuildTime.SHRINK_CLASSPATH_SNAPSHOT) { + ClasspathSnapshotShrinker.shrink(classpathSnapshot, lookupStorage, reporter) + } + reporter.measure(BuildTime.SAVE_SHRUNK_CLASSPATH_SNAPSHOT) { + ListExternalizer(ClassSnapshotWithHashExternalizer).saveToFile( + classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile, + shrunkClasspathSnapshot + ) + } + + reporter.addMetric( + BuildPerformanceMetric.ORIGINAL_CLASSPATH_SNAPSHOT_SIZE, + classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.sumOf { it.length() } + ) + reporter.addMetric( + BuildPerformanceMetric.SHRUNK_CLASSPATH_SNAPSHOT_SIZE, + classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.length() + ) + } } var K2JVMCompilerArguments.destinationAsFile: File diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ChangeSet.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ChangeSet.kt index 4a3b81260a5..861865b0832 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ChangeSet.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ChangeSet.kt @@ -5,12 +5,12 @@ package org.jetbrains.kotlin.incremental.classpathDiff -import org.jetbrains.kotlin.incremental.ClasspathChanges +import org.jetbrains.kotlin.incremental.ChangesEither import org.jetbrains.kotlin.incremental.LookupSymbol import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName -/** Intermediate data to compute [ClasspathChanges] (see [toClasspathChanges]). */ +/** Set of classes, class members, and top-level members that are changed (or impacted by a change). */ class ChangeSet( /** Set of changed classes, preferably ordered by not required. */ @@ -73,7 +73,7 @@ class ChangeSet( changedTopLevelMembers + other.changedTopLevelMembers ) - fun toClasspathChanges(): ClasspathChanges.Available { + internal fun getChanges(): ChangesEither.Known { val lookupSymbols = mutableSetOf() val fqNames = mutableSetOf() @@ -98,6 +98,6 @@ class ChangeSet( fqNames.add(changedPackage) } - return ClasspathChanges.Available(lookupSymbols, fqNames) + return ChangesEither.Known(lookupSymbols, fqNames) } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt index 8806e307337..01552b83689 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt @@ -6,9 +6,14 @@ package org.jetbrains.kotlin.incremental.classpathDiff import com.intellij.openapi.util.io.FileUtil -import org.jetbrains.kotlin.incremental.classpathDiff.ImpactAnalysis.computeImpactedSet +import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter +import org.jetbrains.kotlin.build.report.metrics.BuildTime +import org.jetbrains.kotlin.build.report.metrics.measure import org.jetbrains.kotlin.incremental.* +import org.jetbrains.kotlin.incremental.classpathDiff.ImpactAnalysis.computeImpactedSet import org.jetbrains.kotlin.incremental.storage.FileToCanonicalPathConverter +import org.jetbrains.kotlin.incremental.storage.ListExternalizer +import org.jetbrains.kotlin.incremental.storage.loadFromFile import org.jetbrains.kotlin.metadata.deserialization.TypeTable import org.jetbrains.kotlin.metadata.deserialization.supertypes import org.jetbrains.kotlin.name.ClassId @@ -18,156 +23,101 @@ import org.jetbrains.kotlin.serialization.deserialization.getClassId import java.io.File import java.util.* -/** Computes [ClasspathChanges] between two [ClasspathSnapshot]s .*/ +/** Computes changes between two [ClasspathSnapshot]s .*/ object ClasspathChangesComputer { - fun compute( - currentClasspathEntrySnapshotFiles: List, - previousClasspathEntrySnapshotFiles: List, - unchangedCurrentClasspathEntrySnapshotFiles: List - ): ClasspathChanges { - // To improve performance, we will compute changes for the changed snapshot files only, ignoring unchanged ones. (Duplicate classes - // will make this a bit tricky, but it will be dealt with below.) - // First, align unchanged snapshot files in the current classpath with unchanged snapshot files in the previous classpath. Gradle - // has this information, but doesn't expose it, so we have to reconstruct it here. - val unchangedCurrentToPreviousAlignment: Map = - alignUnchangedSnapshotFiles(unchangedCurrentClasspathEntrySnapshotFiles, previousClasspathEntrySnapshotFiles) - - // Use sets to make presence checks faster - val unchangedCurrentFiles: Set = unchangedCurrentToPreviousAlignment.keys - val unchangedPreviousFiles: Set = unchangedCurrentToPreviousAlignment.values.toSet() - - // We will split the current files into 2 groups: - // 1a) Unchanged current files - // 1b) Added files - // We will split the previous files into 2 groups: - // 2a) Unchanged previous files - // 2b) Removed files - // If the classpath doesn't contain duplicate classes, comparing (1b) with (2b) would be enough. - // However, if the classpath contains duplicate classes, comparing (1b) with (2b) would not be enough. - // Therefore, to deal with duplicate classes while still being able to compare (1b) with (2b), we will find snapshot files in groups - // (1a) and (2a) that have duplicate classes with groups (1b) or (2b) and add them to groups (1b) and (2b). Duplicate classes in - // groups (1b) and (2b) will then be handled in a separate step (see ClasspathChangesComputer.getNonDuplicateClassSnapshots). - val addedFiles: List = currentClasspathEntrySnapshotFiles.filter { it !in unchangedCurrentFiles } - val removedFiles: List = previousClasspathEntrySnapshotFiles.filter { it !in unchangedPreviousFiles } - - val adjustedAddedFiles = addedFiles.toMutableSet() - val adjustedRemovedFiles = removedFiles.toMutableSet() - unchangedCurrentToPreviousAlignment.forEach { (unchangedCurrentFile, unchangedPreviousFile) -> - if (unchangedCurrentFile.containsDuplicatesWith(addedFiles) || unchangedPreviousFile.containsDuplicatesWith(removedFiles)) { - adjustedAddedFiles.add(unchangedCurrentFile) - adjustedRemovedFiles.add(unchangedPreviousFile) - } + /** + * Computes changes between the current and previous [ClasspathSnapshot]s, plus unchanged elements that are impacted by the changes. + * + * NOTE: The original classpath may contain duplicate classes, but the shrunk classpath must not contain duplicate classes. + */ + fun computeChangedAndImpactedSet( + currentClasspathSnapshot: ClasspathSnapshot, + lookupStorageInPreviousRun: LookupStorage, + shrunkPreviousClasspathSnapshotFile: File, + metrics: BuildMetricsReporter + ): ChangeSet { + val shrunkCurrentClasspathSnapshot = metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT) { + ClasspathSnapshotShrinker.shrink(currentClasspathSnapshot, lookupStorageInPreviousRun, metrics) + } + val shrunkPreviousClasspathSnapshot = metrics.measure(BuildTime.LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT) { + ListExternalizer(ClassSnapshotWithHashExternalizer).loadFromFile(shrunkPreviousClasspathSnapshotFile) + } + return metrics.measure(BuildTime.COMPUTE_CHANGED_AND_IMPACTED_SET) { + computeChangedAndImpactedSet(shrunkCurrentClasspathSnapshot, shrunkPreviousClasspathSnapshot, metrics) } - - // Keep the original order of added/removed files as it is important for the handling of duplicate classes. - val finalAddedFiles: List = currentClasspathEntrySnapshotFiles.filter { it in adjustedAddedFiles } - val finalRemovedFiles: List = previousClasspathEntrySnapshotFiles.filter { it in adjustedRemovedFiles } - - val changedCurrentSnapshot = ClasspathSnapshotSerializer.load(finalAddedFiles) - val changedPreviousSnapshot = ClasspathSnapshotSerializer.load(finalRemovedFiles) - - return compute(changedCurrentSnapshot, changedPreviousSnapshot) } /** - * Maps the unchanged snapshot files of the current build to the unchanged snapshot files of the previous build (selected from all the - * snapshot files of the previous build). + * Computes changes between the current and previous lists of classes, plus unchanged elements that are impacted by the changes. * - * Note that the unchanged files of the current build were detected by Gradle, so a mapping must exist for each of them, we only have to - * find it. - * - * IMPORTANT: The alignment algorithm must use the same input normalization that is used for snapshot files in the Gradle task. - * Currently, snapshot files are annotated with `@Classpath` and are regular files (not jars), so (only) their contents and order - * matter. + * NOTE: Each list of classes must not contain duplicates. */ - private fun alignUnchangedSnapshotFiles(unchangedCurrentSnapshotFiles: List, previousSnapshotFiles: List): Map { - val sizeToPreviousFiles: Map>> = previousSnapshotFiles.withIndex().groupBy { it.value.length() } + fun computeChangedAndImpactedSet( + currentClassSnapshots: List, + previousClassSnapshots: List, + metrics: BuildMetricsReporter + ): ChangeSet { + val currentClasses: Map = currentClassSnapshots.associateBy { it.classSnapshot.getClassId() } + val previousClasses: Map = previousClassSnapshots.associateBy { it.classSnapshot.getClassId() } - var startIndexToSearch = 0 - return unchangedCurrentSnapshotFiles.associateWith { unchangedCurrentFile -> - val candidates = (sizeToPreviousFiles[unchangedCurrentFile.length()] ?: emptyList()).filter { it.index >= startIndexToSearch } - val unchangedPreviousFileWithIndex: IndexedValue = if (candidates.size == 1) { - // A matching file must exist, so if there is only one candidate, it is the one. - candidates.single() - } else { - // If there are multiple matching files, select the first one. (Even if it doesn't match Gradle's alignment, it is still a - // correct alignment.) - val unchangedContents = unchangedCurrentFile.readBytes() - candidates.firstOrNull { candidate -> - unchangedContents.contentEquals(candidate.value.readBytes()) - } ?: error("Can't find previous snapshot file of unchanged current snapshot file '${unchangedCurrentFile.path}'") - } - startIndexToSearch = unchangedPreviousFileWithIndex.index + 1 - unchangedPreviousFileWithIndex.value + val changedCurrentClasses: List = currentClasses.filter { (classId, currentClass) -> + val previousClass = previousClasses[classId] + previousClass == null || currentClass.hash != previousClass.hash + }.map { it.value.classSnapshot } + + val changedPreviousClasses: List = previousClasses.filter { (classId, previousClass) -> + val currentClass = currentClasses[classId] + currentClass == null || currentClass.hash != previousClass.hash + }.map { it.value.classSnapshot } + + val classChanges = metrics.measure(BuildTime.COMPUTE_CLASS_CHANGES) { + computeClassChanges(changedCurrentClasses, changedPreviousClasses, metrics) } - } - /** Returns `true` if this snapshot file contains a duplicate class with another snapshot file in the given list. */ - @Suppress("unused", "UNUSED_PARAMETER") - private fun File.containsDuplicatesWith(otherSnapshotFiles: List): Boolean { - // FIXME: Implement and optimize this method - return false - } + if (classChanges.isEmpty()) { + return classChanges + } - fun compute(currentClasspathSnapshot: ClasspathSnapshot, previousClasspathSnapshot: ClasspathSnapshot): ClasspathChanges { - val currentClassSnapshots = currentClasspathSnapshot.getNonDuplicateClassSnapshots() - val previousClassSnapshots = previousClasspathSnapshot.getNonDuplicateClassSnapshots() - - return computeClassChanges(currentClassSnapshots, previousClassSnapshots) + return metrics.measure(BuildTime.COMPUTE_IMPACTED_SET) { + computeImpactedSet(classChanges, previousClasses.map { it.value.classSnapshot }) + } } /** - * Returns all [ClassSnapshot]s in this [ClasspathSnapshot]. + * Computes changes between the current and previous lists of classes. The returned result does not need to include elements that are + * impacted by the changes. * - * If there are duplicate classes on the classpath, retain only the first one to match the compiler's behavior. + * NOTE: Each list of classes must not contain duplicates. */ - private fun ClasspathSnapshot.getNonDuplicateClassSnapshots(): List { - val classSnapshots = LinkedHashMap(classpathEntrySnapshots.sumOf { it.classSnapshots.size }) - for (classpathEntrySnapshot in classpathEntrySnapshots) { - for ((unixStyleRelativePath, classSnapshot) in classpathEntrySnapshot.classSnapshots) { - classSnapshots.putIfAbsent(unixStyleRelativePath, classSnapshot) + fun computeClassChanges( + currentClassSnapshots: List, + previousClassSnapshots: List, + metrics: BuildMetricsReporter + ): ChangeSet { + val asmBasedSnapshotPredicate: (ClassSnapshot) -> Boolean = { + when (it) { + is RegularJavaClassSnapshot -> true + is KotlinClassSnapshot, is ProtoBasedJavaClassSnapshot -> false + else -> error("Unexpected type (it should have been handled earlier): ${it.javaClass.name}") } } - return classSnapshots.values.toList() - } + val (currentAsmBasedSnapshots, currentProtoBasedSnapshots) = currentClassSnapshots.partition(asmBasedSnapshotPredicate) + val (previousAsmBasedSnapshots, previousProtoBasedSnapshots) = previousClassSnapshots.partition(asmBasedSnapshotPredicate) - /** - * Computes changes between two lists of [ClassSnapshot]s. - * - * Each list must not contain duplicate classes. - */ - fun computeClassChanges(currentClassSnapshots: List, previousClassSnapshots: List): ClasspathChanges { - if (currentClassSnapshots.any { it is ContentHashJavaClassSnapshot } - || previousClassSnapshots.any { it is ContentHashJavaClassSnapshot }) { - return ClasspathChanges.NotAvailable.UnableToCompute + val kotlinClassChanges = metrics.measure(BuildTime.COMPUTE_KOTLIN_CLASS_CHANGES) { + computeChangesForProtoBasedSnapshots(currentProtoBasedSnapshots, previousProtoBasedSnapshots) } - // Ignore `EmptyJavaClassSnapshot`s as they don't impact the result - val currentNonEmptyClassSnapshots = currentClassSnapshots.filter { it !is EmptyJavaClassSnapshot } - val previousNonEmptyClassSnapshots = previousClassSnapshots.filter { it !is EmptyJavaClassSnapshot } - - val (currentAsmBasedSnapshots, currentProtoBasedSnapshots) = - currentNonEmptyClassSnapshots.partition { it is RegularJavaClassSnapshot } - val (previousAsmBasedSnapshots, previousProtoBasedSnapshots) = - previousNonEmptyClassSnapshots.partition { it is RegularJavaClassSnapshot } - - val changeSet1 = computeChangesForProtoBasedSnapshots(currentProtoBasedSnapshots, previousProtoBasedSnapshots) - @Suppress("UNCHECKED_CAST") - val changeSet2 = JavaClassChangesComputer.compute( - currentAsmBasedSnapshots as List, - previousAsmBasedSnapshots as List - ) - - val allChanges = changeSet1 + changeSet2 - if (allChanges.isEmpty()) { - return allChanges.toClasspathChanges() + val javaClassChanges = metrics.measure(BuildTime.COMPUTE_JAVA_CLASS_CHANGES) { + JavaClassChangesComputer.compute( + currentAsmBasedSnapshots as List, + previousAsmBasedSnapshots as List + ) } - val impactedSet = computeImpactedSet(allChanges, previousNonEmptyClassSnapshots) - - return impactedSet.toClasspathChanges() + return kotlinClassChanges + javaClassChanges } private fun computeChangesForProtoBasedSnapshots( @@ -203,7 +153,7 @@ object ClasspathChangesComputer { ) incrementalJvmCache.markDirty(JvmClassName.byClassId(previousSnapshot.serializedJavaClass.classId)) } - is RegularJavaClassSnapshot, is ContentHashJavaClassSnapshot, is EmptyJavaClassSnapshot -> { + is RegularJavaClassSnapshot, is EmptyJavaClassSnapshot, is ContentHashJavaClassSnapshot -> { error("Unexpected type (it should have been handled earlier): ${previousSnapshot.javaClass.name}") } } @@ -233,7 +183,7 @@ object ClasspathChangesComputer { collector = changesCollector ) } - is RegularJavaClassSnapshot, is ContentHashJavaClassSnapshot, is EmptyJavaClassSnapshot -> { + is RegularJavaClassSnapshot, is EmptyJavaClassSnapshot, is ContentHashJavaClassSnapshot -> { error("Unexpected type (it should have been handled earlier): ${currentSnapshot.javaClass.name}") } } @@ -255,7 +205,7 @@ object ClasspathChangesComputer { private fun DirtyData.normalize(currentClassSnapshots: List, previousClassSnapshots: List): ChangeSet { val allClassIds = currentClassSnapshots.map { it.getClassId() }.toSet() + previousClassSnapshots.map { it.getClassId() } - val fqNameToClassId = LinkedHashMap(allClassIds.size) + val fqNameToClassId = HashMap(allClassIds.size) allClassIds.forEach { classId -> val fqName = classId.asSingleFqName() check(!fqNameToClassId.contains(fqName)) { @@ -264,68 +214,70 @@ object ClasspathChangesComputer { fqNameToClassId[fqName] = classId } - return ChangeSet.Collector().run { + val changes = ChangeSet.Collector().run { dirtyLookupSymbols.forEach { - fqNameToClassId[FqName(it.scope)]?.let { classIdOfScope -> - // If scope is a class, lookup symbol is a class member and maybe inner class - fqNameToClassId[FqName("${it.scope}.${it.name}")]?.let { innerClass -> - addChangedClass(innerClass) - } ?: addChangedClassMember(classIdOfScope, it.name) - return@forEach + val lookupSymbolFqName = if (it.scope.isEmpty()) FqName(it.name) else FqName("${it.scope}.${it.name}") + val lookupSymbolClassId: ClassId? = fqNameToClassId[lookupSymbolFqName] + if (lookupSymbolClassId != null) { + addChangedClass(lookupSymbolClassId) + } else { + // When lookupSymbolClassId == null, it means that either (1) the LookupSymbol does not refer to a class (it refers to + // a class member or a package-level member), or (2) it refers to a class outside allClassIds. In the following, we + // assume that (1) is the case. + // (2) should typically never happen. In the case that it happens, we will collect incorrect changes, but it's + // acceptable to collect more changes than necessary; also, we do not need to collect changes outside allClassIds, so + // we're not collecting fewer changes than required. + val scopeClassId: ClassId? = fqNameToClassId[FqName(it.scope)] + if (scopeClassId != null) { + addChangedClassMember(scopeClassId, it.name) + } else { + // Similarly, when scopeClassId == null, we assume that LookupSymbol.scope does not refer to a class outside + // allClassIds. It means that the LookupSymbol does not refer to a class member. Therefore, it must refer to a + // package-level member. + addChangedTopLevelMember(FqName(it.scope), it.name) + } } - - // scope is a package, so changed symbol is a top-level member and maybe a class - val potentialClassFqName = if (it.scope.isEmpty()) FqName(it.name) else FqName("${it.scope}.${it.name}") - fqNameToClassId[potentialClassFqName]?.let { classId -> - // Lookup symbol is a class - addChangedClass(classId) - } ?: addChangedTopLevelMember(FqName(it.scope), it.name) } - val changes = getChanges() - - // dirtyClassesFqNames should be derived from dirtyLookupSymbols. Double-check that this is the case. - val changedFqNames: Set = - changes.changedClasses.map { it.asSingleFqName() }.toSet() + - changes.changedClassMembers.keys.map { it.asSingleFqName() } + - changes.changedTopLevelMembers.keys - check(dirtyClassesFqNames.toSet() == changedFqNames) { - "Two sets differ:\n" + - "dirtyClassesFqNames: $dirtyClassesFqNames\n" + - "changedFqNames: $changedFqNames" - } - changes + getChanges() } + + // DirtyData contains: + // 1. dirtyLookupSymbols => This contains all info we need (extracted above). + // 2. dirtyClassesFqNames => This should be derived from dirtyLookupSymbols. + // 3. dirtyClassesFqNamesForceRecompile => Should be irrelevant. + // Double-check that the assumption at bullet 2 above is correct. + val changedFqNames: Set = + changes.changedClasses.map { it.asSingleFqName() }.toSet() + + changes.changedClassMembers.keys.map { it.asSingleFqName() } + + changes.changedTopLevelMembers.keys + check(dirtyClassesFqNames.toSet() == changedFqNames) { + "Two sets differ:\n" + + "dirtyClassesFqNames: $dirtyClassesFqNames\n" + + "changedFqNames: $changedFqNames" + } + + return changes } } -private fun ClassSnapshot.getClassId(): ClassId { - return when (this) { - is KotlinClassSnapshot -> classInfo.classId - is RegularJavaClassSnapshot -> classId - is ProtoBasedJavaClassSnapshot -> serializedJavaClass.classId - is EmptyJavaClassSnapshot, is ContentHashJavaClassSnapshot -> { - error("Unexpected type (it should have been handled earlier): ${javaClass.name}") - } - } -} - -private object ImpactAnalysis { +internal object ImpactAnalysis { /** - * Computes the set of classes/class members that are impacted by the given changes. + * Computes the set of classes, class members, and top-level members that are impacted by the given changes. * - * For example, if a superclass has changed, any of its subclasses will be impacted even if it has not changed, and unchanged source + * For example, if a superclass has changed, any of its subclasses will be impacted even if it has not changed because unchanged source * files in the previous compilation that depended on the subclasses will need to be recompiled. * * The returned set is also a [ChangeSet], which includes the given changes plus the impacted ones. */ fun computeImpactedSet(changes: ChangeSet, previousClassSnapshots: List): ChangeSet { val classIdToSubclasses = getClassIdToSubclassesMap(previousClassSnapshots) + val impactedClassesResolver = { classId: ClassId -> classIdToSubclasses[classId] ?: emptySet() } return ChangeSet.Collector().run { - addChangedClasses(findSubclassesInclusive(changes.changedClasses, classIdToSubclasses)) + addChangedClasses(findImpactedClassesInclusive(changes.changedClasses, impactedClassesResolver)) for ((changedClass, changedClassMembers) in changes.changedClassMembers) { - findSubclassesInclusive(setOf(changedClass), classIdToSubclasses).forEach { + findImpactedClassesInclusive(setOf(changedClass), impactedClassesResolver).forEach { addChangedClassMembers(it, changedClassMembers) } } @@ -337,7 +289,7 @@ private object ImpactAnalysis { } private fun getClassIdToSubclassesMap(classSnapshots: List): Map> { - val classIds = classSnapshots.map { it.getClassId() } + val classIds: Set = classSnapshots.map { it.getClassId() }.toSet() // Use Set for presence check val classNameToClassId = classIds.associateBy { JvmClassName.byClassId(it) } val classNameToClassIdResolver = { className: JvmClassName -> classNameToClassId[className] } @@ -345,7 +297,7 @@ private object ImpactAnalysis { classSnapshots.forEach { classSnapshot -> val classId = classSnapshot.getClassId() classSnapshot.getSupertypes(classNameToClassIdResolver).forEach { supertype -> - // No need to collect supertypes outside the considered class snapshots (e.g., "java/lang/Object") + // No need to collect supertypes outside the given set of classes (e.g., "java/lang/Object") if (supertype in classIds) { classIdToSubclasses.computeIfAbsent(supertype) { mutableSetOf() }.add(classId) } @@ -354,38 +306,17 @@ private object ImpactAnalysis { return classIdToSubclasses } - private fun ClassSnapshot.getSupertypes(classIdResolver: (JvmClassName) -> ClassId?): List { - return when (this) { - is RegularJavaClassSnapshot -> supertypes.mapNotNull { - // The following call returns null if supertype is outside the considered class snapshots (e.g., "java/lang/Object"). - // Use `mapNotNull` as we don't need to collect those supertypes (see getClassIdToSubclassesMap). - classIdResolver.invoke(it) - } - is KotlinClassSnapshot -> supertypes.mapNotNull { - // Same as above - classIdResolver.invoke(it) - } - is ProtoBasedJavaClassSnapshot -> { - val (proto, nameResolver) = serializedJavaClass.toProtoData() - proto.supertypes(TypeTable(proto.typeTable)).map { nameResolver.getClassId(it.className) } - } - is EmptyJavaClassSnapshot, is ContentHashJavaClassSnapshot -> { - error("Unexpected type (it should have been handled earlier): ${javaClass.name}") - } - } - } - /** - * Finds direct and indirect subclasses of the given classes. The return set includes both the given classes and their direct and - * indirect subclasses. + * Finds directly and transitively impacted classes of the given classes. The return set includes both the given classes and the + * impacted classes. */ - private fun findSubclassesInclusive(classIds: Set, classIdsToSubclasses: Map>): Set { + fun findImpactedClassesInclusive(classIds: Set, impactedClassesResolver: (ClassId) -> Set): Set { val visitedClasses = mutableSetOf() val toVisitClasses = classIds.toMutableSet() while (toVisitClasses.isNotEmpty()) { val nextToVisit = mutableSetOf() toVisitClasses.forEach { - nextToVisit.addAll(classIdsToSubclasses[it] ?: emptyList()) + nextToVisit.addAll(impactedClassesResolver.invoke(it)) } visitedClasses.addAll(toVisitClasses) toVisitClasses.clear() @@ -394,3 +325,34 @@ private object ImpactAnalysis { return visitedClasses } } + +internal fun ClassSnapshot.getClassId(): ClassId { + return when (this) { + is KotlinClassSnapshot -> classInfo.classId + is RegularJavaClassSnapshot -> classId + is ProtoBasedJavaClassSnapshot -> serializedJavaClass.classId + is EmptyJavaClassSnapshot, is ContentHashJavaClassSnapshot -> { + error("Unexpected type (it should have been handled earlier): ${javaClass.name}") + } + } +} + +/** + * Returns the [ClassId]s of the supertypes of this class. + * + * @param classIdResolver Resolves the [ClassId] from the [JvmClassName] of a supertype. It may return null if the supertype is outside the + * considered set of classes (e.g., "java/lang/Object"). Those supertypes do not need to be included in the returned result. + */ +internal fun ClassSnapshot.getSupertypes(classIdResolver: (JvmClassName) -> ClassId?): List { + return when (this) { + is KotlinClassSnapshot -> supertypes.mapNotNull { classIdResolver.invoke(it) } + is RegularJavaClassSnapshot -> supertypes.mapNotNull { classIdResolver.invoke(it) } + is ProtoBasedJavaClassSnapshot -> { + val (proto, nameResolver) = serializedJavaClass.toProtoData() + proto.supertypes(TypeTable(proto.typeTable)).map { nameResolver.getClassId(it.className) } + } + is EmptyJavaClassSnapshot, is ContentHashJavaClassSnapshot -> { + error("Unexpected type (it should have been handled earlier): ${javaClass.name}") + } + } +} diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt index f2cbd018be3..ee019a0609a 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt @@ -7,6 +7,8 @@ package org.jetbrains.kotlin.incremental.classpathDiff import org.jetbrains.kotlin.incremental.KotlinClassInfo import org.jetbrains.kotlin.incremental.SerializedJavaClass +import org.jetbrains.kotlin.incremental.md5 +import org.jetbrains.kotlin.incremental.storage.toByteArray import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.resolve.jvm.JvmClassName @@ -25,7 +27,7 @@ class ClasspathEntrySnapshot( * Maps (Unix-style) relative paths of classes to their snapshots. The paths are relative to the containing classpath entry (directory * or jar). */ - val classSnapshots: LinkedHashMap + val classSnapshots: LinkedHashMap ) /** @@ -36,7 +38,14 @@ class ClasspathEntrySnapshot( * `KotlinCompile` task and the task needs to support compile avoidance. For example, this class should contain public method signatures, * and should not contain private method signatures, or method implementations. */ -sealed class ClassSnapshot +sealed class ClassSnapshot { + + /** Computes the hash of this [ClassSnapshot] and returns a [ClassSnapshotWithHash]. */ + fun addHash() = ClassSnapshotWithHash(this, ClassSnapshotExternalizer.toByteArray(this).md5()) +} + +/** Contains a [ClassSnapshot] and its hash. */ +class ClassSnapshotWithHash(val classSnapshot: ClassSnapshot, val hash: Long) /** [ClassSnapshot] of a Kotlin class. */ class KotlinClassSnapshot( diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotSerializer.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotSerializer.kt index 911a5a3ad0a..b8096afc78e 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotSerializer.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotSerializer.kt @@ -10,32 +10,59 @@ import org.jetbrains.kotlin.incremental.JavaClassProtoMapValueExternalizer import org.jetbrains.kotlin.incremental.KotlinClassInfo import org.jetbrains.kotlin.incremental.storage.* import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader -import java.io.* +import java.io.DataInput +import java.io.DataOutput +import java.io.File +import java.util.concurrent.ConcurrentHashMap /** Utility to serialize a [ClasspathSnapshot]. */ -object ClasspathSnapshotSerializer { +object CachedClasspathSnapshotSerializer { + private val cache = ConcurrentHashMap() + private const val RECOMMENDED_MAX_CACHE_SIZE = 100 fun load(classpathEntrySnapshotFiles: List): ClasspathSnapshot { - return ClasspathSnapshot(classpathEntrySnapshotFiles.map { - ClasspathEntrySnapshotSerializer.load(it) - }) + return ClasspathSnapshot(classpathEntrySnapshotFiles.map { snapshotFile -> + cache.computeIfAbsent(snapshotFile) { + ClasspathEntrySnapshotExternalizer.loadFromFile(it) + } + }).also { + handleCacheEviction(recentlyReferencedKeys = classpathEntrySnapshotFiles) + } + } + + private fun handleCacheEviction(recentlyReferencedKeys: List) { + if (cache.size > RECOMMENDED_MAX_CACHE_SIZE) { + // Remove old entries. + // Note: + // - The cache entries after eviction = recently-referenced entries + some other entries (so that + // size = RECOMMENDED_MAX_CACHE_SIZE) + // + Removed entries don't have to be the oldest (for simplicity). + // + If recentlyReferencedKeys.size > RECOMMENDED_MAX_CACHE_SIZE, all of them will be kept. The reason is that + // recently-referenced entries will likely be used again, so we keep them even if the cache is larger than recommended. + // - It's okay to have race condition in this method. + val oldKeys = cache.keys - recentlyReferencedKeys.toSet() + for (oldKey in oldKeys) { + cache.remove(oldKey) + if (cache.size <= RECOMMENDED_MAX_CACHE_SIZE) break + } + } } } -object ClasspathEntrySnapshotSerializer : DataSerializer { +object ClasspathEntrySnapshotExternalizer : DataExternalizer { override fun save(output: DataOutput, snapshot: ClasspathEntrySnapshot) { - LinkedHashMapExternalizer(StringExternalizer, ClassSnapshotDataSerializer).save(output, snapshot.classSnapshots) + LinkedHashMapExternalizer(StringExternalizer, ClassSnapshotWithHashExternalizer).save(output, snapshot.classSnapshots) } override fun read(input: DataInput): ClasspathEntrySnapshot { return ClasspathEntrySnapshot( - classSnapshots = LinkedHashMapExternalizer(StringExternalizer, ClassSnapshotDataSerializer).read(input) + classSnapshots = LinkedHashMapExternalizer(StringExternalizer, ClassSnapshotWithHashExternalizer).read(input) ) } } -object ClassSnapshotDataSerializer : DataSerializer { +object ClassSnapshotExternalizer : DataExternalizer { override fun save(output: DataOutput, snapshot: ClassSnapshot) { output.writeBoolean(snapshot is KotlinClassSnapshot) @@ -55,6 +82,21 @@ object ClassSnapshotDataSerializer : DataSerializer { } } +object ClassSnapshotWithHashExternalizer : DataExternalizer { + + override fun save(output: DataOutput, snapshot: ClassSnapshotWithHash) { + ClassSnapshotExternalizer.save(output, snapshot.classSnapshot) + LongExternalizer.save(output, snapshot.hash) + } + + override fun read(input: DataInput): ClassSnapshotWithHash { + return ClassSnapshotWithHash( + classSnapshot = ClassSnapshotExternalizer.read(input), + hash = LongExternalizer.read(input) + ) + } +} + object KotlinClassSnapshotExternalizer : DataExternalizer { override fun save(output: DataOutput, snapshot: KotlinClassSnapshot) { @@ -183,32 +225,3 @@ object ContentHashJavaClassSnapshotExternalizer : DataExternalizer : DataExternalizer { - - fun save(file: File, value: T) { - return DataOutputStream(FileOutputStream(file).buffered()).use { - save(it, value) - } - } - - fun load(file: File): T { - return DataInputStream(FileInputStream(file).buffered()).use { - read(it) - } - } - - fun toByteArray(value: T): ByteArray { - val byteArrayOutputStream = ByteArrayOutputStream() - DataOutputStream(byteArrayOutputStream.buffered()).use { - save(it, value) - } - return byteArrayOutputStream.toByteArray() - } - - fun fromByteArray(byteArray: ByteArray): T { - return DataInputStream(ByteArrayInputStream(byteArray).buffered()).use { - read(it) - } - } -} diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt new file mode 100644 index 00000000000..eaf17f1f30f --- /dev/null +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt @@ -0,0 +1,123 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental.classpathDiff + +import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter +import org.jetbrains.kotlin.build.report.metrics.BuildTime +import org.jetbrains.kotlin.build.report.metrics.measure +import org.jetbrains.kotlin.incremental.LookupStorage +import org.jetbrains.kotlin.incremental.storage.LookupSymbolKey +import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader +import org.jetbrains.kotlin.name.ClassId +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.resolve.jvm.JvmClassName + +object ClasspathSnapshotShrinker { + + /** + * Shrinks the given [ClasspathSnapshot] by retaining only classes that are referenced. Referencing info is stored in [LookupStorage]. + * + * This method also removes duplicate classes and [EmptyJavaClassSnapshot]s first. + */ + fun shrink( + classpathSnapshot: ClasspathSnapshot, + lookupStorage: LookupStorage, + metrics: BuildMetricsReporter + ): List { + val allClasses = metrics.measure(BuildTime.GET_NON_DUPLICATE_CLASSES) { + // It's important to remove duplicate classes first before removing `EmptyJavaClassSnapshot`s. + // For example, if jar1!/com/example/A.class is empty and jar2!/com/example/A.class is non-empty, incorrect order of the actions + // will lead to incorrect results. + classpathSnapshot.getNonDuplicateClassSnapshots().filter { it.classSnapshot !is EmptyJavaClassSnapshot } + } + val lookupSymbols = metrics.measure(BuildTime.GET_LOOKUP_SYMBOLS) { + lookupStorage.lookupMap.keys + } + val referencedClasses = metrics.measure(BuildTime.FIND_REFERENCED_CLASSES) { + findReferencedClasses(allClasses, lookupSymbols) + } + return metrics.measure(BuildTime.FIND_TRANSITIVELY_REFERENCED_CLASSES) { + findTransitivelyReferencedClasses(allClasses, referencedClasses) + } + } + + /** + * Finds classes that are referenced. Referencing info is stored in [LookupStorage]. + * + * Note: It's okay to over-approximate referenced classes. + */ + private fun findReferencedClasses( + allClasses: List, + lookupSymbols: Collection + ): List { + val potentialClassNamesOfReferencedClasses = + lookupSymbols.flatMap { + val lookupSymbolFqName = if (it.scope.isEmpty()) FqName(it.name) else FqName("${it.scope}.${it.name}") + listOf( + lookupSymbolFqName, // If LookupSymbol refers to a class, the class's FqName will be captured here. + FqName(it.scope) // If LookupSymbol refers to a class member, the class's FqName will be captured here. + ) + }.toSet() // Use Set for presence check + val potentialPackageNamesOfReferencedPackageLevelMembers = + lookupSymbols.map { + FqName(it.scope) // If LookupSymbol refers to a package-level member, the package's FqName will be captured here. + }.toSet() // Use Set for presence check + + return allClasses.filter { + val classId = it.classSnapshot.getClassId() + + (classId.asSingleFqName() in potentialClassNamesOfReferencedClasses) || + (it.classSnapshot is KotlinClassSnapshot + && it.classSnapshot.classInfo.classKind != KotlinClassHeader.Kind.CLASS + && classId.packageFqName in potentialPackageNamesOfReferencedPackageLevelMembers) + } + } + + /** + * Finds classes that are transitively referenced. For example, if a subclass is referenced, its supertypes will potentially be + * referenced. + * + * The returned list includes the given referenced classes plus the transitively referenced ones. + */ + private fun findTransitivelyReferencedClasses( + allClasses: List, + referencedClasses: List + ): List { + val classIdToClassSnapshot = allClasses.associateBy { it.classSnapshot.getClassId() } + val classIds: Set = classIdToClassSnapshot.keys // Use Set for presence check + val classNameToClassId = classIds.associateBy { JvmClassName.byClassId(it) } + val classNameToClassIdResolver = { className: JvmClassName -> classNameToClassId[className] } + + val supertypesResolver = { classId: ClassId -> + // No need to collect supertypes outside the given set of classes (e.g., "java/lang/Object") + @Suppress("SimpleRedundantLet") + classIdToClassSnapshot[classId]?.let { + it.classSnapshot.getSupertypes(classNameToClassIdResolver).filter { supertype -> supertype in classIds }.toSet() + } ?: emptySet() + } + + val referencedClassIds = referencedClasses.map { it.classSnapshot.getClassId() }.toSet() + val transitivelyReferencedClassIds: Set = + ImpactAnalysis.findImpactedClassesInclusive(referencedClassIds, supertypesResolver) // Use Set for presence check + + return allClasses.filter { it.classSnapshot.getClassId() in transitivelyReferencedClassIds } + } +} + +/** + * Returns all [ClassSnapshot]s in this [ClasspathSnapshot]. + * + * If there are duplicate classes on the classpath, retain only the first one to match the compiler's behavior. + */ +internal fun ClasspathSnapshot.getNonDuplicateClassSnapshots(): List { + val classSnapshots = LinkedHashMap(classpathEntrySnapshots.sumOf { it.classSnapshots.size }) + for (classpathEntrySnapshot in classpathEntrySnapshots) { + for ((unixStyleRelativePath, classSnapshot) in classpathEntrySnapshot.classSnapshots) { + classSnapshots.putIfAbsent(unixStyleRelativePath, classSnapshot) + } + } + return classSnapshots.values.toList() +} diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt index af2aae102ff..ad6b577155b 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt @@ -38,10 +38,10 @@ object ClasspathEntrySnapshotter { } val snapshots = try { - ClassSnapshotter.snapshot(classes, protoBased) + ClassSnapshotter.snapshot(classes, protoBased).map { it.addHash() } } catch (e: Throwable) { if (isKnownProblematicClasspathEntry(classpathEntry)) { - classes.map { ContentHashJavaClassSnapshot(it.contents.md5()) } + classes.map { ContentHashJavaClassSnapshot(it.contents.md5()).addHash() } } else throw e } diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt index 246555dac4d..23d6fb05023 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt @@ -5,11 +5,12 @@ package org.jetbrains.kotlin.incremental.classpathDiff +import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter +import org.jetbrains.kotlin.incremental.ChangesEither +import org.jetbrains.kotlin.incremental.LookupSymbol import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotTestCommon.Util.compileAll import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotTestCommon.Util.snapshot import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotTestCommon.Util.snapshotAll -import org.jetbrains.kotlin.incremental.ClasspathChanges -import org.jetbrains.kotlin.incremental.LookupSymbol import org.jetbrains.kotlin.resolve.sam.SAM_LOOKUP_NAME import org.junit.Test import org.junit.rules.TemporaryFolder @@ -46,7 +47,7 @@ class KotlinOnlyClasspathChangesComputerTest : ClasspathChangesComputerTest() { val sourceFile = SimpleKotlinClass(tmpDir) val previousSnapshot = sourceFile.compileAndSnapshot() val currentSnapshot = sourceFile.changePublicMethodSignature().compileAndSnapshot() - val changes = ClasspathChangesComputer.computeClassChanges(listOf(currentSnapshot), listOf(previousSnapshot)).normalize() + val changes = computeClassChanges(currentSnapshot, previousSnapshot) Changes( lookupSymbols = setOf( @@ -62,7 +63,7 @@ class KotlinOnlyClasspathChangesComputerTest : ClasspathChangesComputerTest() { val sourceFile = SimpleKotlinClass(tmpDir) val previousSnapshot = sourceFile.compileAndSnapshot() val currentSnapshot = sourceFile.changeMethodImplementation().compileAndSnapshot() - val changes = ClasspathChangesComputer.computeClassChanges(listOf(currentSnapshot), listOf(previousSnapshot)).normalize() + val changes = computeClassChanges(currentSnapshot, previousSnapshot) Changes(emptySet(), emptySet()).assertEquals(changes) } @@ -72,7 +73,7 @@ class KotlinOnlyClasspathChangesComputerTest : ClasspathChangesComputerTest() { val classpathSourceDir = File(testDataDir, "../ClasspathChangesComputerTest/testVariousAbiChanges/src/kotlin").canonicalFile val currentSnapshot = snapshotClasspath(File(classpathSourceDir, "current-classpath"), tmpDir) val previousSnapshot = snapshotClasspath(File(classpathSourceDir, "previous-classpath"), tmpDir) - val changes = ClasspathChangesComputer.compute(currentSnapshot, previousSnapshot).normalize() + val changes = computeClasspathChanges(currentSnapshot, previousSnapshot) Changes( lookupSymbols = setOf( @@ -121,7 +122,7 @@ class KotlinOnlyClasspathChangesComputerTest : ClasspathChangesComputerTest() { File(testDataDir, "../ClasspathChangesComputerTest/testImpactAnalysis_KotlinOrJava/src/kotlin").canonicalFile val currentSnapshot = snapshotClasspath(File(classpathSourceDir, "current-classpath"), tmpDir) val previousSnapshot = snapshotClasspath(File(classpathSourceDir, "previous-classpath"), tmpDir) - val changes = ClasspathChangesComputer.compute(currentSnapshot, previousSnapshot).normalize() + val changes = computeClasspathChanges(currentSnapshot, previousSnapshot) Changes( lookupSymbols = setOf( @@ -158,7 +159,7 @@ class JavaOnlyClasspathChangesComputerTest(private val protoBased: Boolean) : Cl val sourceFile = SimpleJavaClass(tmpDir) val previousSnapshot = sourceFile.compile().snapshot(protoBased) val currentSnapshot = sourceFile.changePublicMethodSignature().compile().snapshot(protoBased) - val changes = ClasspathChangesComputer.computeClassChanges(listOf(currentSnapshot), listOf(previousSnapshot)).normalize() + val changes = computeClassChanges(currentSnapshot, previousSnapshot) Changes( lookupSymbols = setOf( @@ -174,7 +175,7 @@ class JavaOnlyClasspathChangesComputerTest(private val protoBased: Boolean) : Cl val sourceFile = SimpleJavaClass(tmpDir) val previousSnapshot = sourceFile.compile().snapshot(protoBased) val currentSnapshot = sourceFile.changeMethodImplementation().compile().snapshot(protoBased) - val changes = ClasspathChangesComputer.computeClassChanges(listOf(currentSnapshot), listOf(previousSnapshot)).normalize() + val changes = computeClassChanges(currentSnapshot, previousSnapshot) Changes(emptySet(), emptySet()).assertEquals(changes) } @@ -184,7 +185,7 @@ class JavaOnlyClasspathChangesComputerTest(private val protoBased: Boolean) : Cl val classpathSourceDir = File(testDataDir, "../ClasspathChangesComputerTest/testVariousAbiChanges/src/java").canonicalFile val currentSnapshot = snapshotClasspath(File(classpathSourceDir, "current-classpath"), tmpDir, protoBased) val previousSnapshot = snapshotClasspath(File(classpathSourceDir, "previous-classpath"), tmpDir, protoBased) - val changes = ClasspathChangesComputer.compute(currentSnapshot, previousSnapshot).normalize() + val changes = computeClasspathChanges(currentSnapshot, previousSnapshot) Changes( lookupSymbols = setOf( @@ -229,7 +230,7 @@ class JavaOnlyClasspathChangesComputerTest(private val protoBased: Boolean) : Cl val classpathSourceDir = File(testDataDir, "../ClasspathChangesComputerTest/testImpactAnalysis_KotlinOrJava/src/java").canonicalFile val currentSnapshot = snapshotClasspath(File(classpathSourceDir, "current-classpath"), tmpDir, protoBased) val previousSnapshot = snapshotClasspath(File(classpathSourceDir, "previous-classpath"), tmpDir, protoBased) - val changes = ClasspathChangesComputer.compute(currentSnapshot, previousSnapshot).normalize() + val changes = computeClasspathChanges(currentSnapshot, previousSnapshot) Changes( lookupSymbols = setOf( @@ -259,7 +260,7 @@ class KotlinAndJavaClasspathChangesComputerTest : ClasspathSnapshotTestCommon() val classpathSourceDir = File(testDataDir, "../ClasspathChangesComputerTest/testImpactAnalysis_KotlinAndJava/src").canonicalFile val currentSnapshot = snapshotClasspath(File(classpathSourceDir, "current-classpath"), tmpDir) val previousSnapshot = snapshotClasspath(File(classpathSourceDir, "previous-classpath"), tmpDir) - val changes = ClasspathChangesComputer.compute(currentSnapshot, previousSnapshot).normalize() + val changes = computeClasspathChanges(currentSnapshot, previousSnapshot) Changes( lookupSymbols = setOf( @@ -301,7 +302,7 @@ private fun snapshotClasspath(classpathSourceDir: File, tmpDir: TemporaryFolder, classpath.addAll(listOfNotNull(classFiles.firstOrNull()?.classRoot)) val relativePaths = classFiles.map { it.unixStyleRelativePath } - val classSnapshots = classFiles.snapshotAll(protoBased) + val classSnapshots = classFiles.snapshotAll(protoBased).map { it.addHash() } ClasspathEntrySnapshot( classSnapshots = relativePaths.zip(classSnapshots).toMap(LinkedHashMap()) ) @@ -309,12 +310,31 @@ private fun snapshotClasspath(classpathSourceDir: File, tmpDir: TemporaryFolder, return ClasspathSnapshot(classpathEntrySnapshots) } -/** Adapted version of [ClasspathChanges.Available] for readability in this test. */ +private fun computeClasspathChanges( + currentClasspathSnapshot: ClasspathSnapshot, + previousClasspathSnapshot: ClasspathSnapshot +): Changes { + return ClasspathChangesComputer.computeChangedAndImpactedSet( + currentClasspathSnapshot.getNonDuplicateClassSnapshots(), + previousClasspathSnapshot.getNonDuplicateClassSnapshots(), + DoNothingBuildMetricsReporter + ).normalize() +} + +private fun computeClassChanges(currentClassSnapshot: ClassSnapshot, previousClassSnapshot: ClassSnapshot): Changes { + return ClasspathChangesComputer.computeClassChanges( + listOf(currentClassSnapshot), + listOf(previousClassSnapshot), + DoNothingBuildMetricsReporter + ).normalize() +} + +/** Adapted version of [ChangesEither.Known] for readability in this test. */ private data class Changes(val lookupSymbols: Set, val fqNames: Set) -private fun ClasspathChanges.normalize(): Changes { - this as ClasspathChanges.Available - return Changes(lookupSymbols, fqNames.map { it.asString() }.toSet()) +private fun ChangeSet.normalize(): Changes { + val changes: ChangesEither.Known = getChanges() + return Changes(changes.lookupSymbols.toSet(), changes.fqNames.map { it.asString() }.toSet()) } private fun Changes.assertEquals(actual: Changes) { diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotSerializerTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotSerializerTest.kt index a8377819f81..6bd6008635f 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotSerializerTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotSerializerTest.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.incremental.classpathDiff import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotTestCommon.Util.readBytes +import org.jetbrains.kotlin.incremental.storage.fromByteArray +import org.jetbrains.kotlin.incremental.storage.toByteArray import org.junit.Assert.assertEquals import org.junit.Test @@ -16,8 +18,8 @@ abstract class ClasspathSnapshotSerializerTest : ClasspathSnapshotTestCommon() { @Test open fun `test ClassSnapshotDataSerializer`() { val originalSnapshot = testSourceFile.compileAndSnapshot() - val serializedSnapshot = ClassSnapshotDataSerializer.toByteArray(originalSnapshot) - val deserializedSnapshot = ClassSnapshotDataSerializer.fromByteArray(serializedSnapshot) + val serializedSnapshot = ClassSnapshotExternalizer.toByteArray(originalSnapshot) + val deserializedSnapshot = ClassSnapshotExternalizer.fromByteArray(serializedSnapshot) assertEquals(originalSnapshot.toGson(), deserializedSnapshot.toGson()) } @@ -36,8 +38,8 @@ class JavaClassesClasspathSnapshotSerializerTest : ClasspathSnapshotSerializerTe val originalSnapshot = testSourceFile.compile().let { ClassSnapshotter.snapshot(listOf(ClassFileWithContents(it, it.readBytes())), includeDebugInfoInSnapshot = false) }.single() - val serializedSnapshot = ClassSnapshotDataSerializer.toByteArray(originalSnapshot) - val deserializedSnapshot = ClassSnapshotDataSerializer.fromByteArray(serializedSnapshot) + val serializedSnapshot = ClassSnapshotExternalizer.toByteArray(originalSnapshot) + val deserializedSnapshot = ClassSnapshotExternalizer.fromByteArray(serializedSnapshot) assertEquals(originalSnapshot.toGson(), deserializedSnapshot.toGson()) } diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinkerTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinkerTest.kt new file mode 100644 index 00000000000..7f639b641c7 --- /dev/null +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinkerTest.kt @@ -0,0 +1,9 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental.classpathDiff + +// TODO: Write this test +abstract class ClasspathSnapshotShrinkerTest diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon.kt index 2c13d363a83..e74da080e82 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon.kt @@ -21,7 +21,7 @@ abstract class ClasspathSnapshotTestCommon { companion object { val testDataDir = - File("libraries/tools/kotlin-gradle-plugin/src/testData/org/jetbrains/kotlin/gradle/incremental/ClasspathSnapshotTestCommon") + File("compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon") } @get:Rule diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt index 672c9270b38..af46261ef13 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt @@ -9,8 +9,9 @@ import org.gradle.api.artifacts.transform.* import org.gradle.api.file.FileSystemLocation import org.gradle.api.provider.Provider import org.gradle.api.tasks.Classpath -import org.jetbrains.kotlin.gradle.incremental.ClasspathEntrySnapshotSerializer -import org.jetbrains.kotlin.gradle.incremental.ClasspathEntrySnapshotter +import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotExternalizer +import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathEntrySnapshotter +import org.jetbrains.kotlin.incremental.storage.saveToFile /** Transform to create a snapshot of a classpath entry (directory or jar). */ @CacheableTransform @@ -25,6 +26,6 @@ abstract class ClasspathEntrySnapshotTransform : TransformAction ClasspathChanges.NotAvailable.ClasspathSnapshotIsDisabled - inputChanges.isIncremental -> getClasspathChanges(inputChanges) - else -> ClasspathChanges.NotAvailable.ForNonIncrementalRun - } logger.info(USING_JVM_INCREMENTAL_COMPILATION_MESSAGE) IncrementalCompilationEnvironment( changedFiles = getChangedFiles(inputChanges, incrementalProps), - classpathChanges = classpathChanges, + classpathChanges = getClasspathChanges(inputChanges), workingDir = taskBuildDirectory.get().asFile, usePreciseJavaTracking = usePreciseJavaTracking, disableMultiModuleIC = disableMultiModuleIC, @@ -756,16 +745,9 @@ abstract class KotlinCompile @Inject constructor( ) } else null - with(classpathSnapshotProperties) { - if (isIncrementalCompilationEnabled() && useClasspathSnapshot.get()) { - copyCurrentClasspathEntrySnapshotFiles() - } - } - val environment = GradleCompilerEnvironment( defaultCompilerClasspath, messageCollector, outputItemCollector, - // The compiler runner should not manage (read, modify, or delete) classpathSnapshotDir - outputFiles = allOutputFiles().minus(classpathSnapshotProperties.classpathSnapshotDirFileCollection), + outputFiles = allOutputFiles(), reportingSettings = reportingSettings, incrementalCompilationEnvironment = icEnv, kotlinScriptExtensions = sourceFilesExtensions.get().toTypedArray() @@ -848,62 +830,29 @@ abstract class KotlinCompile @Inject constructor( return super.source(*sources) } - private fun getClasspathChanges(inputChanges: InputChanges): ClasspathChanges { - val fileChanges = inputChanges.getFileChanges(classpathSnapshotProperties.classpathSnapshot).toList() - return if (fileChanges.isEmpty()) { - ClasspathChanges.Available(LinkedHashSet(), LinkedHashSet()) - } else { - val previousClasspathEntrySnapshotFiles = getPreviousClasspathEntrySnapshotFiles() - if (previousClasspathEntrySnapshotFiles.isEmpty()) { - // When this happens, it means that either the previous classpath was empty or there were no source files to compile in the - // previous non-incremental run so the task action was skipped and the classpath snapshot directory was not populated (see - // AbstractKotlinCompile.executeImpl). - // We could improve this handling, but it's fine to return `UnableToCompute` here as it's likely that there are also no - // source files to compile/recompile in this incremental run. - ClasspathChanges.NotAvailable.UnableToCompute - } else { - val currentClasspathEntrySnapshotFiles = classpathSnapshotProperties.classpathSnapshot.files.toList() - val changedCurrentFiles = fileChanges - .filter { it.changeType == ChangeType.ADDED || it.changeType == ChangeType.MODIFIED } - .map { it.file }.toSet() - ClasspathChangesComputer.compute( - currentClasspathEntrySnapshotFiles = currentClasspathEntrySnapshotFiles, - previousClasspathEntrySnapshotFiles = previousClasspathEntrySnapshotFiles, - unchangedCurrentClasspathEntrySnapshotFiles = currentClasspathEntrySnapshotFiles.filter { it !in changedCurrentFiles } - ) + private fun getClasspathChanges(inputChanges: InputChanges): ClasspathChanges = when { + !classpathSnapshotProperties.useClasspathSnapshot.get() -> ClasspathSnapshotDisabled + else -> { + val classpathSnapshotFiles = ClasspathSnapshotFiles( + classpathSnapshotProperties.classpathSnapshot.files.toList(), + classpathSnapshotProperties.classpathSnapshotDir.get().asFile + ) + when { + !inputChanges.isIncremental -> NotAvailableForNonIncrementalRun(classpathSnapshotFiles) + inputChanges.getFileChanges(classpathSnapshotProperties.classpathSnapshot).none() -> Empty(classpathSnapshotFiles) + !classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.exists() -> { + // When this happens, it means that the classpath snapshot in the previous run was not saved for some reason. It's + // likely that there were no source files to compile, so the task action was skipped (see + // AbstractKotlinCompile.executeImpl), and therefore the classpath snapshot was not saved. + // Missing classpath snapshot will make this run non-incremental, but because there were no source files to compile in + // the previous run, *all* source files in this run (if there are any) need to be compiled anyway, so being + // non-incremental is actually okay. + NotAvailableDueToMissingClasspathSnapshot(classpathSnapshotFiles) + } + else -> ToBeComputedByIncrementalCompiler(classpathSnapshotFiles) } } } - - /** - * Copies classpath entry snapshot files of the current build to `classpathSnapshotDir`. They will be used in the next build (see - * [getPreviousClasspathEntrySnapshotFiles]). - * - * To preserve their order, we put them in subdirectories with names being their indices, as shown below: - * classpathSnapshotDir/0/a-snapshot.bin - * classpathSnapshotDir/1/b-snapshot.bin - * ... - * classpathSnapshotDir/N-1/z-snapshot.bin - */ - private fun copyCurrentClasspathEntrySnapshotFiles() { - val snapshotFiles = classpathSnapshotProperties.classpathSnapshot.files.toList() - val classpathSnapshotDir = classpathSnapshotProperties.classpathSnapshotDir.get().asFile - classpathSnapshotDir.deleteRecursively() - classpathSnapshotDir.mkdirs() - snapshotFiles.forEachIndexed { index, snapshotFile -> - snapshotFile.copyTo(File("$classpathSnapshotDir/$index/${snapshotFile.name}")) - } - } - - /** - * Returns classpath entry snapshot files of the previous build, stored in `classpathSnapshotDir` (see - * [copyCurrentClasspathEntrySnapshotFiles]). - */ - private fun getPreviousClasspathEntrySnapshotFiles(): List { - val classpathSnapshotDir = classpathSnapshotProperties.classpathSnapshotDir.get().asFile - val subDirs = classpathSnapshotDir.listFiles()!!.sortedBy { it.name.toInt() } - return subDirs.map { it.listFiles()!!.single() } - } } @CacheableTask @@ -1152,7 +1101,7 @@ abstract class Kotlin2JsCompile @Inject constructor( logger.info(USING_JS_INCREMENTAL_COMPILATION_MESSAGE) IncrementalCompilationEnvironment( getChangedFiles(inputChanges, incrementalProps), - ClasspathChanges.NotAvailable.ForJSCompiler, + ClasspathChanges.NotAvailableForJSCompiler, taskBuildDirectory.get().asFile, multiModuleICSettings = multiModuleICSettings )