From 586fa8af64282fcd12f1658dd0f1557b6b84197d Mon Sep 17 00:00:00 2001 From: Hung Nguyen Date: Sun, 5 Dec 2021 19:54:20 +0000 Subject: [PATCH] KT-45777: Shrink classpath snapshot incrementally Currently, we shrink classpath snapshots at 2 steps: - Classpath diffing: Shrink the current classpath snapshot against the previous lookup symbols - Classpath snapshot saving: Shrink the current classpath snapshot against the current lookup symbols With this commit, the shrinking at the second step is now incremental. The shrinking at the first step is still non-incremental. --- .idea/dictionaries/hungnv.xml | 1 + .../kotlin/build/report/metrics/BuildTime.kt | 17 +- .../kotlin/incremental/ClasspathChanges.kt | 9 +- .../kotlin/incremental/LookupStorage.kt | 80 +++++++- .../incremental/storage/externalizers.kt | 38 ++-- .../incremental/IncrementalCachesManager.kt | 18 +- .../IncrementalJsCompilerRunner.kt | 2 +- .../IncrementalJvmCompilerRunner.kt | 72 ++++--- .../classpathDiff/ClasspathChangesComputer.kt | 9 +- .../classpathDiff/ClasspathSnapshot.kt | 16 +- .../ClasspathSnapshotShrinker.kt | 181 ++++++++++++++++-- .../classpathDiff/ClasspathSnapshotter.kt | 4 +- .../ClasspathChangesComputerTest.kt | 2 +- ...JavaClassWithNestedClasses$InnerClass.json | 4 + .../SimpleJavaClass-protoBased=false.json | 4 + .../SimpleJavaClass-protoBased=true.json | 4 + .../kotlin/com/example/SimpleKotlinClass.json | 6 +- .../GradleCompilerRunnerWithWorkers.kt | 9 +- .../report/PlainTextBuildReportWriter.kt | 3 +- .../jetbrains/kotlin/gradle/tasks/Tasks.kt | 3 +- 20 files changed, 366 insertions(+), 116 deletions(-) diff --git a/.idea/dictionaries/hungnv.xml b/.idea/dictionaries/hungnv.xml index dd849670d74..a965abf7d17 100644 --- a/.idea/dictionaries/hungnv.xml +++ b/.idea/dictionaries/hungnv.xml @@ -1,6 +1,7 @@ + shrinker snapshotter snapshotter's diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt index 89aa290e2bc..f39f32493fe 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt @@ -29,7 +29,11 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String) IC_CALCULATE_INITIAL_DIRTY_SET(INCREMENTAL_COMPILATION, "Init dirty symbols set"), COMPUTE_CLASSPATH_CHANGES(IC_CALCULATE_INITIAL_DIRTY_SET, "Compute classpath changes"), LOAD_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load current classpath snapshot"), + REMOVE_DUPLICATE_CLASSES(LOAD_CURRENT_CLASSPATH_SNAPSHOT, "Remove duplicate classes"), SHRINK_CURRENT_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Shrink current classpath snapshot"), + GET_LOOKUP_SYMBOLS(SHRINK_CURRENT_CLASSPATH_SNAPSHOT, "Get lookup symbols"), + FIND_REFERENCED_CLASSES(SHRINK_CURRENT_CLASSPATH_SNAPSHOT, "Find referenced classes"), + FIND_TRANSITIVELY_REFERENCED_CLASSES(SHRINK_CURRENT_CLASSPATH_SNAPSHOT, "Find transitively referenced classes"), LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT(COMPUTE_CLASSPATH_CHANGES, "Load shrunk previous classpath snapshot"), COMPUTE_CHANGED_AND_IMPACTED_SET(COMPUTE_CLASSPATH_CHANGES, "Compute changed and impacted set"), COMPUTE_CLASS_CHANGES(COMPUTE_CHANGED_AND_IMPACTED_SET, "Compute class changes"), @@ -47,14 +51,11 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String) INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Incremental iteration"), NON_INCREMENTAL_ITERATION(INCREMENTAL_COMPILATION, "Non-incremental iteration"), IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION, "Write history file"), - SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION, "Save shrunk current classpath snapshot after compilation"), - LOAD_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load classpath snapshot"), - SHRINK_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink classpath snapshot"), - GET_NON_DUPLICATE_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Get non-duplicate classes"), - GET_LOOKUP_SYMBOLS(SHRINK_CLASSPATH_SNAPSHOT, "Get lookup symbols"), - FIND_REFERENCED_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Find referenced classes"), - FIND_TRANSITIVELY_REFERENCED_CLASSES(SHRINK_CLASSPATH_SNAPSHOT, "Find transitively referenced classes"), - SAVE_SHRUNK_CLASSPATH_SNAPSHOT(SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk classpath snapshot"), + SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION, "Shrink and save current classpath snapshot after compilation"), + LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load shrunk previous classpath snapshot after compilation"), + LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Load current classpath snapshot after compilation"), + SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink current classpath snapshot after compilation"), + SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk classpath snapshot after compilation"), COMPILER_PERFORMANCE(readableString = "Compiler time"), COMPILER_INITIALIZATION(COMPILER_PERFORMANCE, "Compiler initialization time"), CODE_ANALYSIS(COMPILER_PERFORMANCE, "Compiler code analyse"), diff --git a/build-common/src/org/jetbrains/kotlin/incremental/ClasspathChanges.kt b/build-common/src/org/jetbrains/kotlin/incremental/ClasspathChanges.kt index 5a78b8034f6..332260cff3a 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/ClasspathChanges.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/ClasspathChanges.kt @@ -11,7 +11,7 @@ import java.io.Serializable /** * Changes to the classpath of the `KotlinCompile` task, or information to compute them later by the Kotlin incremental compiler (see - * [ClasspathSnapshotEnabled.ToBeComputedByIncrementalCompiler]. + * [ClasspathSnapshotEnabled.IncrementalRun.ToBeComputedByIncrementalCompiler]. */ sealed class ClasspathChanges : Serializable { @@ -19,9 +19,12 @@ sealed class ClasspathChanges : Serializable { abstract val classpathSnapshotFiles: ClasspathSnapshotFiles - class Empty(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled() + sealed class IncrementalRun : ClasspathSnapshotEnabled() { - class ToBeComputedByIncrementalCompiler(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled() + class NoChanges(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : IncrementalRun() + + class ToBeComputedByIncrementalCompiler(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : IncrementalRun() + } class NotAvailableDueToMissingClasspathSnapshot(override val classpathSnapshotFiles: ClasspathSnapshotFiles) : ClasspathSnapshotEnabled() diff --git a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt index ff6f37b4fb6..7d7d939f4ce 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt @@ -33,7 +33,8 @@ import java.util.* open class LookupStorage( targetDataDir: File, pathConverter: FileToPathConverter, - storeFullFqNames: Boolean = false + storeFullFqNames: Boolean = false, + private val trackChanges: Boolean = false ) : BasicMapsOwner(targetDataDir) { val LOG = Logger.getInstance("#org.jetbrains.kotlin.jps.build.KotlinBuilder") @@ -45,7 +46,7 @@ open class LookupStorage( private val countersFile = "counters".storageFile private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile, pathConverter)) private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile, pathConverter)) - val lookupMap = registerMap(LookupMap("lookups".storageFile, storeFullFqNames)) + private val lookupMap = TrackedLookupMap(registerMap(LookupMap("lookups".storageFile, storeFullFqNames)), trackChanges) @Volatile private var size: Int = 0 @@ -56,7 +57,8 @@ open class LookupStorage( if (countersFile.exists()) { val lines = countersFile.readLines() size = lines.firstOrNull()?.toIntOrNull() ?: throw IOException("$countersFile exists, but it is empty. " + - "Counters file is corrupted") + "Counters file is corrupted" + ) oldSize = size } } catch (e: IOException) { @@ -66,6 +68,24 @@ open class LookupStorage( } } + /** Set of [LookupSymbol]s that have been added after the initialization of this [LookupStorage] instance. */ + val addedLookupSymbols: Set + get() = run { + check(trackChanges) { "trackChanges is not enabled" } + lookupMap.addedKeys!! + } + + /** Set of [LookupSymbol]s that have been removed after the initialization of this [LookupStorage] instance. */ + val removedLookupSymbols: Set + get() = run { + check(trackChanges) { "trackChanges is not enabled" } + lookupMap.removedKeys!! + } + + /** Returns all [LookupSymbol]s in this storage. Note that this call takes a bit of time to run. */ + val lookupSymbols: Collection + get() = lookupMap.keys + @Synchronized fun get(lookupSymbol: LookupSymbol): Collection { val key = LookupSymbolKey(lookupSymbol.name, lookupSymbol.scope) @@ -266,3 +286,57 @@ data class LookupSymbol(val name: String, val scope: String) : Comparable() else null + val removedKeys = if (trackChanges) mutableSetOf() else null + + val keys: Collection + get() = lookupMap.keys + + operator fun get(key: LookupSymbolKey): Collection? = lookupMap[key] + + operator fun set(key: LookupSymbolKey, fileIds: Set) { + recordSet(key) + lookupMap[key] = fileIds + } + + fun append(key: LookupSymbolKey, fileIds: Collection) { + recordSet(key) + lookupMap.append(key, fileIds) + } + + fun remove(key: LookupSymbolKey) { + recordRemove(key) + lookupMap.remove(key) + } + + private fun recordSet(key: LookupSymbolKey) { + if (!trackChanges) return + if (lookupMap[key] == null) { + if (key in removedKeys!!) { + removedKeys.remove(key) + } else { + addedKeys!!.add(key) + } + } + } + + private fun recordRemove(key: LookupSymbolKey) { + if (!trackChanges) return + if (lookupMap[key] != null) { + if (key in addedKeys!!) { + addedKeys.remove(key) + } else { + removedKeys!!.add(key) + } + } + } +} diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt index 6b33e124bc6..e72b38c96cf 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt @@ -27,40 +27,38 @@ import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.resolve.jvm.JvmClassName import java.io.* -/** - * Storage versioning: - * 0 - only name and value hashes are saved - * 1 - name and scope are saved - */ class LookupSymbolKeyDescriptor( /** If `true`, original values are saved; if `false`, only hashes are saved. */ private val storeFullFqNames: Boolean = false ) : KeyDescriptor { override fun read(input: DataInput): LookupSymbolKey { - val version = input.readByte() - return when (version.toInt()) { - 0 -> { - val name = input.readUTF() - val scope = input.readUTF() - LookupSymbolKey(name.hashCode(), scope.hashCode(), name, scope) - } - 1 -> { - val first = input.readInt() - val second = input.readInt() - LookupSymbolKey(first, second, "", "") - } - else -> throw IllegalArgumentException("Unknown version of LookupSymbolKeyDescriptor=${version}") + // Note: The value of the storeFullFqNames variable below may or may not be the same as LookupSymbolKeyDescriptor.storeFullFqNames. + // Byte value `0` means storeFullFqNames == true, see `save` function below. + val storeFullFqNames = when (val byteValue = input.readByte().toInt()) { + 0 -> true + 1 -> false + else -> error("Unexpected byte value for storeFullFqNames: $byteValue") + } + return if (storeFullFqNames) { + val name = input.readUTF() + val scope = input.readUTF() + LookupSymbolKey(name.hashCode(), scope.hashCode(), name, scope) + } else { + val nameHash = input.readInt() + val scopeHash = input.readInt() + LookupSymbolKey(nameHash, scopeHash, "", "") } } override fun save(output: DataOutput, value: LookupSymbolKey) { + // Write a Byte value `0` to represent storeFullFqNames == true for historical reasons (if we switch this value to `1` or write a + // Boolean instead, it might impact some tests). + output.writeByte(if (storeFullFqNames) 0 else 1) if (storeFullFqNames) { - output.writeByte(0) output.writeUTF(value.name) output.writeUTF(value.scope) } else { - output.writeByte(1) output.writeInt(value.nameHash) output.writeInt(value.scopeHash) } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt index c5097e348fe..b70c1a1a740 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt @@ -27,7 +27,9 @@ abstract class IncrementalCachesManager() @@ -43,7 +45,8 @@ abstract class IncrementalCachesManager(cacheDirectory, rootProjectDir, reporter, storeFullFqNamesInLookupCache) { + storeFullFqNamesInLookupCache: Boolean = false, + trackChangesInLookupCache: Boolean = false +) : IncrementalCachesManager( + cacheDirectory, + rootProjectDir, + reporter, + storeFullFqNamesInLookupCache, + trackChangesInLookupCache +) { private val jvmCacheDir = File(cacheDirectory, "jvm").apply { mkdirs() } override val platformCache = IncrementalJvmCache(jvmCacheDir, outputDir, pathConverter).apply { registerCache() } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt index e014983a068..1a40a2efab6 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJsCompilerRunner.kt @@ -126,7 +126,7 @@ class IncrementalJsCompilerRunner( val classpathChanges = getClasspathChanges( libs, changedFiles, lastBuildInfo, modulesApiHistory, reporter, mapOf(), false, caches.platformCache, - caches.lookupCache.lookupMap.keys.map { if (it.scope.isBlank()) it.name else it.scope }.distinct() + caches.lookupCache.lookupSymbols.map { if (it.scope.isBlank()) it.name else it.scope }.distinct() ) @Suppress("UNUSED_VARIABLE") // for sealed when diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index baedbdf7d24..be8481be35b 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -29,7 +29,10 @@ import org.jetbrains.kotlin.build.GeneratedJvmClass import org.jetbrains.kotlin.build.report.BuildReporter import org.jetbrains.kotlin.build.report.ICReporter import org.jetbrains.kotlin.build.report.ICReporterBase -import org.jetbrains.kotlin.build.report.metrics.* +import org.jetbrains.kotlin.build.report.metrics.BuildAttribute +import org.jetbrains.kotlin.build.report.metrics.BuildTime +import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter +import org.jetbrains.kotlin.build.report.metrics.measure import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.cli.common.ExitCode import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments @@ -42,15 +45,16 @@ import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.IncrementalCompilation import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotDisabled -import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.* +import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.NoChanges +import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.ToBeComputedByIncrementalCompiler +import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableDueToMissingClasspathSnapshot +import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailableForJSCompiler import org.jetbrains.kotlin.incremental.classpathDiff.* import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory import org.jetbrains.kotlin.incremental.multiproject.ModulesApiHistory -import org.jetbrains.kotlin.incremental.storage.ListExternalizer -import org.jetbrains.kotlin.incremental.storage.saveToFile import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector import org.jetbrains.kotlin.incremental.util.Either import org.jetbrains.kotlin.load.java.JavaClassesTracker @@ -141,8 +145,9 @@ class IncrementalJvmCompilerRunner( projectDir, File(args.destination), reporter, - storeFullFqNamesInLookupCache = withSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled - ) + storeFullFqNamesInLookupCache = withSnapshot || classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled, + trackChangesInLookupCache = classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun + ) override fun destinationDir(args: K2JVMCompilerArguments): File = args.destinationAsFile @@ -210,9 +215,10 @@ class IncrementalJvmCompilerRunner( return abiSnapshots } - // Used by `calculateSourcesToCompileImpl` and `performWorkAfterCompilation` methods below. - // Thread safety: There is no concurrent access to this variable. - private var currentClasspathSnapshot: ClasspathSnapshot? = null + // Used by `calculateSourcesToCompileImpl` and `performWorkAfterSuccessfulCompilation` methods below. + // Thread safety: There is no concurrent access to these variables. + private var currentClasspathSnapshot: List? = null + private var shrunkCurrentClasspathAgainstPreviousLookups: List? = null private fun calculateSourcesToCompileImpl( caches: IncrementalJvmCachesManager, @@ -225,15 +231,22 @@ class IncrementalJvmCompilerRunner( val classpathChanges = when (classpathChanges) { // Note: classpathChanges is deserialized, so they are no longer singleton objects and need to be compared using `is` (not `==`) - is Empty -> ChangesEither.Known(emptySet(), emptySet()) + is NoChanges -> ChangesEither.Known(emptySet(), emptySet()) is ToBeComputedByIncrementalCompiler -> reporter.measure(BuildTime.COMPUTE_CLASSPATH_CHANGES) { check(currentClasspathSnapshot == null) currentClasspathSnapshot = reporter.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT) { - CachedClasspathSnapshotSerializer.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) + val classpathSnapshot = + CachedClasspathSnapshotSerializer.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) + reporter.measure(BuildTime.REMOVE_DUPLICATE_CLASSES) { + classpathSnapshot.removeDuplicateAndInaccessibleClasses() + } + } + check(shrunkCurrentClasspathAgainstPreviousLookups == null) + shrunkCurrentClasspathAgainstPreviousLookups = reporter.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT) { + ClasspathSnapshotShrinker.shrink(currentClasspathSnapshot!!, caches.lookupCache, reporter) } ClasspathChangesComputer.computeChangedAndImpactedSet( - currentClasspathSnapshot!!, - caches.lookupCache, + shrunkCurrentClasspathAgainstPreviousLookups!!, classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile, reporter ).getChanges() @@ -243,7 +256,7 @@ class IncrementalJvmCompilerRunner( is ClasspathSnapshotDisabled -> reporter.measure(BuildTime.IC_ANALYZE_CHANGES_IN_DEPENDENCIES) { val lastBuildInfo = BuildInfo.read(lastBuildInfoFile) ?: return CompilationMode.Rebuild(BuildAttribute.IC_IS_NOT_ENABLED) reporter.reportVerbose { "Last Kotlin Build info -- $lastBuildInfo" } - val scopes = caches.lookupCache.lookupMap.keys.map { it.scope.ifBlank { it.name } }.distinct() + val scopes = caches.lookupCache.lookupSymbols.map { it.scope.ifBlank { it.name } }.distinct() getClasspathChanges( args.classpathAsList, changedFiles, lastBuildInfo, modulesApiHistory, reporter, abiSnapshots, withSnapshot, @@ -456,35 +469,14 @@ class IncrementalJvmCompilerRunner( override fun performWorkAfterSuccessfulCompilation(caches: IncrementalJvmCachesManager) { if (classpathChanges is ClasspathChanges.ClasspathSnapshotEnabled) { - reporter.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { - shrinkAndSaveClasspathSnapshot(classpathChanges, caches.lookupCache) + reporter.measure(BuildTime.SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + shrinkAndSaveClasspathSnapshot( + classpathChanges, caches.lookupCache, currentClasspathSnapshot, shrunkCurrentClasspathAgainstPreviousLookups, + reporter + ) } } } - - private fun shrinkAndSaveClasspathSnapshot(classpathChanges: ClasspathChanges.ClasspathSnapshotEnabled, lookupStorage: LookupStorage) { - val classpathSnapshot = currentClasspathSnapshot ?: reporter.measure(BuildTime.LOAD_CLASSPATH_SNAPSHOT) { - CachedClasspathSnapshotSerializer.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) - } - val shrunkClasspathSnapshot = reporter.measure(BuildTime.SHRINK_CLASSPATH_SNAPSHOT) { - ClasspathSnapshotShrinker.shrink(classpathSnapshot, lookupStorage, reporter) - } - reporter.measure(BuildTime.SAVE_SHRUNK_CLASSPATH_SNAPSHOT) { - ListExternalizer(ClassSnapshotWithHashExternalizer).saveToFile( - classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile, - shrunkClasspathSnapshot - ) - } - - reporter.addMetric( - BuildPerformanceMetric.ORIGINAL_CLASSPATH_SNAPSHOT_SIZE, - classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.sumOf { it.length() } - ) - reporter.addMetric( - BuildPerformanceMetric.SHRUNK_CLASSPATH_SNAPSHOT_SIZE, - classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.length() - ) - } } var K2JVMCompilerArguments.destinationAsFile: File diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt index 453bbee2d6c..362465c67e0 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt @@ -27,19 +27,16 @@ import java.util.* object ClasspathChangesComputer { /** - * Computes changes between the current and previous [ClasspathSnapshot]s, plus unchanged elements that are impacted by the changes. + * Computes changes between the current and previous shrunk [ClasspathSnapshot]s, plus unchanged elements that are impacted by the + * changes. * * NOTE: The original classpath may contain duplicate classes, but the shrunk classpath must not contain duplicate classes. */ fun computeChangedAndImpactedSet( - currentClasspathSnapshot: ClasspathSnapshot, - lookupStorageInPreviousRun: LookupStorage, + shrunkCurrentClasspathSnapshot: List, shrunkPreviousClasspathSnapshotFile: File, metrics: BuildMetricsReporter ): ChangeSet { - val shrunkCurrentClasspathSnapshot = metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT) { - ClasspathSnapshotShrinker.shrink(currentClasspathSnapshot, lookupStorageInPreviousRun, metrics) - } val shrunkPreviousClasspathSnapshot = metrics.measure(BuildTime.LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT) { ListExternalizer(ClassSnapshotWithHashExternalizer).loadFromFile(shrunkPreviousClasspathSnapshotFile) } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt index c15b7cb2e59..48edb4353c1 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt @@ -41,17 +41,25 @@ class ClasspathEntrySnapshot( sealed class ClassSnapshot { /** Computes the hash of this [ClassSnapshot] and returns a [ClassSnapshotWithHash]. */ - fun addHash() = ClassSnapshotWithHash(this, ClassSnapshotExternalizer.toByteArray(this).md5()) + val withHash: ClassSnapshotWithHash by lazy { + ClassSnapshotWithHash(this, ClassSnapshotExternalizer.toByteArray(this).md5()) + } } /** Contains a [ClassSnapshot] and its hash. */ -class ClassSnapshotWithHash(val classSnapshot: ClassSnapshot, val hash: Long) +class ClassSnapshotWithHash(val classSnapshot: ClassSnapshot, val hash: Long) { + + override fun toString() = classSnapshot.toString() +} /** [ClassSnapshot] of a Kotlin class. */ class KotlinClassSnapshot( val classInfo: KotlinClassInfo, val supertypes: List -) : ClassSnapshot() +) : ClassSnapshot() { + + override fun toString() = classInfo.classId.toString() +} /** [ClassSnapshot] of a Java class. */ sealed class JavaClassSnapshot : ClassSnapshot() @@ -81,6 +89,8 @@ class RegularJavaClassSnapshot( check(it == JvmClassName.byInternalName(classAbiExcludingMembers.name)) } } + + override fun toString() = classId.toString() } /** The ABI snapshot of a Java element (e.g., class, field, or method). */ diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt index 2c1ce8345f3..c517861de11 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt @@ -5,11 +5,18 @@ package org.jetbrains.kotlin.incremental.classpathDiff -import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter -import org.jetbrains.kotlin.build.report.metrics.BuildTime -import org.jetbrains.kotlin.build.report.metrics.measure +import org.jetbrains.kotlin.build.report.metrics.* +import org.jetbrains.kotlin.incremental.ClasspathChanges +import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.NoChanges +import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.ToBeComputedByIncrementalCompiler +import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableDueToMissingClasspathSnapshot +import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun import org.jetbrains.kotlin.incremental.LookupStorage +import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotShrinker.shrink +import org.jetbrains.kotlin.incremental.storage.ListExternalizer import org.jetbrains.kotlin.incremental.storage.LookupSymbolKey +import org.jetbrains.kotlin.incremental.storage.loadFromFile +import org.jetbrains.kotlin.incremental.storage.saveToFile import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName @@ -18,24 +25,25 @@ import org.jetbrains.kotlin.resolve.jvm.JvmClassName object ClasspathSnapshotShrinker { /** - * Shrinks the given [ClasspathSnapshot] by retaining only classes that are referenced. Referencing info is stored in [LookupStorage]. - * - * This method also removes duplicate classes and [InaccessibleClassSnapshot]s first. + * Shrinks the given classes by retaining only classes that are referenced by the lookup symbols stored in the given [LookupStorage]. */ fun shrink( - classpathSnapshot: ClasspathSnapshot, + allClasses: List, lookupStorage: LookupStorage, - metrics: BuildMetricsReporter + metrics: BuildMetricsReporter = DoNothingBuildMetricsReporter ): List { - val allClasses = metrics.measure(BuildTime.GET_NON_DUPLICATE_CLASSES) { - // It's important to remove duplicate classes first before removing `InaccessibleClassSnapshot`s. - // For example, if jar1!/com/example/A.class is empty and jar2!/com/example/A.class is non-empty, incorrect order of the actions - // will lead to incorrect results. - classpathSnapshot.getNonDuplicateClassSnapshots().filter { it.classSnapshot !is InaccessibleClassSnapshot } - } val lookupSymbols = metrics.measure(BuildTime.GET_LOOKUP_SYMBOLS) { - lookupStorage.lookupMap.keys + lookupStorage.lookupSymbols } + return shrink(allClasses, lookupSymbols, metrics) + } + + /** Shrinks the given classes by retaining only classes that are referenced by the given lookup symbols. */ + fun shrink( + allClasses: List, + lookupSymbols: Collection, + metrics: BuildMetricsReporter = DoNothingBuildMetricsReporter + ): List { val referencedClasses = metrics.measure(BuildTime.FIND_REFERENCED_CLASSES) { findReferencedClasses(allClasses, lookupSymbols) } @@ -45,7 +53,7 @@ object ClasspathSnapshotShrinker { } /** - * Finds classes that are referenced. Referencing info is stored in [LookupStorage]. + * Finds classes that are referenced by the given lookup symbols. * * Note: It's okay to over-approximate referenced classes. */ @@ -107,6 +115,29 @@ object ClasspathSnapshotShrinker { } } +/** + * Removes duplicate classes and [InaccessibleClassSnapshot]s from the given [ClasspathSnapshot]. + * + * To see why removing duplicate classes is important, consider this example: + * - Current classpath: (Unchanged) jar2!/com/example/A.class containing A.foo, (Added) jar3!/com/example/A.class containing A.bar + * - Previous classpath: (Removed) jar1!/com/example/A.class containing A.bar, (Unchanged) jar2!/com/example/A.class containing A.foo + * Without removing duplicates, we might report that there are no changes (both the current classpath and previous classpath have A.foo and + * A.bar). However, the correct report should be that A.bar is removed and A.foo is added because the second A class on each classpath does + * not have any effect. + * + * It's also important to remove duplicate classes first before removing [InaccessibleClassSnapshot]s. For example, if + * jar1!/com/example/A.class is inaccessible and jar2!/com/example/A.class is accessible, removing inaccessible classes first would mean + * that jar2!/com/example/A.class would be kept whereas it shouldn't be since it is a duplicate class (keeping a duplicate class can + * lead to incorrect change reports as shown in the previous example). + * + * That is also why we cannot remove inaccessible classes from each classpath entry in isolation (i.e., during classpath entry + * snapshotting), even though it seems more efficient to do so. For correctness, we need to look at the entire classpath first, remove + * duplicate classes, and then remove inaccessible classes. + */ +internal fun ClasspathSnapshot.removeDuplicateAndInaccessibleClasses(): List { + return getNonDuplicateClassSnapshots().filter { it.classSnapshot !is InaccessibleClassSnapshot } +} + /** * Returns all [ClassSnapshot]s in this [ClasspathSnapshot]. * @@ -121,3 +152,121 @@ internal fun ClasspathSnapshot.getNonDuplicateClassSnapshots(): List, + ) : ShrinkMode() + + class Incremental( + val currentClasspathSnapshot: List, + val shrunkCurrentClasspathAgainstPreviousLookups: List, + val addedLookupSymbols: Set + ) : ShrinkMode() + + object NonIncremental : ShrinkMode() +} + +internal fun shrinkAndSaveClasspathSnapshot( + classpathChanges: ClasspathChanges.ClasspathSnapshotEnabled, + lookupStorage: LookupStorage, + currentClasspathSnapshot: List?, // Not null iff classpathChanges is ToBeComputedByIncrementalCompiler + shrunkCurrentClasspathAgainstPreviousLookups: List?, // Same as above + metrics: BuildMetricsReporter +) { + // In the following, we'll try to shrink the classpath snapshot incrementally when possible. + // For incremental shrinking, we currently use only lookupStorage.addedLookupSymbols, not lookupStorage.removedLookupSymbols. It is + // because updating the shrunk classpath snapshot for removedLookupSymbols is expensive. Therefore, the shrunk classpath snapshot may be + // larger than necessary (and non-deterministic), but it is okay for it to be an over-approximation. + val shrinkMode = when (classpathChanges) { + is NoChanges -> { + val addedLookupSymbols = lookupStorage.addedLookupSymbols + if (addedLookupSymbols.isEmpty()) { + ShrinkMode.NoChanges + } else { + val shrunkPreviousClasspathAgainstPreviousLookups = + metrics.measure(BuildTime.LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + ListExternalizer(ClassSnapshotWithHashExternalizer) + .loadFromFile(classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile) + } + ShrinkMode.Incremental( + currentClasspathSnapshot = metrics.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + CachedClasspathSnapshotSerializer + .load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) + .removeDuplicateAndInaccessibleClasses() + }, + // In the current case, there are no classpath changes, so + // shrunk[*Current*]ClasspathAgainstPreviousLookups == shrunk[*Previous*]ClasspathAgainstPreviousLookups + shrunkCurrentClasspathAgainstPreviousLookups = shrunkPreviousClasspathAgainstPreviousLookups, + addedLookupSymbols = addedLookupSymbols + ) + } + } + is ToBeComputedByIncrementalCompiler -> { + val addedLookupSymbols = lookupStorage.addedLookupSymbols + if (addedLookupSymbols.isEmpty()) { + ShrinkMode.IncrementalNoNewLookups(shrunkCurrentClasspathAgainstPreviousLookups!!) + } else { + ShrinkMode.Incremental( + currentClasspathSnapshot!!, shrunkCurrentClasspathAgainstPreviousLookups!!, addedLookupSymbols + ) + } + } + is NotAvailableDueToMissingClasspathSnapshot, is NotAvailableForNonIncrementalRun -> ShrinkMode.NonIncremental + } + + // Shrink current classpath against current lookups + val shrunkCurrentClasspath: List? = when (shrinkMode) { + is ShrinkMode.NoChanges -> null + is ShrinkMode.IncrementalNoNewLookups -> { + // There are no new lookups, so + // shrunkCurrentClasspathAgainst[*Current*]Lookups == shrunkCurrentClasspathAgainst[*Previous*]Lookups + shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups + } + is ShrinkMode.Incremental -> metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + val shrunkClasses = shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups.map { it.classSnapshot.getClassId() }.toSet() + val notYetShrunkClasses = shrinkMode.currentClasspathSnapshot.filter { it.classSnapshot.getClassId() !in shrunkClasses } + // Don't provide a BuildMetricsReporter for the following call as the sub-BuildTimes in it have a different parent + val shrunkRemainingClassesAgainstNewLookups = shrink(notYetShrunkClasses, shrinkMode.addedLookupSymbols) + + shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups + shrunkRemainingClassesAgainstNewLookups + } + is ShrinkMode.NonIncremental -> { + val classpathSnapshot = metrics.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + CachedClasspathSnapshotSerializer + .load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) + .removeDuplicateAndInaccessibleClasses() + } + metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + // Don't provide a BuildMetricsReporter for the following call as the sub-BuildTimes in it have a different parent + shrink(classpathSnapshot, lookupStorage) + } + } + } + + if (shrinkMode == ShrinkMode.NoChanges) { + // There are no updates to the file so just check that it exists + check(classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.exists()) { + "File '${classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.path}' does not exist" + } + } else { + metrics.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { + ListExternalizer(ClassSnapshotWithHashExternalizer).saveToFile( + classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile, + shrunkCurrentClasspath!! + ) + } + } + + metrics.addMetric( + BuildPerformanceMetric.ORIGINAL_CLASSPATH_SNAPSHOT_SIZE, + classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.sumOf { it.length() } + ) + metrics.addMetric( + BuildPerformanceMetric.SHRUNK_CLASSPATH_SNAPSHOT_SIZE, + classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.length() + ) +} diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt index ea55ea4ddde..acb41c744a1 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt @@ -34,10 +34,10 @@ object ClasspathEntrySnapshotter { } val snapshots = try { - ClassSnapshotter.snapshot(classes, protoBased).map { it.addHash() } + ClassSnapshotter.snapshot(classes, protoBased).map { it.withHash } } catch (e: Throwable) { if ((protoBased ?: protoBasedDefaultValue) && isKnownProblematicClasspathEntry(classpathEntry)) { - classes.map { ContentHashJavaClassSnapshot(it.contents.md5()).addHash() } + classes.map { ContentHashJavaClassSnapshot(it.contents.md5()).withHash } } else throw e } diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt index cd9fc9d294d..fddde8719ef 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt @@ -293,7 +293,7 @@ private fun snapshotClasspath(classpathSourceDir: File, tmpDir: TemporaryFolder, classpath.addAll(listOfNotNull(classFiles.firstOrNull()?.classRoot)) val relativePaths = classFiles.map { it.unixStyleRelativePath } - val classSnapshots = classFiles.snapshotAll(protoBased).map { it.addHash() } + val classSnapshots = classFiles.snapshotAll(protoBased).map { it.withHash } ClasspathEntrySnapshot( classSnapshots = relativePaths.zip(classSnapshots).toMap(LinkedHashMap()) ) diff --git a/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/java/com/example/JavaClassWithNestedClasses$InnerClass.json b/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/java/com/example/JavaClassWithNestedClasses$InnerClass.json index 35856008f52..04462ee148f 100644 --- a/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/java/com/example/JavaClassWithNestedClasses$InnerClass.json +++ b/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/java/com/example/JavaClassWithNestedClasses$InnerClass.json @@ -54,5 +54,9 @@ "className$delegate": { "initializer": {}, "_value": {} + }, + "withHash$delegate": { + "initializer": {}, + "_value": {} } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/java/com/example/SimpleJavaClass-protoBased=false.json b/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/java/com/example/SimpleJavaClass-protoBased=false.json index f352b01a6be..2681c70bade 100644 --- a/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/java/com/example/SimpleJavaClass-protoBased=false.json +++ b/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/java/com/example/SimpleJavaClass-protoBased=false.json @@ -44,5 +44,9 @@ "className$delegate": { "initializer": {}, "_value": {} + }, + "withHash$delegate": { + "initializer": {}, + "_value": {} } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/java/com/example/SimpleJavaClass-protoBased=true.json b/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/java/com/example/SimpleJavaClass-protoBased=true.json index f958567f59c..69659aa0388 100644 --- a/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/java/com/example/SimpleJavaClass-protoBased=true.json +++ b/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/java/com/example/SimpleJavaClass-protoBased=true.json @@ -1175,5 +1175,9 @@ "memoizedSerializedSize": -1, "memoizedHashCode": 0 } + }, + "withHash$delegate": { + "initializer": {}, + "_value": {} } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/kotlin/com/example/SimpleKotlinClass.json b/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/kotlin/com/example/SimpleKotlinClass.json index 4deed64ff55..3196d838c2f 100644 --- a/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/kotlin/com/example/SimpleKotlinClass.json +++ b/compiler/incremental-compilation-impl/testData/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotTestCommon/expected-snapshot/kotlin/com/example/SimpleKotlinClass.json @@ -41,5 +41,9 @@ { "internalName": "java/lang/Object" } - ] + ], + "withHash$delegate": { + "initializer": {}, + "_value": {} + } } \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt index b4ef11297c3..e0e4d86f72b 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/compilerRunner/GradleCompilerRunnerWithWorkers.kt @@ -6,7 +6,6 @@ package org.jetbrains.kotlin.compilerRunner import org.gradle.api.GradleException -import org.gradle.api.file.ConfigurableFileCollection import org.gradle.api.file.DirectoryProperty import org.gradle.api.file.FileSystemOperations import org.gradle.api.logging.Logging @@ -45,7 +44,7 @@ internal class GradleCompilerRunnerWithWorkers( workQueue.submit(GradleKotlinCompilerWorkAction::class.java) { params -> params.compilerWorkArguments.set(workArgs) if (taskOutputsBackup != null) { - params.taskOutputs.from(taskOutputsBackup.outputs) + params.taskOutputs.set(taskOutputsBackup.outputs) params.buildDir.set(taskOutputsBackup.buildDirectory) params.snapshotsDir.set(taskOutputsBackup.snapshotsDir) params.metricsReporter.set(buildMetrics) @@ -66,8 +65,8 @@ internal class GradleCompilerRunnerWithWorkers( fileSystemOperations, parameters.buildDir, parameters.snapshotsDir, - parameters.taskOutputs.files.toList(), - emptyList(), + parameters.taskOutputs.get(), + outputsToExclude = emptyList(), logger ) } else { @@ -98,7 +97,7 @@ internal class GradleCompilerRunnerWithWorkers( internal interface GradleKotlinCompilerWorkParameters : WorkParameters { val compilerWorkArguments: Property - val taskOutputs: ConfigurableFileCollection + val taskOutputs: ListProperty val snapshotsDir: DirectoryProperty val buildDir: DirectoryProperty val metricsReporter: Property diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt index 286e980a8c9..a9b88b5f323 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/report/PlainTextBuildReportWriter.kt @@ -12,7 +12,6 @@ import org.jetbrains.kotlin.gradle.report.data.BuildExecutionDataProcessor import org.jetbrains.kotlin.gradle.report.data.TaskExecutionData import org.jetbrains.kotlin.gradle.utils.Printer import java.io.File -import java.util.* import kotlin.math.max internal class PlainTextBuildReportWriter( @@ -102,7 +101,7 @@ internal class PlainTextBuildReportWriter( p.withIndent("Build performance metrics:") { for (metric in BuildPerformanceMetric.values()) { - p.println("${metric.name}: ${allBuildMetrics[metric]}") + allBuildMetrics[metric]?.let { p.println("${metric.name}: $it") } } } p.println() diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt index a51eca9909d..71db176a33f 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt @@ -57,6 +57,7 @@ import org.jetbrains.kotlin.incremental.ChangedFiles import org.jetbrains.kotlin.incremental.ClasspathChanges import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotDisabled import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.* +import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.IncrementalRun.* import org.jetbrains.kotlin.incremental.ClasspathSnapshotFiles import org.jetbrains.kotlin.incremental.IncrementalCompilerRunner import org.jetbrains.kotlin.library.impl.isKotlinLibrary @@ -852,7 +853,7 @@ abstract class KotlinCompile @Inject constructor( ) when { !inputChanges.isIncremental -> NotAvailableForNonIncrementalRun(classpathSnapshotFiles) - inputChanges.getFileChanges(classpathSnapshotProperties.classpathSnapshot).none() -> Empty(classpathSnapshotFiles) + inputChanges.getFileChanges(classpathSnapshotProperties.classpathSnapshot).none() -> NoChanges(classpathSnapshotFiles) !classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.exists() -> { // When this happens, it means that the classpath snapshot in the previous run was not saved for some reason. It's // likely that there were no source files to compile, so the task action was skipped (see