diff --git a/build-common/src/org/jetbrains/kotlin/build/report/BuildReporter.kt b/build-common/src/org/jetbrains/kotlin/build/report/BuildReporter.kt index c6547bff350..6cbd989c0d0 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/BuildReporter.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/BuildReporter.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.build.report import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter +import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter import org.jetbrains.kotlin.build.report.metrics.RemoteBuildMetricsReporter open class BuildReporter( @@ -22,3 +23,5 @@ class RemoteBuildReporter( buildMetricsReporter.flush() } } + +object DoNothingBuildReporter : BuildReporter(DoNothingICReporter, DoNothingBuildMetricsReporter) diff --git a/build-common/src/org/jetbrains/kotlin/build/report/ICReporter.kt b/build-common/src/org/jetbrains/kotlin/build/report/ICReporter.kt index b08e2feef41..34467edf1d3 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/ICReporter.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/ICReporter.kt @@ -18,4 +18,11 @@ interface ICReporter { fun reportMarkDirty(affectedFiles: Iterable, reason: String) } - +object DoNothingICReporter : ICReporter { + override fun report(message: () -> String) {} + override fun reportVerbose(message: () -> String) {} + override fun reportCompileIteration(incremental: Boolean, sourceFiles: Collection, exitCode: ExitCode) {} + override fun reportMarkDirtyClass(affectedFiles: Iterable, classFqName: String) {} + override fun reportMarkDirtyMember(affectedFiles: Iterable, scope: String, name: String) {} + override fun reportMarkDirty(affectedFiles: Iterable, reason: String) {} +} diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt index 94e079dea08..9d1dd225e54 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt @@ -17,8 +17,9 @@ enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, va // Metrics for the `kotlin.incremental.useClasspathSnapshot` feature CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT(parent = null, "Number of times 'ClasspathEntrySnapshotTransform' ran", type = SizeMetricType.NUMBER), - JAR_CLASSPATH_ENTRY_SIZE(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, "Size of classpath entry (if it's a jar)", type = SizeMetricType.BYTES), - CLASSPATH_ENTRY_SNAPSHOT_SIZE(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, "Size of classpath entry (directory or jar)'s snapshot", type = SizeMetricType.BYTES), + JAR_CLASSPATH_ENTRY_SIZE(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, "Size of jar classpath entry", type = SizeMetricType.BYTES), + JAR_CLASSPATH_ENTRY_SNAPSHOT_SIZE(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, "Size of jar classpath entry's snapshot", type = SizeMetricType.BYTES), + DIRECTORY_CLASSPATH_ENTRY_SNAPSHOT_SIZE(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, "Size of directory classpath entry's snapshot", type = SizeMetricType.BYTES), SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT(parent = null, "Number of times classpath snapshot is shrunk and saved after compilation", type = SizeMetricType.NUMBER), CLASSPATH_ENTRY_COUNT(parent = SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Number of classpath entries", type = SizeMetricType.NUMBER), CLASSPATH_SNAPSHOT_SIZE(parent = SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Size of classpath snapshot", type = SizeMetricType.BYTES), diff --git a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt index 38cdf918892..0619819c7ea 100644 --- a/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt +++ b/build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildTime.kt @@ -51,11 +51,11 @@ enum class BuildTime(val parent: BuildTime? = null, val readableString: String) COMPILATION_ROUND(INCREMENTAL_COMPILATION_DAEMON, "Sources compilation round"), IC_WRITE_HISTORY_FILE(INCREMENTAL_COMPILATION_DAEMON, "Write history file"), SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION(INCREMENTAL_COMPILATION_DAEMON, "Shrink and save current classpath snapshot after compilation"), - INCREMENTAL_LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Incremental shrinking - Load shrunk previous classpath snapshot"), - INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Incremental shrinking - Load current classpath snapshot"), - INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Incremental shrinking - Shrink current classpath snapshot"), - NON_INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Non-incremental shrinking - Load current classpath snapshot"), - NON_INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Non-incremental shrinking - Shrink current classpath snapshot"), + INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink current classpath snapshot incrementally"), + INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT(INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT, "Load current classpath snapshot"), + INCREMENTAL_LOAD_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AGAINST_PREVIOUS_LOOKUPS(INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT, "Load shrunk current classpath snapshot against previous lookups"), + NON_INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Shrink current classpath snapshot non-incrementally"), + NON_INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT(NON_INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT, "Load current classpath snapshot"), SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT(SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION, "Save shrunk current classpath snapshot"), COMPILER_PERFORMANCE(readableString = "Compiler time"), COMPILER_INITIALIZATION(COMPILER_PERFORMANCE, "Compiler initialization time"), diff --git a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt index 55cc4ed7874..92707920892 100644 --- a/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt +++ b/compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/CompilationOptions.kt @@ -86,16 +86,16 @@ class IncrementalCompilationOptions( override fun toString(): String { return "IncrementalCompilationOptions(" + - "super=${super.toString()}, " + - "areFileChangesKnown=$areFileChangesKnown, " + - "modifiedFiles=$modifiedFiles, " + - "deletedFiles=$deletedFiles, " + - "classpathChanges=$classpathChanges, " + - "workingDir=$workingDir, " + - "multiModuleICSettings=$multiModuleICSettings, " + - "usePreciseJavaTracking=$usePreciseJavaTracking" + - "outputFiles=$outputFiles" + - ")" + "super=${super.toString()}, " + + "areFileChangesKnown=$areFileChangesKnown, " + + "modifiedFiles=$modifiedFiles, " + + "deletedFiles=$deletedFiles, " + + "classpathChanges=${classpathChanges::class.simpleName}, " + + "workingDir=$workingDir, " + + "multiModuleICSettings=$multiModuleICSettings, " + + "usePreciseJavaTracking=$usePreciseJavaTracking" + + "outputFiles=$outputFiles" + + ")" } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index 92ed2d90c07..0b1ebbde159 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -51,9 +51,11 @@ import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnable import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableDueToMissingClasspathSnapshot import org.jetbrains.kotlin.incremental.ClasspathChanges.ClasspathSnapshotEnabled.NotAvailableForNonIncrementalRun import org.jetbrains.kotlin.incremental.ClasspathChanges.NotAvailableForJSCompiler -import org.jetbrains.kotlin.incremental.classpathDiff.* -import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathChangesComputer.computeChangedAndImpactedSet -import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotShrinker.shrinkClasspath +import org.jetbrains.kotlin.incremental.classpathDiff.AccessibleClassSnapshot +import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathChangesComputer.computeClasspathChanges +import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotBuildReporter +import org.jetbrains.kotlin.incremental.classpathDiff.shrinkAndSaveClasspathSnapshot +import org.jetbrains.kotlin.incremental.classpathDiff.toChangesEither import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.multiproject.EmptyModulesApiHistory @@ -235,32 +237,24 @@ class IncrementalJvmCompilerRunner( val dirtyFiles = DirtyFilesContainer(caches, reporter, kotlinSourceFilesExtensions) initDirtyFiles(dirtyFiles, changedFiles) + reporter.reportVerbose { "Classpath changes info passed from Gradle task: ${classpathChanges::class.simpleName}" } val classpathChanges = when (classpathChanges) { // Note: classpathChanges is deserialized, so they are no longer singleton objects and need to be compared using `is` (not `==`) is NoChanges -> ChangesEither.Known(emptySet(), emptySet()) is ToBeComputedByIncrementalCompiler -> reporter.measure(BuildTime.COMPUTE_CLASSPATH_CHANGES) { - check(currentClasspathSnapshot == null) - currentClasspathSnapshot = reporter.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT) { - val classpathSnapshot = - CachedClasspathSnapshotSerializer.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) - reporter.measure(BuildTime.REMOVE_DUPLICATE_CLASSES) { - classpathSnapshot.removeDuplicateAndInaccessibleClasses() + val storeCurrentClasspathSnapshotForReuse = + { currentClasspathSnapshotArg: List, + shrunkCurrentClasspathAgainstPreviousLookupsArg: List -> + check(currentClasspathSnapshot == null) + check(shrunkCurrentClasspathAgainstPreviousLookups == null) + currentClasspathSnapshot = currentClasspathSnapshotArg + shrunkCurrentClasspathAgainstPreviousLookups = shrunkCurrentClasspathAgainstPreviousLookupsArg } - } - check(shrunkCurrentClasspathAgainstPreviousLookups == null) - shrunkCurrentClasspathAgainstPreviousLookups = reporter.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT) { - shrinkClasspath( - currentClasspathSnapshot!!, caches.lookupCache, - ClasspathSnapshotShrinker.MetricsReporter( - reporter, - BuildTime.GET_LOOKUP_SYMBOLS, BuildTime.FIND_REFERENCED_CLASSES, BuildTime.FIND_TRANSITIVELY_REFERENCED_CLASSES - ) - ) - } - computeChangedAndImpactedSet( - shrunkCurrentClasspathAgainstPreviousLookups!!, - classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile, - reporter + computeClasspathChanges( + classpathChanges.classpathSnapshotFiles, + caches.lookupCache, + storeCurrentClasspathSnapshotForReuse, + ClasspathSnapshotBuildReporter(reporter) ).toChangesEither() } is NotAvailableDueToMissingClasspathSnapshot -> ChangesEither.Unknown(BuildAttribute.CLASSPATH_SNAPSHOT_NOT_FOUND) @@ -491,7 +485,7 @@ class IncrementalJvmCompilerRunner( reporter.measure(BuildTime.SHRINK_AND_SAVE_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) { shrinkAndSaveClasspathSnapshot( classpathChanges, caches.lookupCache, currentClasspathSnapshot, shrunkCurrentClasspathAgainstPreviousLookups, - reporter + ClasspathSnapshotBuildReporter(reporter) ) } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt index 1c0acb59e36..ba19b618135 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt @@ -10,6 +10,7 @@ import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter import org.jetbrains.kotlin.build.report.metrics.BuildTime import org.jetbrains.kotlin.build.report.metrics.measure import org.jetbrains.kotlin.incremental.* +import org.jetbrains.kotlin.incremental.classpathDiff.ClasspathSnapshotShrinker.shrinkClasspath import org.jetbrains.kotlin.incremental.classpathDiff.ImpactAnalysis.computeImpactedSetInclusive import org.jetbrains.kotlin.incremental.storage.FileToCanonicalPathConverter import org.jetbrains.kotlin.incremental.storage.ListExternalizer @@ -17,28 +18,53 @@ import org.jetbrains.kotlin.incremental.storage.loadFromFile import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.resolve.jvm.JvmClassName import org.jetbrains.kotlin.resolve.sam.SAM_LOOKUP_NAME -import java.io.File import java.util.* /** Computes changes between two [ClasspathSnapshot]s .*/ object ClasspathChangesComputer { /** - * Computes changes between the current and previous shrunk [ClasspathSnapshot]s, plus unchanged elements that are impacted by the - * changes. + * Computes changes between the current and previous classpath, plus unchanged elements that are impacted by the changes. * - * NOTE: The original classpath may contain duplicate classes, but the shrunk classpath must not contain duplicate classes. + * NOTE: We shrink the classpath first before comparing them. The original classpath may contain duplicate classes, but the shrunk + * classpath must not contain duplicate classes. */ - fun computeChangedAndImpactedSet( - shrunkCurrentClasspathSnapshot: List, - shrunkPreviousClasspathSnapshotFile: File, - metrics: BuildMetricsReporter + fun computeClasspathChanges( + classpathSnapshotFiles: ClasspathSnapshotFiles, + lookupStorage: LookupStorage, + storeCurrentClasspathSnapshotForReuse: (currentClasspathSnapshot: List, shrunkCurrentClasspathAgainstPreviousLookups: List) -> Unit, + reporter: ClasspathSnapshotBuildReporter ): ProgramSymbolSet { - val shrunkPreviousClasspathSnapshot = metrics.measure(BuildTime.LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT) { - ListExternalizer(AccessibleClassSnapshotExternalizer).loadFromFile(shrunkPreviousClasspathSnapshotFile) + val currentClasspathSnapshot = reporter.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT) { + val classpathSnapshot = CachedClasspathSnapshotSerializer.load(classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) + reporter.measure(BuildTime.REMOVE_DUPLICATE_CLASSES) { + classpathSnapshot.removeDuplicateAndInaccessibleClasses() + } } - return metrics.measure(BuildTime.COMPUTE_CHANGED_AND_IMPACTED_SET) { - computeChangedAndImpactedSet(shrunkCurrentClasspathSnapshot, shrunkPreviousClasspathSnapshot, metrics) + val shrunkCurrentClasspathAgainstPreviousLookups = reporter.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT) { + shrinkClasspath( + currentClasspathSnapshot, lookupStorage, + ClasspathSnapshotShrinker.MetricsReporter( + reporter, + BuildTime.GET_LOOKUP_SYMBOLS, BuildTime.FIND_REFERENCED_CLASSES, BuildTime.FIND_TRANSITIVELY_REFERENCED_CLASSES + ) + ) + } + reporter.reportVerbose { + "Shrunk current classpath snapshot for diffing," + + " retained ${shrunkCurrentClasspathAgainstPreviousLookups.size} / ${currentClasspathSnapshot.size} classes" + } + storeCurrentClasspathSnapshotForReuse(currentClasspathSnapshot, shrunkCurrentClasspathAgainstPreviousLookups) + + val shrunkPreviousClasspathSnapshot = reporter.measure(BuildTime.LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT) { + ListExternalizer(AccessibleClassSnapshotExternalizer).loadFromFile(classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile) + } + reporter.reportVerbose { + "Loaded shrunk previous classpath snapshot for diffing, found ${shrunkPreviousClasspathSnapshot.size} classes" + } + + return reporter.measure(BuildTime.COMPUTE_CHANGED_AND_IMPACTED_SET) { + computeChangedAndImpactedSet(shrunkCurrentClasspathAgainstPreviousLookups, shrunkPreviousClasspathSnapshot, reporter) } } @@ -50,7 +76,7 @@ object ClasspathChangesComputer { fun computeChangedAndImpactedSet( currentClassSnapshots: List, previousClassSnapshots: List, - metrics: BuildMetricsReporter + reporter: ClasspathSnapshotBuildReporter ): ProgramSymbolSet { val currentClasses: Map = currentClassSnapshots.associateBy { it.classId } val previousClasses: Map = previousClassSnapshots.associateBy { it.classId } @@ -69,15 +95,16 @@ object ClasspathChangesComputer { } else null } - val classChanges = metrics.measure(BuildTime.COMPUTE_CLASS_CHANGES) { - computeClassChanges(changedCurrentClasses, changedPreviousClasses, metrics) + val changedSet = reporter.measure(BuildTime.COMPUTE_CLASS_CHANGES) { + computeClassChanges(changedCurrentClasses, changedPreviousClasses, reporter) + } + reporter.reportVerboseWithLimit { "Changed set = ${changedSet.toDebugString()}" } + + if (changedSet.isEmpty()) { + return changedSet } - if (classChanges.isEmpty()) { - return classChanges - } - - return metrics.measure(BuildTime.COMPUTE_IMPACTED_SET) { + val changedAndImpactedSet = reporter.measure(BuildTime.COMPUTE_IMPACTED_SET) { // Note that changes may contain added symbols (they can also impact recompilation -- see examples in JavaClassChangesComputer). // So ideally, the result should be: // computeImpactedSetInclusive(changes = changesOnPreviousClasspath, allClasses = classesOnPreviousClasspath) + @@ -88,10 +115,16 @@ object ClasspathChangesComputer { // Note: `allClasses` may contain overlapping ClassIds, but it won't be an issue. Also, we will replace // classesOnCurrentClasspath with changedClassesOnCurrentClasspath to avoid listing unchanged classes twice. computeImpactedSetInclusive( - changes = classChanges, + changes = changedSet, allClasses = (previousClassSnapshots.asSequence() + changedCurrentClasses.asSequence()).asIterable() ) } + reporter.reportVerboseWithLimit { + "Impacted classes = " + + (changedAndImpactedSet.run { classes + classMembers.keys } - changedSet.run { classes + classMembers.keys }) + } + + return changedAndImpactedSet } /** diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotBuildReporter.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotBuildReporter.kt new file mode 100644 index 00000000000..ba06014ccbf --- /dev/null +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotBuildReporter.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.incremental.classpathDiff + +import org.jetbrains.kotlin.build.report.BuildReporter +import org.jetbrains.kotlin.build.report.ICReporter +import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter + +class ClasspathSnapshotBuildReporter(private val buildReporter: BuildReporter) : + ICReporter by buildReporter, BuildMetricsReporter by buildReporter { + + override fun reportVerbose(message: () -> String) { + buildReporter.reportVerbose { "[ClasspathSnapshot] ${message()}" } + } + + fun reportVerboseWithLimit(maxLength: Int = 1000, message: () -> String) { + reportVerbose { + message().let { + if (it.length > maxLength) { + it.substring(0, maxLength) + "... (string too long, showing $maxLength / ${it.length} chars)" + } else it + } + } + } +} diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt index 70476c40b9e..28cee575956 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt @@ -174,27 +174,36 @@ private fun ClasspathSnapshot.getNonDuplicateClassSnapshots(): List, - ) : ShrinkMode() - - class Incremental( + class UnchangedLookupsChangedClasspath( val currentClasspathSnapshot: List, - val shrunkCurrentClasspathAgainstPreviousLookups: List, - val addedLookupSymbols: Set + val shrunkCurrentClasspathAgainstPreviousLookups: List ) : ShrinkMode() + sealed class ChangedLookups : ShrinkMode() { + abstract val addedLookupSymbols: Set + } + + class ChangedLookupsUnchangedClasspath( + override val addedLookupSymbols: Set + ) : ChangedLookups() + + class ChangedLookupsChangedClasspath( + override val addedLookupSymbols: Set, + val currentClasspathSnapshot: List, + val shrunkCurrentClasspathAgainstPreviousLookups: List + ) : ChangedLookups() + object NonIncremental : ShrinkMode() } internal fun shrinkAndSaveClasspathSnapshot( classpathChanges: ClasspathChanges.ClasspathSnapshotEnabled, lookupStorage: LookupStorage, - currentClasspathSnapshot: List?, // Not null iff classpathChanges is ToBeComputedByIncrementalCompiler - shrunkCurrentClasspathAgainstPreviousLookups: List?, // Same as above - metrics: BuildMetricsReporter + precomputedCurrentClasspathSnapshot: List?, // Not null iff classpathChanges is ToBeComputedByIncrementalCompiler + precomputedShrunkCurrentClasspathAgainstPreviousLookups: List?, // Not null iff classpathChanges is ToBeComputedByIncrementalCompiler + reporter: ClasspathSnapshotBuildReporter ) { // In the following, we'll try to shrink the classpath snapshot incrementally when possible. // For incremental shrinking, we currently use only lookupStorage.addedLookupSymbols, not lookupStorage.removedLookupSymbols. It is @@ -204,72 +213,94 @@ internal fun shrinkAndSaveClasspathSnapshot( is NoChanges -> { val addedLookupSymbols = lookupStorage.addedLookupSymbols if (addedLookupSymbols.isEmpty()) { - ShrinkMode.NoChanges + ShrinkMode.UnchangedLookupsUnchangedClasspath } else { - val shrunkPreviousClasspathAgainstPreviousLookups = - metrics.measure(BuildTime.INCREMENTAL_LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT) { - ListExternalizer(AccessibleClassSnapshotExternalizer) - .loadFromFile(classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile) - } - ShrinkMode.Incremental( - currentClasspathSnapshot = metrics.measure(BuildTime.INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT) { - CachedClasspathSnapshotSerializer - .load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) - .removeDuplicateAndInaccessibleClasses() - }, - // In the current case, there are no classpath changes, so - // shrunk[*Current*]ClasspathAgainstPreviousLookups == shrunk[*Previous*]ClasspathAgainstPreviousLookups - shrunkCurrentClasspathAgainstPreviousLookups = shrunkPreviousClasspathAgainstPreviousLookups, - addedLookupSymbols = addedLookupSymbols - ) + ShrinkMode.ChangedLookupsUnchangedClasspath(addedLookupSymbols) } } is ToBeComputedByIncrementalCompiler -> { val addedLookupSymbols = lookupStorage.addedLookupSymbols if (addedLookupSymbols.isEmpty()) { - ShrinkMode.IncrementalNoNewLookups(shrunkCurrentClasspathAgainstPreviousLookups!!) + ShrinkMode.UnchangedLookupsChangedClasspath( + precomputedCurrentClasspathSnapshot!!, + precomputedShrunkCurrentClasspathAgainstPreviousLookups!! + ) } else { - ShrinkMode.Incremental(currentClasspathSnapshot!!, shrunkCurrentClasspathAgainstPreviousLookups!!, addedLookupSymbols) + ShrinkMode.ChangedLookupsChangedClasspath( + addedLookupSymbols, + precomputedCurrentClasspathSnapshot!!, + precomputedShrunkCurrentClasspathAgainstPreviousLookups!! + ) } } is NotAvailableDueToMissingClasspathSnapshot, is NotAvailableForNonIncrementalRun -> ShrinkMode.NonIncremental } // Shrink current classpath against current lookups - val shrunkCurrentClasspath: List? = when (shrinkMode) { - is ShrinkMode.NoChanges -> null - is ShrinkMode.IncrementalNoNewLookups -> { - // There are no new lookups, so - // shrunkCurrentClasspathAgainst[*Current*]Lookups == shrunkCurrentClasspathAgainst[*Previous*]Lookups - shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups + val (currentClasspath: List?, shrunkCurrentClasspath: List?) = when (shrinkMode) { + is ShrinkMode.UnchangedLookupsUnchangedClasspath -> { + // There are no changes in the lookups and classpath, so there will be no changes in the shrunk classpath snapshot compared to + // the previous run. Return null here as we don't need to compute this. + null to null } - is ShrinkMode.Incremental -> metrics.measure(BuildTime.INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT) { - val shrunkClasses = shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups.mapTo(mutableSetOf()) { it.classId } - val notYetShrunkClasses = shrinkMode.currentClasspathSnapshot.filter { it.classId !in shrunkClasses } - val addedLookupSymbols = shrinkMode.addedLookupSymbols - val shrunkRemainingClassesAgainstNewLookups = shrinkClasses(notYetShrunkClasses, addedLookupSymbols) + is ShrinkMode.UnchangedLookupsChangedClasspath -> { + // There are no changes in the lookups, so + // shrunkCurrentClasspathAgainst[*Current*]Lookups == shrunkCurrentClasspathAgainst[*Previous*]Lookups + shrinkMode.currentClasspathSnapshot to shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups + } + is ShrinkMode.ChangedLookups -> reporter.measure(BuildTime.INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT) { + // There are changes in the lookups, so we will shrink incrementally. + val currentClasspath = reporter.measure(BuildTime.INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT) { + when (shrinkMode) { + is ShrinkMode.ChangedLookupsUnchangedClasspath -> + CachedClasspathSnapshotSerializer + .load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) + .removeDuplicateAndInaccessibleClasses() + is ShrinkMode.ChangedLookupsChangedClasspath -> shrinkMode.currentClasspathSnapshot + } + } + val shrunkCurrentClasspathAgainstPreviousLookups = + reporter.measure(BuildTime.INCREMENTAL_LOAD_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AGAINST_PREVIOUS_LOOKUPS) { + when (shrinkMode) { + is ShrinkMode.ChangedLookupsUnchangedClasspath -> { + // There are no changes in the classpath, so + // shrunk[*Current*]ClasspathAgainstPreviousLookups == shrunk[*Previous*]ClasspathAgainstPreviousLookups + ListExternalizer(AccessibleClassSnapshotExternalizer) + .loadFromFile(classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile) + } + is ShrinkMode.ChangedLookupsChangedClasspath -> shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups + } + } - shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups + shrunkRemainingClassesAgainstNewLookups + val shrunkClasses = shrunkCurrentClasspathAgainstPreviousLookups.mapTo(mutableSetOf()) { it.classId } + val notYetShrunkClasses = currentClasspath.filter { it.classId !in shrunkClasses } + val shrunkRemainingClassesAgainstNewLookups = shrinkClasses(notYetShrunkClasses, shrinkMode.addedLookupSymbols) + + val shrunkCurrentClasspath = shrunkCurrentClasspathAgainstPreviousLookups + shrunkRemainingClassesAgainstNewLookups + currentClasspath to shrunkCurrentClasspath } is ShrinkMode.NonIncremental -> { - val classpathSnapshot = metrics.measure(BuildTime.NON_INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT) { - CachedClasspathSnapshotSerializer - .load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) - .removeDuplicateAndInaccessibleClasses() - } - metrics.measure(BuildTime.NON_INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT) { - shrinkClasspath(classpathSnapshot, lookupStorage) + // Changes in the lookups and classpath are not available, so we will shrink non-incrementally. + reporter.measure(BuildTime.NON_INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT) { + val currentClasspath = reporter.measure(BuildTime.NON_INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT) { + CachedClasspathSnapshotSerializer + .load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles) + .removeDuplicateAndInaccessibleClasses() + } + val shrunkCurrentClasspath = shrinkClasspath(currentClasspath, lookupStorage) + currentClasspath to shrunkCurrentClasspath } } } - if (shrinkMode == ShrinkMode.NoChanges) { - // There are no updates to the file so just check that it exists - check(classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.exists()) { + if (shrinkMode == ShrinkMode.UnchangedLookupsUnchangedClasspath) { + // There are no changes in the lookups and classpath, so there will be no changes in the shrunk classpath snapshot compared to the + // previous run. Just double-check that the file still exists. + check(classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.isFile) { "File '${classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.path}' does not exist" } } else { - metrics.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT) { + reporter.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT) { ListExternalizer(AccessibleClassSnapshotExternalizer).saveToFile( classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile, shrunkCurrentClasspath!! @@ -277,16 +308,23 @@ internal fun shrinkAndSaveClasspathSnapshot( } } - metrics.addMetric(BuildPerformanceMetric.SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, 1) - metrics.addMetric( + reporter.reportVerbose { + "Shrunk current classpath snapshot after compilation (shrink mode = ${shrinkMode::class.simpleName})" + when (shrinkMode) { + is ShrinkMode.UnchangedLookupsUnchangedClasspath -> ", no updates since previous run" + else -> ", retained ${shrunkCurrentClasspath!!.size} / ${currentClasspath!!.size} classes" + } + } + + reporter.addMetric(BuildPerformanceMetric.SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, 1) + reporter.addMetric( BuildPerformanceMetric.CLASSPATH_ENTRY_COUNT, classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.size.toLong() ) - metrics.addMetric( + reporter.addMetric( BuildPerformanceMetric.CLASSPATH_SNAPSHOT_SIZE, classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.sumOf { it.length() } ) - metrics.addMetric( + reporter.addMetric( BuildPerformanceMetric.SHRUNK_CLASSPATH_SNAPSHOT_SIZE, classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.length() ) diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ProgramSymbol.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ProgramSymbol.kt index d217fd378e8..72c48ab64d7 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ProgramSymbol.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ProgramSymbol.kt @@ -58,6 +58,10 @@ class ProgramSymbolSet private constructor( .flatMap { (packageFqName, memberNames) -> memberNames.asSequence().map { PackageMember(packageFqName, it) } } } + fun toDebugString(): String { + return "${ProgramSymbolSet::class.simpleName}(classes = $classes, classMembers = $classMembers, packageMembers = $packageMembers)" + } + /** * Collects [ProgramSymbol]s and returns a [ProgramSymbolSet]. * diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt index 74bfcda35b5..119c3abb3c2 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputerTest.kt @@ -5,7 +5,7 @@ package org.jetbrains.kotlin.incremental.classpathDiff -import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter +import org.jetbrains.kotlin.build.report.DoNothingBuildReporter import org.jetbrains.kotlin.incremental.ChangesEither import org.jetbrains.kotlin.incremental.LookupSymbol import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity.CLASS_LEVEL @@ -437,7 +437,7 @@ private fun computeClasspathChanges( return ClasspathChangesComputer.computeChangedAndImpactedSet( currentClasspathSnapshot.removeDuplicateAndInaccessibleClasses(), previousClasspathSnapshot.removeDuplicateAndInaccessibleClasses(), - DoNothingBuildMetricsReporter + ClasspathSnapshotBuildReporter(DoNothingBuildReporter) ).normalize() } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt index b7867b215fd..f9727e79606 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt @@ -95,8 +95,11 @@ abstract class ClasspathEntrySnapshotTransform : TransformAction @Inject constr private fun getKotlinBuildDir(task: T): Provider = task.project.layout.buildDirectory.dir("$KOTLIN_BUILD_DIR_NAME/${task.name}") - protected fun getClasspathSnapshotDir(task: T): Provider = + protected open fun getClasspathSnapshotDir(task: T): Provider = getKotlinBuildDir(task).map { it.dir("classpath-snapshot") } }