azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

KT-45777: Count cache misses when loading classpath snapshot

Browse Source 
to get more insights into the build performance of that step.

^KT-45777 In Progress
This commit is contained in:
Hung Nguyen
2022-03-24 20:32:07 +00:00
committed by teamcity
parent f23e66a6e2
commit bd8f49c5d6
7 changed files with 40 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-common/src/org/jetbrains/kotlin/build/report/metrics/BuildPerformanceMetric.kt
+2
View File
@@ -20,10 +20,12 @@ enum class BuildPerformanceMetric(val parent: BuildPerformanceMetric? = null, va
JAR_CLASSPATH_ENTRY_SIZE(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, "Size of jar classpath entry", type = SizeMetricType.BYTES),
JAR_CLASSPATH_ENTRY_SNAPSHOT_SIZE(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, "Size of jar classpath entry's snapshot", type = SizeMetricType.BYTES),
DIRECTORY_CLASSPATH_ENTRY_SNAPSHOT_SIZE(parent = CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, "Size of directory classpath entry's snapshot", type = SizeMetricType.BYTES),
COMPUTE_CLASSPATH_CHANGES_EXECUTION_COUNT(parent = null, "Number of times classpath changes are computed", type = SizeMetricType.NUMBER),
SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT(parent = null, "Number of times classpath snapshot is shrunk and saved after compilation", type = SizeMetricType.NUMBER),
CLASSPATH_ENTRY_COUNT(parent = SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Number of classpath entries", type = SizeMetricType.NUMBER),
CLASSPATH_SNAPSHOT_SIZE(parent = SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Size of classpath snapshot", type = SizeMetricType.BYTES),
SHRUNK_CLASSPATH_SNAPSHOT_SIZE(parent = SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, "Size of shrunk classpath snapshot", type = SizeMetricType.BYTES),
LOAD_CLASSPATH_SNAPSHOT_CACHE_MISSES(parent = null, "Number of cache misses when loading classpath snapshot", type = SizeMetricType.NUMBER),
;
companion object {
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt
+6 -8
View File
@@ -29,10 +29,7 @@ import org.jetbrains.kotlin.build.GeneratedJvmClass
import org.jetbrains.kotlin.build.report.BuildReporter
import org.jetbrains.kotlin.build.report.DoNothingICReporter
import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.build.report.metrics.BuildAttribute
import org.jetbrains.kotlin.build.report.metrics.BuildTime
import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.measure
import org.jetbrains.kotlin.build.report.metrics.*
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
@@ -213,8 +210,10 @@ class IncrementalJvmCompilerRunner(
return abiSnapshots
}
// Used by `calculateSourcesToCompileImpl` and `performWorkAfterSuccessfulCompilation` methods below.
// Thread safety: There is no concurrent access to these variables.
// There are 2 steps where we need to load the current classpath snapshot and shrink it:
// - Before classpath diffing when `classpathChanges` is ToBeComputedByIncrementalCompiler (see `calculateSourcesToCompileImpl`)
// - After compilation (see `performWorkAfterSuccessfulCompilation`)
// To avoid duplicated work, we store the snapshots after the first step for reuse (if the first step is executed).
private var currentClasspathSnapshot: List<AccessibleClassSnapshot>? = null
private var shrunkCurrentClasspathAgainstPreviousLookups: List<AccessibleClassSnapshot>? = null
@@ -233,11 +232,10 @@ class IncrementalJvmCompilerRunner(
// Note: classpathChanges is deserialized, so they are no longer singleton objects and need to be compared using `is` (not `==`)
is NoChanges -> ChangesEither.Known(emptySet(), emptySet())
is ToBeComputedByIncrementalCompiler -> reporter.measure(BuildTime.COMPUTE_CLASSPATH_CHANGES) {
reporter.addMetric(BuildPerformanceMetric.COMPUTE_CLASSPATH_CHANGES_EXECUTION_COUNT, 1)
val storeCurrentClasspathSnapshotForReuse =
{ currentClasspathSnapshotArg: List<AccessibleClassSnapshot>,
shrunkCurrentClasspathAgainstPreviousLookupsArg: List<AccessibleClassSnapshot> ->
check(currentClasspathSnapshot == null)
check(shrunkCurrentClasspathAgainstPreviousLookups == null)
currentClasspathSnapshot = currentClasspathSnapshotArg
shrunkCurrentClasspathAgainstPreviousLookups = shrunkCurrentClasspathAgainstPreviousLookupsArg
}
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathChangesComputer.kt
+2 -1
View File
@@ -37,7 +37,8 @@ object ClasspathChangesComputer {
reporter: ClasspathSnapshotBuildReporter
): ProgramSymbolSet {
val currentClasspathSnapshot = reporter.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT) {
val classpathSnapshot = CachedClasspathSnapshotSerializer.load(classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
val classpathSnapshot =
CachedClasspathSnapshotSerializer.load(classpathSnapshotFiles.currentClasspathEntrySnapshotFiles, reporter)
reporter.measure(BuildTime.REMOVE_DUPLICATE_CLASSES) {
classpathSnapshot.removeDuplicateAndInaccessibleClasses()
}
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotSerializer.kt
+9 -1
View File
@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.incremental.classpathDiff
import com.intellij.util.io.DataExternalizer
import org.jetbrains.kotlin.build.report.metrics.BuildPerformanceMetric
import org.jetbrains.kotlin.incremental.KotlinClassInfo
import org.jetbrains.kotlin.incremental.storage.*
import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
@@ -19,9 +20,16 @@ object CachedClasspathSnapshotSerializer {
private val cache = ConcurrentHashMap<File, ClasspathEntrySnapshot>()
private const val RECOMMENDED_MAX_CACHE_SIZE = 100
fun load(classpathEntrySnapshotFiles: List<File>): ClasspathSnapshot {
fun load(classpathEntrySnapshotFiles: List<File>, reporter: ClasspathSnapshotBuildReporter): ClasspathSnapshot {
return ClasspathSnapshot(classpathEntrySnapshotFiles.map { snapshotFile ->
// Note: This cache is shared across builds, so we need to be careful if the snapshot file's path hasn't changed but its
// contents have changed. Luckily, each snapshot file is the output of a Gradle transform and Gradle has the hash of the file's
// contents encoded in the file's path. Therefore, if the file's path hasn't changed, then its contents must also not have
// changed, and we can reuse the cache entry.
// TODO: Make this code safer (not relying on how the snapshot files are produced and whether Gradle maintains the above
// guarantee). For example, maybe we can write the file's content hash in the file's name or to another file next to it.
cache.computeIfAbsent(snapshotFile) {
reporter.addMetric(BuildPerformanceMetric.LOAD_CLASSPATH_SNAPSHOT_CACHE_MISSES, 1)
ClasspathEntrySnapshotExternalizer.loadFromFile(it)
}
}).also {
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt
+11 -11
View File
@@ -201,8 +201,8 @@ private sealed class ShrinkMode {
internal fun shrinkAndSaveClasspathSnapshot(
classpathChanges: ClasspathChanges.ClasspathSnapshotEnabled,
lookupStorage: LookupStorage,
precomputedCurrentClasspathSnapshot: List<AccessibleClassSnapshot>?, // Not null iff classpathChanges is ToBeComputedByIncrementalCompiler
precomputedShrunkCurrentClasspathAgainstPreviousLookups: List<AccessibleClassSnapshot>?, // Not null iff classpathChanges is ToBeComputedByIncrementalCompiler
currentClasspathSnapshot: List<AccessibleClassSnapshot>?, // Not null iff classpathChanges is ToBeComputedByIncrementalCompiler
shrunkCurrentClasspathAgainstPreviousLookups: List<AccessibleClassSnapshot>?, // Not null iff classpathChanges is ToBeComputedByIncrementalCompiler
reporter: ClasspathSnapshotBuildReporter
) {
// In the following, we'll try to shrink the classpath snapshot incrementally when possible.
@@ -222,14 +222,14 @@ internal fun shrinkAndSaveClasspathSnapshot(
val addedLookupSymbols = lookupStorage.addedLookupSymbols
if (addedLookupSymbols.isEmpty()) {
ShrinkMode.UnchangedLookupsChangedClasspath(
precomputedCurrentClasspathSnapshot!!,
precomputedShrunkCurrentClasspathAgainstPreviousLookups!!
currentClasspathSnapshot!!,
shrunkCurrentClasspathAgainstPreviousLookups!!
)
} else {
ShrinkMode.ChangedLookupsChangedClasspath(
addedLookupSymbols,
precomputedCurrentClasspathSnapshot!!,
precomputedShrunkCurrentClasspathAgainstPreviousLookups!!
currentClasspathSnapshot!!,
shrunkCurrentClasspathAgainstPreviousLookups!!
)
}
}
@@ -254,12 +254,12 @@ internal fun shrinkAndSaveClasspathSnapshot(
when (shrinkMode) {
is ShrinkMode.ChangedLookupsUnchangedClasspath ->
CachedClasspathSnapshotSerializer
.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles, reporter)
.removeDuplicateAndInaccessibleClasses()
is ShrinkMode.ChangedLookupsChangedClasspath -> shrinkMode.currentClasspathSnapshot
}
}
val shrunkCurrentClasspathAgainstPreviousLookups =
val shrunkCurrentClasspathAgainstPrevLookups =
reporter.measure(BuildTime.INCREMENTAL_LOAD_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AGAINST_PREVIOUS_LOOKUPS) {
when (shrinkMode) {
is ShrinkMode.ChangedLookupsUnchangedClasspath -> {
@@ -272,11 +272,11 @@ internal fun shrinkAndSaveClasspathSnapshot(
}
}
val shrunkClasses = shrunkCurrentClasspathAgainstPreviousLookups.mapTo(mutableSetOf()) { it.classId }
val shrunkClasses = shrunkCurrentClasspathAgainstPrevLookups.mapTo(mutableSetOf()) { it.classId }
val notYetShrunkClasses = currentClasspath.filter { it.classId !in shrunkClasses }
val shrunkRemainingClassesAgainstNewLookups = shrinkClasses(notYetShrunkClasses, shrinkMode.addedLookupSymbols)
val shrunkCurrentClasspath = shrunkCurrentClasspathAgainstPreviousLookups + shrunkRemainingClassesAgainstNewLookups
val shrunkCurrentClasspath = shrunkCurrentClasspathAgainstPrevLookups + shrunkRemainingClassesAgainstNewLookups
currentClasspath to shrunkCurrentClasspath
}
is ShrinkMode.NonIncremental -> {
@@ -284,7 +284,7 @@ internal fun shrinkAndSaveClasspathSnapshot(
reporter.measure(BuildTime.NON_INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT) {
val currentClasspath = reporter.measure(BuildTime.NON_INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT) {
CachedClasspathSnapshotSerializer
.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
.load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles, reporter)
.removeDuplicateAndInaccessibleClasses()
}
val shrunkCurrentClasspath = shrinkClasspath(currentClasspath, lookupStorage)
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/transforms/ClasspathEntrySnapshotTransform.kt
+8 -8
View File
@@ -91,15 +91,15 @@ abstract class ClasspathEntrySnapshotTransform : TransformAction<ClasspathEntryS
metrics.measure(BuildTime.SAVE_CLASSPATH_ENTRY_SNAPSHOT) {
ClasspathEntrySnapshotExternalizer.saveToFile(snapshotOutputFile, snapshot)
}
}
metrics.addMetric(BuildPerformanceMetric.CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, 1)
if (classpathEntryInputDirOrJar.extension.equals("jar", ignoreCase = true)) {
metrics.addMetric(BuildPerformanceMetric.JAR_CLASSPATH_ENTRY_SIZE, classpathEntryInputDirOrJar.length())
metrics.addMetric(BuildPerformanceMetric.JAR_CLASSPATH_ENTRY_SNAPSHOT_SIZE, snapshotOutputFile.length())
} else {
// Only compute the size of the snapshot, not the size of the input directory as walking the file tree has a small overhead
metrics.addMetric(BuildPerformanceMetric.DIRECTORY_CLASSPATH_ENTRY_SNAPSHOT_SIZE, snapshotOutputFile.length())
}
metrics.addMetric(BuildPerformanceMetric.CLASSPATH_ENTRY_SNAPSHOT_TRANSFORM_EXECUTION_COUNT, 1)
if (classpathEntryInputDirOrJar.extension.equals("jar", ignoreCase = true)) {
metrics.addMetric(BuildPerformanceMetric.JAR_CLASSPATH_ENTRY_SIZE, classpathEntryInputDirOrJar.length())
metrics.addMetric(BuildPerformanceMetric.JAR_CLASSPATH_ENTRY_SNAPSHOT_SIZE, snapshotOutputFile.length())
} else {
// Only compute the size of the snapshot, not the size of the input directory as walking the file tree has a small overhead
metrics.addMetric(BuildPerformanceMetric.DIRECTORY_CLASSPATH_ENTRY_SNAPSHOT_SIZE, snapshotOutputFile.length())
}
}
}
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/Tasks.kt
+2 -2
View File
@@ -282,7 +282,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
}
).disallowChanges()
task.taskBuildCacheableOutputDirectory.value(getKotlinBuildDir(task).map { it.dir("cacheable") }).disallowChanges()
task.taskBuildLocalStateDirectory.value(getKotlinBuildDir(task).map { it.dir("localstate") }).disallowChanges()
task.taskBuildLocalStateDirectory.value(getKotlinBuildDir(task).map { it.dir("local-state") }).disallowChanges()
task.localStateDirectories.from(task.taskBuildLocalStateDirectory).disallowChanges()
@@ -292,7 +292,7 @@ abstract class AbstractKotlinCompile<T : CommonCompilerArguments> @Inject constr
private fun getKotlinBuildDir(task: T): Provider<Directory> =
task.project.layout.buildDirectory.dir("$KOTLIN_BUILD_DIR_NAME/${task.name}")
protected open fun getClasspathSnapshotDir(task: T): Provider<Directory> =
protected fun getClasspathSnapshotDir(task: T): Provider<Directory> =
getKotlinBuildDir(task).map { it.dir("classpath-snapshot") }
}