KT-45777: Take coarse-grained snapshots of external libraries

to reduce the size of the snapshots.

- Track metrics for Gradle classpath snapshot artifact transform
- Format size metrics so it's more readable

^KT-45777 In Progress

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshot.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.incremental.classpathDiff
 
 import org.jetbrains.kotlin.incremental.KotlinClassInfo
+import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity.CLASS_LEVEL
 import org.jetbrains.kotlin.incremental.classpathDiff.ClassSnapshotGranularity.CLASS_MEMBER_LEVEL
 import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader.Kind.*
 import org.jetbrains.kotlin.name.ClassId
@@ -141,7 +142,18 @@ class JavaElementSnapshotForTests(
  */
 object InaccessibleClassSnapshot : ClassSnapshot()
 
-/** The granularity of a [ClassSnapshot]. */
+/**
+ * The granularity of a [ClassSnapshot].
+ *
+ * There are currently two granularity levels:
+ *   - [CLASS_LEVEL]) (coarse-grained): The size of the snapshot will be smaller, but we will have coarse-grained classpath changes, which
+ *     means more source files will be recompiled.
+ *   - [CLASS_MEMBER_LEVEL] (fine-grained): The size of the snapshot will be larger, but we will have fine-grained classpath changes, which
+ *     means fewer source files will be recompiled.
+ *
+ * Therefore, [CLASS_LEVEL] is typically suitable for classes that are infrequently changed (e.g., external libraries), whereas
+ * [CLASS_MEMBER_LEVEL] is suitable for classes that are frequently changed (e.g., classes produced by the current project).
+ */
 enum class ClassSnapshotGranularity {
 
     /**

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotShrinker.kt

@@ -207,12 +207,12 @@ internal fun shrinkAndSaveClasspathSnapshot(
                 ShrinkMode.NoChanges
             } else {
                 val shrunkPreviousClasspathAgainstPreviousLookups =
-                    metrics.measure(BuildTime.LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+                    metrics.measure(BuildTime.INCREMENTAL_LOAD_SHRUNK_PREVIOUS_CLASSPATH_SNAPSHOT) {
                         ListExternalizer(AccessibleClassSnapshotExternalizer)
                             .loadFromFile(classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile)
                     }
                 ShrinkMode.Incremental(
-                    currentClasspathSnapshot = metrics.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+                    currentClasspathSnapshot = metrics.measure(BuildTime.INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT) {
                         CachedClasspathSnapshotSerializer
                             .load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
                             .removeDuplicateAndInaccessibleClasses()
@@ -243,7 +243,7 @@ internal fun shrinkAndSaveClasspathSnapshot(
             // shrunkCurrentClasspathAgainst[*Current*]Lookups == shrunkCurrentClasspathAgainst[*Previous*]Lookups
             shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups
         }
-        is ShrinkMode.Incremental -> metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+        is ShrinkMode.Incremental -> metrics.measure(BuildTime.INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT) {
             val shrunkClasses = shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups.mapTo(mutableSetOf()) { it.classId }
             val notYetShrunkClasses = shrinkMode.currentClasspathSnapshot.filter { it.classId !in shrunkClasses }
             val addedLookupSymbols = shrinkMode.addedLookupSymbols
@@ -252,12 +252,12 @@ internal fun shrinkAndSaveClasspathSnapshot(
             shrinkMode.shrunkCurrentClasspathAgainstPreviousLookups + shrunkRemainingClassesAgainstNewLookups
         }
         is ShrinkMode.NonIncremental -> {
-            val classpathSnapshot = metrics.measure(BuildTime.LOAD_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+            val classpathSnapshot = metrics.measure(BuildTime.NON_INCREMENTAL_LOAD_CURRENT_CLASSPATH_SNAPSHOT) {
                 CachedClasspathSnapshotSerializer
                     .load(classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles)
                     .removeDuplicateAndInaccessibleClasses()
             }
-            metrics.measure(BuildTime.SHRINK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+            metrics.measure(BuildTime.NON_INCREMENTAL_SHRINK_CURRENT_CLASSPATH_SNAPSHOT) {
                 shrinkClasspath(classpathSnapshot, lookupStorage)
             }
         }
@@ -269,7 +269,7 @@ internal fun shrinkAndSaveClasspathSnapshot(
             "File '${classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile.path}' does not exist"
         }
     } else {
-        metrics.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT_AFTER_COMPILATION) {
+        metrics.measure(BuildTime.SAVE_SHRUNK_CURRENT_CLASSPATH_SNAPSHOT) {
             ListExternalizer(AccessibleClassSnapshotExternalizer).saveToFile(
                 classpathChanges.classpathSnapshotFiles.shrunkPreviousClasspathSnapshotFile,
                 shrunkCurrentClasspath!!
@@ -277,8 +277,13 @@ internal fun shrinkAndSaveClasspathSnapshot(
         }
     }
 
+    metrics.addMetric(BuildPerformanceMetric.SHRINK_AND_SAVE_CLASSPATH_SNAPSHOT_EXECUTION_COUNT, 1)
     metrics.addMetric(
-        BuildPerformanceMetric.ORIGINAL_CLASSPATH_SNAPSHOT_SIZE,
+        BuildPerformanceMetric.CLASSPATH_ENTRY_COUNT,
+        classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.size.toLong()
+    )
+    metrics.addMetric(
+        BuildPerformanceMetric.CLASSPATH_SNAPSHOT_SIZE,
         classpathChanges.classpathSnapshotFiles.currentClasspathEntrySnapshotFiles.sumOf { it.length() }
     )
     metrics.addMetric(

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/classpathDiff/ClasspathSnapshotter.kt

@@ -5,6 +5,10 @@
 
 package org.jetbrains.kotlin.incremental.classpathDiff
 
+import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
+import org.jetbrains.kotlin.build.report.metrics.BuildTime
+import org.jetbrains.kotlin.build.report.metrics.DoNothingBuildMetricsReporter
+import org.jetbrains.kotlin.build.report.metrics.measure
 import org.jetbrains.kotlin.incremental.ChangesCollector.Companion.getNonPrivateMemberNames
 import org.jetbrains.kotlin.incremental.KotlinClassInfo
 import org.jetbrains.kotlin.incremental.PackagePartProtoData
@@ -26,14 +30,22 @@ object ClasspathEntrySnapshotter {
                 && !unixStyleRelativePath.equals("module-info.class", ignoreCase = true)
     }
 
-    fun snapshot(classpathEntry: File): ClasspathEntrySnapshot {
-        val classes = DirectoryOrJarContentsReader
-            .read(classpathEntry, DEFAULT_CLASS_FILTER)
-            .map { (unixStyleRelativePath, contents) ->
-                ClassFileWithContents(ClassFile(classpathEntry, unixStyleRelativePath), contents)
-            }
+    fun snapshot(
+        classpathEntry: File,
+        granularity: ClassSnapshotGranularity,
+        metrics: BuildMetricsReporter = DoNothingBuildMetricsReporter
+    ): ClasspathEntrySnapshot {
+        val classes = metrics.measure(BuildTime.LOAD_CLASSES) {
+            DirectoryOrJarContentsReader
+                .read(classpathEntry, DEFAULT_CLASS_FILTER)
+                .map { (unixStyleRelativePath, contents) ->
+                    ClassFileWithContents(ClassFile(classpathEntry, unixStyleRelativePath), contents)
+                }
+        }
 
-        val snapshots = ClassSnapshotter.snapshot(classes)
+        val snapshots = metrics.measure(BuildTime.SNAPSHOT_CLASSES) {
+            ClassSnapshotter.snapshot(classes, granularity, metrics = metrics)
+        }
 
         val relativePathsToSnapshotsMap = classes.map { it.classFile.unixStyleRelativePath }.zip(snapshots).toMap(LinkedHashMap())
         return ClasspathEntrySnapshot(relativePathsToSnapshotsMap)
@@ -47,17 +59,24 @@ object ClassSnapshotter {
     fun snapshot(
         classes: List<ClassFileWithContents>,
         granularity: ClassSnapshotGranularity = CLASS_MEMBER_LEVEL,
-        includeDebugInfoInJavaSnapshot: Boolean = false
+        includeDebugInfoInJavaSnapshot: Boolean = false,
+        metrics: BuildMetricsReporter = DoNothingBuildMetricsReporter
     ): List<ClassSnapshot> {
-        // Find inaccessible classes first
-        val classesInfo: List<BasicClassInfo> = classes.map { it.classInfo }
-        val inaccessibleClassesInfo: Set<BasicClassInfo> = getInaccessibleClasses(classesInfo).toSet()
-
+        val classesInfo: List<BasicClassInfo> = metrics.measure(BuildTime.READ_CLASSES_BASIC_INFO) {
+            classes.map { it.classInfo }
+        }
+        val inaccessibleClassesInfo: Set<BasicClassInfo> = metrics.measure(BuildTime.FIND_INACCESSIBLE_CLASSES) {
+            findInaccessibleClasses(classesInfo)
+        }
         return classes.map {
             when {
                 it.classInfo in inaccessibleClassesInfo -> InaccessibleClassSnapshot
-                it.classInfo.isKotlinClass -> snapshotKotlinClass(it, granularity)
-                else -> JavaClassSnapshotter.snapshot(it, granularity, includeDebugInfoInJavaSnapshot)
+                it.classInfo.isKotlinClass -> metrics.measure(BuildTime.SNAPSHOT_KOTLIN_CLASSES) {
+                    snapshotKotlinClass(it, granularity)
+                }
+                else -> metrics.measure(BuildTime.SNAPSHOT_JAVA_CLASSES) {
+                    JavaClassSnapshotter.snapshot(it, granularity, includeDebugInfoInJavaSnapshot)
+                }
             }
         }
     }
@@ -96,7 +115,7 @@ object ClassSnapshotter {
      * NOTE: If we do not have enough info to determine whether a class is inaccessible, we will assume that the class is accessible to be
      * safe.
      */
-    private fun getInaccessibleClasses(classesInfo: List<BasicClassInfo>): List<BasicClassInfo> {
+    private fun findInaccessibleClasses(classesInfo: List<BasicClassInfo>): Set<BasicClassInfo> {
         fun BasicClassInfo.isInaccessible(): Boolean {
             return if (this.isKotlinClass) {
                 when (this.kotlinClassHeader!!.kind) {
@@ -130,7 +149,7 @@ object ClassSnapshotter {
             }
         }
 
-        return classesInfo.filter { it.isTransitivelyInaccessible() }
+        return classesInfo.filterTo(mutableSetOf()) { it.isTransitivelyInaccessible() }
     }
 }