New IC: Detect changes to class annotations

Both the new and old incremental compilation (IC) analysis rely on
Kotlin class metadata to detect a change.

However, Kotlin metadata currently doesn't contain info about
annotations (KT-57919), so the IC will not be able to detect a change
to them.

With this commit, we'll fix the new IC such that it can detect a change
to class annotations by not relying only on metadata.

We currently scope this fix to the new IC (cross-module analysis) first.
We'll fix this issue for within-module analysis later.

Performance: There seems to be no performance impact from this change.
Snapshotting the 400MB ideaIC-2022.1.4/app.jar takes 4.1s before and
after this change.

Test: Added ClasspathChangesComputerTest.testChangedAnnotations
^KT-58289: Fixed

build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt

@@ -441,7 +441,7 @@ open class IncrementalJvmCache(
         storageFile: File,
         icContext: IncrementalCompilationContext,
     ) :
-        BasicStringMap<Map<String, Any>>(storageFile, MapExternalizer(StringExternalizer, ConstantValueExternalizer), icContext) {
+        BasicStringMap<Map<String, Long>>(storageFile, MapExternalizer(StringExternalizer, LongExternalizer), icContext) {
 
         operator fun contains(className: JvmClassName): Boolean =
             className.internalName in storage
@@ -451,7 +451,7 @@ open class IncrementalJvmCache(
             val key = kotlinClassInfo.className.internalName
             val oldMap = storage[key] ?: emptyMap()
 
-            val newMap = kotlinClassInfo.constantsMap
+            val newMap = kotlinClassInfo.extraInfo.constantSnapshots
             if (newMap.isNotEmpty()) {
                 storage[key] = newMap
             } else {
@@ -489,8 +489,8 @@ open class IncrementalJvmCache(
             storage.remove(className.internalName)
         }
 
-        override fun dumpValue(value: Map<String, Any>): String =
-            value.dumpMap(Any::toString)
+        override fun dumpValue(value: Map<String, Long>): String =
+            value.dumpMap(Long::toString)
     }
 
     private inner class PackagePartMap(
@@ -592,7 +592,7 @@ open class IncrementalJvmCache(
             val key = kotlinClassInfo.className.internalName
             val oldMap = storage[key] ?: emptyMap()
 
-            val newMap = kotlinClassInfo.inlineFunctionsAndAccessorsMap
+            val newMap = kotlinClassInfo.extraInfo.inlineFunctionOrAccessorSnapshots
             if (newMap.isNotEmpty()) {
                 storage[key] = newMap
             } else {

build-common/src/org/jetbrains/kotlin/incremental/KotlinClassInfo.kt

@@ -5,8 +5,12 @@
 
 package org.jetbrains.kotlin.incremental
 
-import com.intellij.util.io.DataExternalizer
-import org.jetbrains.kotlin.incremental.storage.*
+import org.jetbrains.kotlin.incremental.ClassNodeSnapshotter.snapshotClassExcludingMembers
+import org.jetbrains.kotlin.incremental.ClassNodeSnapshotter.snapshotField
+import org.jetbrains.kotlin.incremental.ClassNodeSnapshotter.snapshotMethod
+import org.jetbrains.kotlin.incremental.ClassNodeSnapshotter.sortClassMembers
+import org.jetbrains.kotlin.incremental.KotlinClassInfo.ExtraInfo
+import org.jetbrains.kotlin.incremental.storage.ProtoMapValue
 import org.jetbrains.kotlin.inline.InlineFunctionOrAccessor
 import org.jetbrains.kotlin.inline.inlineFunctionsAndAccessors
 import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
@@ -16,6 +20,9 @@ import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 import org.jetbrains.org.objectweb.asm.*
+import org.jetbrains.org.objectweb.asm.tree.ClassNode
+import org.jetbrains.org.objectweb.asm.tree.FieldNode
+import org.jetbrains.org.objectweb.asm.tree.MethodNode
 
 /**
  * Minimal information about a Kotlin class to compute recompilation-triggering changes during an incremental run of the `KotlinCompile`
@@ -25,16 +32,34 @@ import org.jetbrains.org.objectweb.asm.*
  * `KotlinCompile` task and the task needs to support compile avoidance. For example, this class should contain public method signatures,
  * and should not contain private method signatures, or method implementations.
  */
-class KotlinClassInfo constructor(
+class KotlinClassInfo(
     val classId: ClassId,
     val classKind: KotlinClassHeader.Kind,
     val classHeaderData: Array<String>, // Can be empty
     val classHeaderStrings: Array<String>, // Can be empty
     val multifileClassName: String?, // Not null iff classKind == KotlinClassHeader.Kind.MULTIFILE_CLASS_PART
-    val constantsMap: Map<String, Any>,
-    val inlineFunctionsAndAccessorsMap: Map<InlineFunctionOrAccessor, Long>
+    val extraInfo: ExtraInfo
 ) {
 
+    /** Extra information about a Kotlin class that is not captured in the Kotlin class metadata. */
+    class ExtraInfo(
+
+        /**
+         * Snapshot of the class excluding its fields and methods and Kotlin metadata (iff classKind == [KotlinClassHeader.Kind.CLASS]).
+         *
+         * For example, the class's annotations which are currently not captured by Kotlin metadata (KT-57919) will be captured here.
+         *
+         * Note: It also excludes Kotlin metadata as [ExtraInfo] should only contain info not yet captured in Kotlin metadata.
+         */
+        val classSnapshotExcludingMembers: Long?,
+
+        /** Snapshots of the class's constants (including their values). The map's keys are the constants' names. */
+        val constantSnapshots: Map<String, Long>,
+
+        /** Snapshots of the class's inline functions and property accessors (including their implementation). */
+        val inlineFunctionOrAccessorSnapshots: Map<InlineFunctionOrAccessor, Long>
+    )
+
     val className: JvmClassName by lazy { JvmClassName.byClassId(classId) }
 
     val protoMapValue: ProtoMapValue by lazy {
@@ -84,123 +109,148 @@ class KotlinClassInfo constructor(
         }
 
         fun createFrom(classId: ClassId, classHeader: KotlinClassHeader, classContents: ByteArray): KotlinClassInfo {
-            val (constants, inlineFunctionsAndAccessors) = getConstantsAndInlineFunctionsOrAccessors(classHeader, classContents)
-
             return KotlinClassInfo(
                 classId,
                 classHeader.kind,
                 classHeader.data ?: classHeader.incompatibleData ?: emptyArray(),
                 classHeader.strings ?: emptyArray(),
                 classHeader.multifileClassName,
-                constants.mapKeys { it.key.name },
-                inlineFunctionsAndAccessors
+                extraInfo = getExtraInfo(classHeader, classContents)
             )
         }
     }
 }
 
-/**
- * Parses the class file only once to get both constants and inline functions/property accessors. This is faster than getting them
- * separately in two passes.
- */
-private fun getConstantsAndInlineFunctionsOrAccessors(
-    classHeader: KotlinClassHeader,
-    classContents: ByteArray
-): Pair<Map<JvmMemberSignature.Field, Any>, Map<InlineFunctionOrAccessor, Long>> {
-    val constantsClassVisitor = ConstantsClassVisitor()
-    val inlineFunctionsAndAccessors = inlineFunctionsAndAccessors(classHeader, excludePrivateMembers = true)
+private fun getExtraInfo(classHeader: KotlinClassHeader, classContents: ByteArray): ExtraInfo {
+    // Get the list of (non-private) inline functions and accessors from Kotlin class metadata, then find and snapshot them in the bytecode.
+    // Note:
+    //   - Some of them may not be found in the bytecode. Specifically, internal/private inline functions/accessors may be removed from the
+    // bytecode if code shrinker is used. For example, `kotlin-reflect-1.7.20.jar` contains `/kotlin/reflect/jvm/internal/UtilKt.class` in
+    // which the internal inline function `reflectionCall` appears in the Kotlin class metadata (also in the source file), but not in the
+    // bytecode. When that happens, we will ignore those methods. It is safe to ignore because the methods are not declared in the bytecode
+    // and therefore can't be referenced.
+    //   - Look for private methods as well because a *public* inline function/accessor may have a *private* corresponding method in the
+    // bytecode (see `InlineOnlyKt.isInlineOnlyPrivateInBytecode`).
+    val inlineFunctionsAndAccessors: List<InlineFunctionOrAccessor> = inlineFunctionsAndAccessors(classHeader, excludePrivateMembers = true)
+    val inlineFunctionOrAccessorSignatures: Map<JvmMemberSignature.Method, InlineFunctionOrAccessor> =
+        inlineFunctionsAndAccessors.associateBy { it.jvmMethodSignature }
 
-    return if (inlineFunctionsAndAccessors.isEmpty()) {
-        // Handle this case differently to improve performance
-        // parsingOptions = (SKIP_CODE, SKIP_DEBUG) as method bodies and debug info are not important for constants
-        ClassReader(classContents).accept(constantsClassVisitor, ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG)
-        Pair(constantsClassVisitor.getResult(), emptyMap())
+    val parsingOptions = if (inlineFunctionsAndAccessors.isNotEmpty()) {
+        // Do not pass (SKIP_CODE, SKIP_DEBUG) as method bodies and debug info (e.g., line numbers) are important for inline
+        // functions/accessors
+        0
     } else {
-        val inlineFunctionsAndAccessorsClassVisitor = InlineFunctionsAndAccessorsClassVisitor(
-            inlineFunctionsAndAccessors.map { it.jvmMethodSignature }.toSet(),
-            constantsClassVisitor
-        )
-        // parsingOptions must not include (SKIP_CODE, SKIP_DEBUG) as method bodies and debug info (e.g., line numbers) are important for
+        // Pass (SKIP_CODE, SKIP_DEBUG) to improve performance as method bodies and debug info are not important when we're not analyzing
         // inline functions/accessors
-        ClassReader(classContents).accept(inlineFunctionsAndAccessorsClassVisitor, 0)
-        val constantsMap = constantsClassVisitor.getResult()
-        val methodHashesMap = inlineFunctionsAndAccessorsClassVisitor.getResult()
-        val inlineFunctionsAndAccessorsMap = inlineFunctionsAndAccessors.mapNotNull { inline ->
-            // Note that internal/private inline functions may be removed from the bytecode if code shrinker is used. For example,
-            // `kotlin-reflect-1.7.20.jar` contains `/kotlin/reflect/jvm/internal/UtilKt.class` in which the internal inline function
-            // `reflectionCall` appears in the Kotlin metadata (also in the source file), but not in the bytecode.
-            // When that happens (i.e., when the map lookup below returns null), we will ignore the method. It is safe to ignore because the
-            // method is not declared in the bytecode and therefore can't be referenced.
-            methodHashesMap[inline.jvmMethodSignature]?.let { inline to it }
-        }.toMap()
-        Pair(constantsMap, inlineFunctionsAndAccessorsMap)
+        ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG
     }
-}
 
-private class ConstantsClassVisitor : ClassVisitor(Opcodes.API_VERSION) {
-    private val result = mutableMapOf<JvmMemberSignature.Field, Any>()
+    // Load class contents into a `ClassNode`.
+    // Note that we'll only load methods that are inline functions/accessors (including private methods -- see comment at the top of
+    // `getExtraInfo`) as we don't need to snapshot the other methods when computing `ExtraInfo`.
+    val classNode = ClassNode()
+    val classReader = ClassReader(classContents)
+    classReader.accept(SkipMethodClassVisitor(classNode) { it !in inlineFunctionOrAccessorSignatures.keys }, parsingOptions)
+    sortClassMembers(classNode)
 
-    override fun visitField(access: Int, name: String, desc: String, signature: String?, value: Any?): FieldVisitor? {
-        if (access and Opcodes.ACC_PRIVATE == Opcodes.ACC_PRIVATE) return null
+    // Snapshot the class excluding its fields and methods and metadata
+    val classSnapshotExcludingMembers = if (classHeader.kind == KotlinClassHeader.Kind.CLASS) {
+        // Also exclude Kotlin metadata (see `ExtraInfo.classSnapshotExcludingMembers`'s kdoc)
+        snapshotClassExcludingMembers(classNode, alsoExcludeKotlinMetaData = true)
+    } else null
 
-        val staticFinal = Opcodes.ACC_STATIC or Opcodes.ACC_FINAL
-        if (value != null && access and staticFinal == staticFinal) {
-            result[JvmMemberSignature.Field(name, desc)] = value
+    // Snapshot constants
+    fun FieldNode.isPrivate() = (access and Opcodes.ACC_PRIVATE) != 0
+    fun FieldNode.isStaticFinal() = (access and (Opcodes.ACC_STATIC or Opcodes.ACC_FINAL)) == (Opcodes.ACC_STATIC or Opcodes.ACC_FINAL)
+
+    val constantSnapshots: Map<String, Long> = classNode.fields
+        .filter { !it.isPrivate() && it.isStaticFinal() }
+        .associate { it.name to snapshotField(it) }
+
+    // Snapshot inline functions and accessors
+    fun MethodNode.signature() = JvmMemberSignature.Method(name = name, desc = desc)
+
+    val inlineFunctionOrAccessorSnapshots: Map<InlineFunctionOrAccessor, Long> = classNode.methods
+        .associate { methodNode ->
+            // `methodNode` must be an inline function/accessor because we loaded only inline functions/accessors into `classNode`
+            inlineFunctionOrAccessorSignatures[methodNode.signature()]!! to snapshotMethod(methodNode, classNode.version)
         }
-        return null
-    }
 
-    fun getResult() = result
+    return ExtraInfo(classSnapshotExcludingMembers, constantSnapshots, inlineFunctionOrAccessorSnapshots)
 }
 
-private class InlineFunctionsAndAccessorsClassVisitor(
-    private val inlineFunctionsAndAccessors: Set<JvmMemberSignature.Method>,
-    cv: ConstantsClassVisitor // Note: cv must not override `visitMethod` (it will not be called with the current implementation below)
+/** [ClassVisitor] which skips visiting methods where `[shouldSkipMethod] == true`. */
+private class SkipMethodClassVisitor(
+    cv: ClassVisitor,
+    private val shouldSkipMethod: (JvmMemberSignature.Method) -> Boolean,
 ) : ClassVisitor(Opcodes.API_VERSION, cv) {
 
-    private val result = mutableMapOf<JvmMemberSignature.Method, Long>()
-    private var classVersion: Int? = null
+    override fun visitMethod(access: Int, name: String, desc: String, signature: String?, exceptions: Array<out String>?): MethodVisitor? {
+        return if (shouldSkipMethod(JvmMemberSignature.Method(name, desc))) {
+            null
+        } else {
+            cv.visitMethod(access, name, desc, signature, exceptions)
+        }
+    }
+}
 
-    override fun visit(version: Int, access: Int, name: String, signature: String?, superName: String?, interfaces: Array<out String>?) {
-        super.visit(version, access, name, signature, superName, interfaces)
-        classVersion = version
+/** Computes the snapshot of a Java class represented by a [ClassNode]. */
+object ClassNodeSnapshotter {
+
+    fun snapshotClass(classNode: ClassNode): Long {
+        val classWriter = ClassWriter(0)
+        classNode.accept(classWriter)
+        return classWriter.toByteArray().hashToLong()
     }
 
-    override fun visitMethod(access: Int, name: String, desc: String, signature: String?, exceptions: Array<out String>?): MethodVisitor? {
-        // Note: Do not filter out private methods here because a *public* inline function may actually have a *private* corresponding JVM
-        // method in the bytecode (see `InlineOnlyKt.isInlineOnlyPrivateInBytecode`).
-        // Just filter the methods based on the given `inlineFunctionsAndAccessors` set.
-        val method = JvmMemberSignature.Method(name, desc)
-        if (method !in inlineFunctionsAndAccessors) return null
-
-        val classWriter = ClassWriter(0)
-
-        // The `version` and `name` parameters are important (see KT-38857), the others can be null.
-        classWriter.visit(/* version */ classVersion!!, /* access */ 0, /* name */ "ClassWithOneMethod", null, null, null)
-
-        return object : MethodVisitor(Opcodes.API_VERSION, classWriter.visitMethod(access, name, desc, signature, exceptions)) {
-            override fun visitEnd() {
-                result[method] = classWriter.toByteArray().md5()
-            }
+    fun snapshotClassExcludingMembers(classNode: ClassNode, alsoExcludeKotlinMetaData: Boolean = false): Long {
+        val originalFields = classNode.fields
+        val originalMethods = classNode.methods
+        val originalVisibleAnnotations = classNode.visibleAnnotations
+        classNode.fields = emptyList()
+        classNode.methods = emptyList()
+        if (alsoExcludeKotlinMetaData) {
+            classNode.visibleAnnotations = originalVisibleAnnotations?.filterNot { it.desc == "Lkotlin/Metadata;" }
+        }
+        return snapshotClass(classNode).also {
+            classNode.fields = originalFields
+            classNode.methods = originalMethods
+            classNode.visibleAnnotations = originalVisibleAnnotations
         }
     }
 
-    fun getResult() = result
+    fun snapshotField(fieldNode: FieldNode): Long {
+        val classNode = emptyClass()
+        classNode.fields.add(fieldNode)
+        return snapshotClass(classNode)
+    }
+
+    fun snapshotMethod(methodNode: MethodNode, classVersion: Int): Long {
+        val classNode = emptyClass()
+        classNode.version = classVersion // Class version is required for method bodies (see KT-38857)
+        classNode.methods.add(methodNode)
+        return snapshotClass(classNode)
+    }
+
+    /**
+     * Sorts fields and methods in the given class.
+     *
+     * This is useful when we want to ensure a change in the order of the fields and methods doesn't impact the snapshot (i.e., if their
+     * order has changed in the `.class` file, it shouldn't require recompilation of the other source files).
+     */
+    fun sortClassMembers(classNode: ClassNode) {
+        classNode.fields.sortWith(compareBy({ it.name }, { it.desc }))
+        classNode.methods.sortWith(compareBy({ it.name }, { it.desc }))
+    }
+
+    private fun emptyClass() = ClassNode().also {
+        // A name is required
+        it.name = "EmptyClass"
+    }
 }
 
-/**
- * [DataExternalizer] for the value of a Kotlin constant.
- *
- * The constants' values are provided by ASM (see the javadoc of [ConstantsClassVisitor.visitField]), so their types can only be the
- * following: Integer, Long, Float, Double, String. (Boolean constants have Integer (0, 1) values in ASM.)
- */
-object ConstantValueExternalizer : DataExternalizer<Any> by DelegateDataExternalizer(
-    listOf(
-        java.lang.Integer::class.java,
-        java.lang.Long::class.java,
-        java.lang.Float::class.java,
-        java.lang.Double::class.java,
-        java.lang.String::class.java
-    ),
-    listOf(IntExternalizer, LongExternalizer, FloatExternalizer, DoubleExternalizer, StringExternalizer)
-)
+fun ByteArray.hashToLong(): Long {
+    // Note: The returned type `Long` is 64-bit, but we currently don't have a good 64-bit hash function.
+    // The method below uses `md5` which is 128-bit and converts it to `Long`.
+    return md5()
+}

build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt

@@ -274,8 +274,7 @@ class DelegateDataExternalizer<T>(
 
     override fun read(input: DataInput): T {
         val typeIndex = input.readByte().toInt()
-        @Suppress("UNCHECKED_CAST")
-        return typesExternalizers[typeIndex].read(input) as T
+        return typesExternalizers[typeIndex].read(input)
     }
 }