add protoDifferenceUtils and implementation for calculation difference between proto data for classes.

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt

@@ -31,7 +31,6 @@ import org.jetbrains.jps.incremental.storage.BuildDataManager
 import org.jetbrains.jps.incremental.storage.PathStringDescriptor
 import org.jetbrains.jps.incremental.storage.StorageOwner
 import org.jetbrains.kotlin.config.IncrementalCompilation
-import org.jetbrains.kotlin.descriptors.Visibilities
 import org.jetbrains.kotlin.jps.build.GeneratedJvmClass
 import org.jetbrains.kotlin.jps.build.KotlinBuilder
 import org.jetbrains.kotlin.jps.incremental.storage.BasicMap
@@ -46,11 +45,7 @@ import org.jetbrains.kotlin.load.kotlin.header.isCompatiblePackageFacadeKind
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName.byInternalName
-import org.jetbrains.kotlin.serialization.Flags
-import org.jetbrains.kotlin.serialization.ProtoBuf
-import org.jetbrains.kotlin.serialization.deserialization.Deserialization
 import org.jetbrains.kotlin.serialization.jvm.BitEncoding
-import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
 import org.jetbrains.org.objectweb.asm.*
 import java.io.DataInput
 import java.io.DataInputStream
@@ -67,7 +62,7 @@ private val CACHE_DIRECTORY_NAME = "kotlin"
 class CacheFormatVersion(targetDataRoot: File) {
     companion object {
         // Change this when incremental cache format changes
-        private val INCREMENTAL_CACHE_OWN_VERSION = 4
+        private val INCREMENTAL_CACHE_OWN_VERSION = 5
 
         private val CACHE_FORMAT_VERSION =
                 INCREMENTAL_CACHE_OWN_VERSION * 1000000 +
@@ -301,11 +296,11 @@ public class IncrementalCacheImpl(
     }
 
     override fun getPackagePartData(fqName: String): ByteArray? {
-        return protoMap[JvmClassName.byInternalName(fqName)]
+        return protoMap[JvmClassName.byInternalName(fqName)]?.bytes
     }
 
     override fun getModuleMappingData(): ByteArray? {
-        return protoMap[JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)]
+        return protoMap[JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)]?.bytes
     }
 
     override fun flush(memoryCachesOnly: Boolean) {
@@ -321,7 +316,7 @@ public class IncrementalCacheImpl(
         maps.forEach { it.close () }
     }
 
-    private inner class ProtoMap(storageFile: File) : BasicStringMap<ByteArray>(storageFile, ByteArrayExternalizer) {
+    private inner class ProtoMap(storageFile: File) : BasicStringMap<ProtoMapValue>(storageFile, ProtoMapValueExternalizer) {
 
         public fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean, checkChangesIsOpenPart: Boolean = true): ChangesInfo {
             val header = kotlinClass.classHeader
@@ -333,20 +328,21 @@ public class IncrementalCacheImpl(
             return put(className, data, isPackage, checkChangesIsOpenPart)
         }
 
-        private fun put(className: JvmClassName, data: ByteArray, isPackage: Boolean, checkChangesIsOpenPart: Boolean): ChangesInfo {
+        private fun put(className: JvmClassName, bytes: ByteArray, isPackage: Boolean, checkChangesIsOpenPart: Boolean): ChangesInfo {
             val key = className.internalName
             val oldData = storage[key]
+            val data = ProtoMapValue(isPackage, bytes)
 
-            if (!Arrays.equals(data, oldData)) {
+            if (oldData == null || !Arrays.equals(bytes, oldData.bytes) || isPackage != oldData.isPackageFacade) {
                 storage.put(key, data)
             }
 
             return ChangesInfo(protoChanged = oldData == null ||
                                               !checkChangesIsOpenPart ||
-                                              !isOpenPartNotChanged(oldData, data, isPackage))
+                                              difference(oldData, data) != DifferenceKind.NONE)
         }
 
-        public fun get(className: JvmClassName): ByteArray? {
+        public fun get(className: JvmClassName): ProtoMapValue? {
             return storage[className.getInternalName()]
         }
 
@@ -354,55 +350,8 @@ public class IncrementalCacheImpl(
             storage.remove(className.getInternalName())
         }
 
-        override fun dumpValue(value: ByteArray): String {
+        override fun dumpValue(value: ProtoMapValue): String {
-            return java.lang.Long.toHexString(value.md5())
+            return (if (value.isPackageFacade) "1" else "0") + java.lang.Long.toHexString(value.bytes.md5())
-        }
-
-        private fun isOpenPartNotChanged(oldData: ByteArray, newData: ByteArray, isPackageFacade: Boolean): Boolean {
-            if (isPackageFacade) {
-                return isPackageFacadeOpenPartNotChanged(oldData, newData)
-            }
-            else {
-                return isClassOpenPartNotChanged(oldData, newData)
-            }
-        }
-
-        private fun isPackageFacadeOpenPartNotChanged(oldData: ByteArray, newData: ByteArray): Boolean {
-            val oldPackageData = JvmProtoBufUtil.readPackageDataFrom(oldData)
-            val newPackageData = JvmProtoBufUtil.readPackageDataFrom(newData)
-
-            val compareObject = ProtoCompareGenerated(oldPackageData.nameResolver, newPackageData.nameResolver)
-            return compareObject.checkEquals(oldPackageData.packageProto, newPackageData.packageProto)
-        }
-
-        private fun isClassOpenPartNotChanged(oldData: ByteArray, newData: ByteArray): Boolean {
-            val oldClassData = JvmProtoBufUtil.readClassDataFrom(oldData)
-            val newClassData = JvmProtoBufUtil.readClassDataFrom(newData)
-
-            val compareObject = object : ProtoCompareGenerated(oldClassData.nameResolver, newClassData.nameResolver) {
-                override fun checkEqualsClassMember(old: ProtoBuf.Class, new: ProtoBuf.Class): Boolean =
-                        checkEquals(old.memberList, new.memberList)
-
-                override fun checkEqualsClassSecondaryConstructor(old: ProtoBuf.Class, new: ProtoBuf.Class): Boolean =
-                        checkEquals(old.secondaryConstructorList, new.secondaryConstructorList)
-
-                private fun checkEquals(oldList: List<ProtoBuf.Callable>, newList: List<ProtoBuf.Callable>): Boolean {
-                    val oldListFiltered = oldList.filter { !it.isPrivate() }
-                    val newListFiltered = newList.filter { !it.isPrivate() }
-
-                    if (oldListFiltered.size() != newListFiltered.size()) return false
-
-                    for (i in oldListFiltered.indices) {
-                        if (!checkEquals(oldListFiltered[i], newListFiltered[i])) return false
-                    }
-
-                    return true
-                }
-
-                private fun ProtoBuf.Callable.isPrivate(): Boolean = Visibilities.isPrivate(Deserialization.visibility(Flags.VISIBILITY.get(flags)))
-            }
-
-            return compareObject.checkEquals(oldClassData.classProto, newClassData.classProto)
         }
     }
 
@@ -454,7 +403,7 @@ public class IncrementalCacheImpl(
     private object ConstantsMapExternalizer : DataExternalizer<Map<String, Any>> {
         override fun save(out: DataOutput, map: Map<String, Any>?) {
             out.writeInt(map!!.size())
-            for (name in map.keySet().toSortedList()) {
+            for (name in map.keySet().sorted()) {
                 IOUtil.writeString(name, out)
                 val value = map[name]!!
                 when (value) {
@@ -737,20 +686,6 @@ private fun ByteArray.md5(): Long {
            )
 }
 
-private object ByteArrayExternalizer : DataExternalizer<ByteArray> {
-    override fun save(out: DataOutput, value: ByteArray) {
-        out.writeInt(value.size())
-        out.write(value)
-    }
-
-    override fun read(`in`: DataInput): ByteArray {
-        val length = `in`.readInt()
-        val buf = ByteArray(length)
-        `in`.readFully(buf)
-        return buf
-    }
-}
-
 private abstract class StringMapExternalizer<T> : DataExternalizer<Map<String, T>> {
     override fun save(out: DataOutput, map: Map<String, T>?) {
         out.writeInt(map!!.size())
@@ -825,7 +760,7 @@ private val File.normalizedPath: String
 private fun <K : Comparable<K>, V> Map<K, V>.dumpMap(dumpValue: (V)->String): String =
         StringBuilder {
             append("{")
-            for (key in keySet().sort()) {
+            for (key in keySet().sorted()) {
                 if (length() != 1) {
                     append(", ")
                 }
@@ -838,7 +773,7 @@ private fun <K : Comparable<K>, V> Map<K, V>.dumpMap(dumpValue: (V)->String): St
 
 @TestOnly
 public fun <T : Comparable<T>> Collection<T>.dumpCollection(): String =
-        "[${sort().map(Any::toString).join(", ")}]"
+        "[${sorted().map(Any::toString).join(", ")}]"
 
 private class PathFunctionPair(
         public val path: String,
@@ -882,3 +817,20 @@ private object PathFunctionPairKeyDescriptor : KeyDescriptor<PathFunctionPair> {
     }
 
 }
+
+private object ProtoMapValueExternalizer : DataExternalizer<ProtoMapValue> {
+    override fun save(out: DataOutput, value: ProtoMapValue) {
+        out.writeBoolean(value.isPackageFacade)
+        out.writeInt(value.bytes.size())
+        out.write(value.bytes)
+    }
+
+    override fun read(`in`: DataInput): ProtoMapValue {
+        val isPackageFacade = `in`.readBoolean()
+        val length = `in`.readInt()
+        val buf = ByteArray(length)
+        `in`.readFully(buf)
+        return ProtoMapValue(isPackageFacade, buf)
+    }
+}
+

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/protoDifferenceUtils.kt

@@ -0,0 +1,250 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.jps.incremental
+
+import org.jetbrains.kotlin.descriptors.Visibilities
+import org.jetbrains.kotlin.serialization.Flags
+import org.jetbrains.kotlin.serialization.ProtoBuf
+import org.jetbrains.kotlin.serialization.deserialization.Deserialization
+import org.jetbrains.kotlin.serialization.deserialization.NameResolver
+import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
+import org.jetbrains.kotlin.utils.HashSetUtil
+import java.util.*
+
+public sealed class DifferenceKind() {
+    public object NONE: DifferenceKind()
+    public object CLASS_SIGNATURE: DifferenceKind()
+    public class MEMBERS(val names: Collection<String>): DifferenceKind()
+}
+
+data class ProtoMapValue(val isPackageFacade: Boolean, val bytes: ByteArray)
+
+public fun difference(oldData: ProtoMapValue, newData: ProtoMapValue): DifferenceKind {
+    if (oldData.isPackageFacade != newData.isPackageFacade) return DifferenceKind.CLASS_SIGNATURE
+
+    val differenceObject =
+            if (oldData.isPackageFacade) DifferenceCalculatorForPackageFacade(oldData, newData) else DifferenceCalculatorForClass(oldData, newData)
+
+    return differenceObject.difference()
+}
+
+private abstract class DifferenceCalculator() {
+    protected abstract val oldNameResolver: NameResolver
+    protected abstract val newNameResolver: NameResolver
+
+    protected val compareObject by lazy { ProtoCompareGenerated(oldNameResolver, newNameResolver) }
+
+    abstract fun difference(): DifferenceKind
+
+    protected fun membersOrNone(names: Collection<String>): DifferenceKind = if (names.isEmpty()) DifferenceKind.NONE else DifferenceKind.MEMBERS(names)
+
+    protected fun calcDifferenceForMembers(
+            oldList: List<ProtoBuf.Callable>,
+            newList: List<ProtoBuf.Callable>
+    ): Collection<String> {
+        val result = hashSetOf<String>()
+
+        val oldMap = oldList.groupBy { it.hashCode(compareObject.oldStringIndexes, compareObject.oldFqNameIndexes) }
+        val newMap = newList.groupBy { it.hashCode(compareObject.newStringIndexes, compareObject.newFqNameIndexes) }
+
+        fun List<ProtoBuf.Callable>.names(nameResolver: NameResolver): List<String> =
+                map { nameResolver.getString(it.name) }
+
+        val hashes = oldMap.keySet() + newMap.keySet()
+        for (hash in hashes) {
+            val oldMembers = oldMap[hash]
+            val newMembers = newMap[hash]
+
+            val differentMembers = when {
+                newMembers == null -> oldMembers!!.names(compareObject.oldNameResolver)
+                oldMembers == null -> newMembers.names(compareObject.newNameResolver)
+                else -> calcDifferenceForEqualHashes(oldMembers, newMembers)
+            }
+            result.addAll(differentMembers)
+        }
+
+        return result
+    }
+
+    private fun calcDifferenceForEqualHashes(
+            oldList: List<ProtoBuf.Callable>,
+            newList: List<ProtoBuf.Callable>
+    ): Collection<String> {
+        val result = hashSetOf<String>()
+        val newSet = HashSet(newList)
+
+        oldList.forEach { oldMember ->
+            val newMember = newSet.firstOrNull { compareObject.checkEquals(oldMember, it) }
+            if (newMember != null) {
+                newSet.remove(newMember)
+            }
+            else {
+                result.add(compareObject.oldNameResolver.getString(oldMember.name))
+            }
+        }
+
+        newSet.forEach { newMember ->
+            result.add(compareObject.newNameResolver.getString(newMember.name))
+        }
+
+        return result
+    }
+
+    protected fun calcDifferenceForNames(
+            oldList: List<Int>,
+            newList: List<Int>
+    ): Collection<String> {
+        val oldNames = oldList.map { compareObject.oldNameResolver.getString(it) }.toSet()
+        val newNames = newList.map { compareObject.newNameResolver.getString(it) }.toSet()
+        return HashSetUtil.symmetricDifference(oldNames, newNames)
+    }
+}
+
+private class DifferenceCalculatorForClass(oldData: ProtoMapValue, newData: ProtoMapValue) : DifferenceCalculator() {
+    companion object {
+        private val CONSTRUCTOR = "<init>"
+
+        private val CLASS_SIGNATURE_ENUMS = EnumSet.of(
+                ProtoCompareGenerated.ProtoBufClassKind.FLAGS,
+                ProtoCompareGenerated.ProtoBufClassKind.FQ_NAME,
+                ProtoCompareGenerated.ProtoBufClassKind.TYPE_PARAMETER_LIST,
+                ProtoCompareGenerated.ProtoBufClassKind.SUPERTYPE_LIST,
+                ProtoCompareGenerated.ProtoBufClassKind.CLASS_ANNOTATION_LIST
+        )
+    }
+
+    val oldClassData = JvmProtoBufUtil.readClassDataFrom(oldData.bytes)
+    val newClassData = JvmProtoBufUtil.readClassDataFrom(newData.bytes)
+
+    val oldProto = oldClassData.classProto
+    val newProto = newClassData.classProto
+
+    override val oldNameResolver = oldClassData.nameResolver
+    override val newNameResolver = newClassData.nameResolver
+
+    val diff = compareObject.difference(oldProto, newProto)
+
+    override fun difference(): DifferenceKind {
+        if (diff.isEmpty()) return DifferenceKind.NONE
+
+        CLASS_SIGNATURE_ENUMS.forEach { if (it in diff) return DifferenceKind.CLASS_SIGNATURE }
+
+        return membersOrNone(getChangedMembersNames())
+    }
+
+    private fun getChangedMembersNames(): Set<String> {
+        val names = hashSetOf<String>()
+
+        fun Int.oldToNames() = names.add(oldNameResolver.getString(this))
+        fun Int.newToNames() = names.add(newNameResolver.getString(this))
+
+        for (kind in diff) {
+            when (kind!!) {
+                ProtoCompareGenerated.ProtoBufClassKind.COMPANION_OBJECT_NAME -> {
+                    if (oldProto.hasCompanionObjectName()) oldProto.companionObjectName.oldToNames()
+                    if (newProto.hasCompanionObjectName()) newProto.companionObjectName.newToNames()
+                }
+                ProtoCompareGenerated.ProtoBufClassKind.NESTED_CLASS_NAME_LIST ->
+                    names.addAll(calcDifferenceForNames(oldProto.nestedClassNameList, newProto.nestedClassNameList))
+                ProtoCompareGenerated.ProtoBufClassKind.MEMBER_LIST -> {
+                    val oldMembers = oldProto.memberList.filter { !it.isPrivate }
+                    val newMembers = newProto.memberList.filter { !it.isPrivate }
+                    names.addAll(calcDifferenceForMembers(oldMembers, newMembers))
+                }
+                ProtoCompareGenerated.ProtoBufClassKind.ENUM_ENTRY_LIST ->
+                    names.addAll(calcDifferenceForNames(oldProto.enumEntryList, newProto.enumEntryList))
+                ProtoCompareGenerated.ProtoBufClassKind.PRIMARY_CONSTRUCTOR ->
+                    if (areNonPrivatePrimaryConstructorsDifferent()) {
+                        names.add(CONSTRUCTOR)
+                    }
+                ProtoCompareGenerated.ProtoBufClassKind.SECONDARY_CONSTRUCTOR_LIST ->
+                    if (areNonPrivateSecondaryConstructorsDifferent()) {
+                        names.add(CONSTRUCTOR)
+                    }
+                ProtoCompareGenerated.ProtoBufClassKind.FLAGS,
+                ProtoCompareGenerated.ProtoBufClassKind.FQ_NAME,
+                ProtoCompareGenerated.ProtoBufClassKind.TYPE_PARAMETER_LIST,
+                ProtoCompareGenerated.ProtoBufClassKind.SUPERTYPE_LIST,
+                ProtoCompareGenerated.ProtoBufClassKind.CLASS_ANNOTATION_LIST ->
+                    throw IllegalArgumentException("Unexpected kind: $kind")
+                else ->
+                    throw IllegalArgumentException("Unsupported kind: $kind")
+            }
+        }
+        return names
+    }
+
+    private fun areNonPrivatePrimaryConstructorsDifferent(): Boolean {
+        val oldPrimaryConstructor = oldProto.getNonPrivatePrimaryConstructor
+        val newPrimaryConstructor = newProto.getNonPrivatePrimaryConstructor
+        if (oldPrimaryConstructor == null && newPrimaryConstructor == null) return false
+
+        if (oldPrimaryConstructor == null || newPrimaryConstructor == null) return true
+
+        return !compareObject.checkEquals(oldPrimaryConstructor, newPrimaryConstructor)
+    }
+
+    private fun areNonPrivateSecondaryConstructorsDifferent(): Boolean {
+        val oldSecondaryConstructors = oldProto.secondaryConstructorList.filter { !it.isPrivate }
+        val newSecondaryConstructors = newProto.secondaryConstructorList.filter { !it.isPrivate }
+        return (oldSecondaryConstructors.size() != newSecondaryConstructors.size() ||
+            oldSecondaryConstructors.indices.any { !compareObject.checkEquals(oldSecondaryConstructors[it], newSecondaryConstructors[it]) })
+    }
+
+    private val ProtoBuf.Class.getNonPrivatePrimaryConstructor: ProtoBuf.Class.PrimaryConstructor?
+        get() {
+            if (!hasPrimaryConstructor()) return null
+
+            return if (primaryConstructor?.data?.isPrivate ?: false) null else primaryConstructor
+        }
+
+    private val ProtoBuf.Callable.isPrivate: Boolean
+        get() = Visibilities.isPrivate(Deserialization.visibility(Flags.VISIBILITY.get(flags)))
+}
+
+private class DifferenceCalculatorForPackageFacade(oldData: ProtoMapValue, newData: ProtoMapValue) : DifferenceCalculator() {
+    val oldPackageData = JvmProtoBufUtil.readPackageDataFrom(oldData.bytes)
+    val newPackageData = JvmProtoBufUtil.readPackageDataFrom(newData.bytes)
+
+    val oldProto = oldPackageData.packageProto
+    val newProto = newPackageData.packageProto
+
+    override val oldNameResolver = oldPackageData.nameResolver
+    override val newNameResolver = newPackageData.nameResolver
+
+    val diff = compareObject.difference(oldProto, newProto)
+
+    override fun difference(): DifferenceKind {
+        if (diff.isEmpty()) return DifferenceKind.NONE
+
+        return membersOrNone(getChangedMembersNames())
+    }
+
+    private fun getChangedMembersNames(): Set<String> {
+        val names = hashSetOf<String>()
+
+        for (kind in diff) {
+            when (kind!!) {
+                ProtoCompareGenerated.ProtoBufPackageKind.MEMBER_LIST ->
+                    names.addAll(calcDifferenceForMembers(oldProto.memberList, newProto.memberList))
+                else ->
+                    throw IllegalArgumentException("Unsupported kind: $kind")
+            }
+        }
+        return names
+    }
+}