diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt index b43043365e1..d9a9c37e082 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt @@ -31,7 +31,6 @@ import org.jetbrains.jps.incremental.storage.BuildDataManager import org.jetbrains.jps.incremental.storage.PathStringDescriptor import org.jetbrains.jps.incremental.storage.StorageOwner import org.jetbrains.kotlin.config.IncrementalCompilation -import org.jetbrains.kotlin.descriptors.Visibilities import org.jetbrains.kotlin.jps.build.GeneratedJvmClass import org.jetbrains.kotlin.jps.build.KotlinBuilder import org.jetbrains.kotlin.jps.incremental.storage.BasicMap @@ -46,11 +45,7 @@ import org.jetbrains.kotlin.load.kotlin.header.isCompatiblePackageFacadeKind import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache import org.jetbrains.kotlin.resolve.jvm.JvmClassName import org.jetbrains.kotlin.resolve.jvm.JvmClassName.byInternalName -import org.jetbrains.kotlin.serialization.Flags -import org.jetbrains.kotlin.serialization.ProtoBuf -import org.jetbrains.kotlin.serialization.deserialization.Deserialization import org.jetbrains.kotlin.serialization.jvm.BitEncoding -import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil import org.jetbrains.org.objectweb.asm.* import java.io.DataInput import java.io.DataInputStream @@ -67,7 +62,7 @@ private val CACHE_DIRECTORY_NAME = "kotlin" class CacheFormatVersion(targetDataRoot: File) { companion object { // Change this when incremental cache format changes - private val INCREMENTAL_CACHE_OWN_VERSION = 4 + private val INCREMENTAL_CACHE_OWN_VERSION = 5 private val CACHE_FORMAT_VERSION = INCREMENTAL_CACHE_OWN_VERSION * 1000000 + @@ -301,11 +296,11 @@ public class IncrementalCacheImpl( } override fun getPackagePartData(fqName: String): ByteArray? { - return protoMap[JvmClassName.byInternalName(fqName)] + return protoMap[JvmClassName.byInternalName(fqName)]?.bytes } override fun getModuleMappingData(): ByteArray? { - return protoMap[JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)] + return protoMap[JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)]?.bytes } override fun flush(memoryCachesOnly: Boolean) { @@ -321,7 +316,7 @@ public class IncrementalCacheImpl( maps.forEach { it.close () } } - private inner class ProtoMap(storageFile: File) : BasicStringMap(storageFile, ByteArrayExternalizer) { + private inner class ProtoMap(storageFile: File) : BasicStringMap(storageFile, ProtoMapValueExternalizer) { public fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean, checkChangesIsOpenPart: Boolean = true): ChangesInfo { val header = kotlinClass.classHeader @@ -333,20 +328,21 @@ public class IncrementalCacheImpl( return put(className, data, isPackage, checkChangesIsOpenPart) } - private fun put(className: JvmClassName, data: ByteArray, isPackage: Boolean, checkChangesIsOpenPart: Boolean): ChangesInfo { + private fun put(className: JvmClassName, bytes: ByteArray, isPackage: Boolean, checkChangesIsOpenPart: Boolean): ChangesInfo { val key = className.internalName val oldData = storage[key] + val data = ProtoMapValue(isPackage, bytes) - if (!Arrays.equals(data, oldData)) { + if (oldData == null || !Arrays.equals(bytes, oldData.bytes) || isPackage != oldData.isPackageFacade) { storage.put(key, data) } return ChangesInfo(protoChanged = oldData == null || !checkChangesIsOpenPart || - !isOpenPartNotChanged(oldData, data, isPackage)) + difference(oldData, data) != DifferenceKind.NONE) } - public fun get(className: JvmClassName): ByteArray? { + public fun get(className: JvmClassName): ProtoMapValue? { return storage[className.getInternalName()] } @@ -354,55 +350,8 @@ public class IncrementalCacheImpl( storage.remove(className.getInternalName()) } - override fun dumpValue(value: ByteArray): String { - return java.lang.Long.toHexString(value.md5()) - } - - private fun isOpenPartNotChanged(oldData: ByteArray, newData: ByteArray, isPackageFacade: Boolean): Boolean { - if (isPackageFacade) { - return isPackageFacadeOpenPartNotChanged(oldData, newData) - } - else { - return isClassOpenPartNotChanged(oldData, newData) - } - } - - private fun isPackageFacadeOpenPartNotChanged(oldData: ByteArray, newData: ByteArray): Boolean { - val oldPackageData = JvmProtoBufUtil.readPackageDataFrom(oldData) - val newPackageData = JvmProtoBufUtil.readPackageDataFrom(newData) - - val compareObject = ProtoCompareGenerated(oldPackageData.nameResolver, newPackageData.nameResolver) - return compareObject.checkEquals(oldPackageData.packageProto, newPackageData.packageProto) - } - - private fun isClassOpenPartNotChanged(oldData: ByteArray, newData: ByteArray): Boolean { - val oldClassData = JvmProtoBufUtil.readClassDataFrom(oldData) - val newClassData = JvmProtoBufUtil.readClassDataFrom(newData) - - val compareObject = object : ProtoCompareGenerated(oldClassData.nameResolver, newClassData.nameResolver) { - override fun checkEqualsClassMember(old: ProtoBuf.Class, new: ProtoBuf.Class): Boolean = - checkEquals(old.memberList, new.memberList) - - override fun checkEqualsClassSecondaryConstructor(old: ProtoBuf.Class, new: ProtoBuf.Class): Boolean = - checkEquals(old.secondaryConstructorList, new.secondaryConstructorList) - - private fun checkEquals(oldList: List, newList: List): Boolean { - val oldListFiltered = oldList.filter { !it.isPrivate() } - val newListFiltered = newList.filter { !it.isPrivate() } - - if (oldListFiltered.size() != newListFiltered.size()) return false - - for (i in oldListFiltered.indices) { - if (!checkEquals(oldListFiltered[i], newListFiltered[i])) return false - } - - return true - } - - private fun ProtoBuf.Callable.isPrivate(): Boolean = Visibilities.isPrivate(Deserialization.visibility(Flags.VISIBILITY.get(flags))) - } - - return compareObject.checkEquals(oldClassData.classProto, newClassData.classProto) + override fun dumpValue(value: ProtoMapValue): String { + return (if (value.isPackageFacade) "1" else "0") + java.lang.Long.toHexString(value.bytes.md5()) } } @@ -454,7 +403,7 @@ public class IncrementalCacheImpl( private object ConstantsMapExternalizer : DataExternalizer> { override fun save(out: DataOutput, map: Map?) { out.writeInt(map!!.size()) - for (name in map.keySet().toSortedList()) { + for (name in map.keySet().sorted()) { IOUtil.writeString(name, out) val value = map[name]!! when (value) { @@ -737,20 +686,6 @@ private fun ByteArray.md5(): Long { ) } -private object ByteArrayExternalizer : DataExternalizer { - override fun save(out: DataOutput, value: ByteArray) { - out.writeInt(value.size()) - out.write(value) - } - - override fun read(`in`: DataInput): ByteArray { - val length = `in`.readInt() - val buf = ByteArray(length) - `in`.readFully(buf) - return buf - } -} - private abstract class StringMapExternalizer : DataExternalizer> { override fun save(out: DataOutput, map: Map?) { out.writeInt(map!!.size()) @@ -825,7 +760,7 @@ private val File.normalizedPath: String private fun , V> Map.dumpMap(dumpValue: (V)->String): String = StringBuilder { append("{") - for (key in keySet().sort()) { + for (key in keySet().sorted()) { if (length() != 1) { append(", ") } @@ -838,7 +773,7 @@ private fun , V> Map.dumpMap(dumpValue: (V)->String): St @TestOnly public fun > Collection.dumpCollection(): String = - "[${sort().map(Any::toString).join(", ")}]" + "[${sorted().map(Any::toString).join(", ")}]" private class PathFunctionPair( public val path: String, @@ -882,3 +817,20 @@ private object PathFunctionPairKeyDescriptor : KeyDescriptor { } } + +private object ProtoMapValueExternalizer : DataExternalizer { + override fun save(out: DataOutput, value: ProtoMapValue) { + out.writeBoolean(value.isPackageFacade) + out.writeInt(value.bytes.size()) + out.write(value.bytes) + } + + override fun read(`in`: DataInput): ProtoMapValue { + val isPackageFacade = `in`.readBoolean() + val length = `in`.readInt() + val buf = ByteArray(length) + `in`.readFully(buf) + return ProtoMapValue(isPackageFacade, buf) + } +} + diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/protoDifferenceUtils.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/protoDifferenceUtils.kt new file mode 100644 index 00000000000..f575e4fda2e --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/protoDifferenceUtils.kt @@ -0,0 +1,250 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.jps.incremental + +import org.jetbrains.kotlin.descriptors.Visibilities +import org.jetbrains.kotlin.serialization.Flags +import org.jetbrains.kotlin.serialization.ProtoBuf +import org.jetbrains.kotlin.serialization.deserialization.Deserialization +import org.jetbrains.kotlin.serialization.deserialization.NameResolver +import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil +import org.jetbrains.kotlin.utils.HashSetUtil +import java.util.* + +public sealed class DifferenceKind() { + public object NONE: DifferenceKind() + public object CLASS_SIGNATURE: DifferenceKind() + public class MEMBERS(val names: Collection): DifferenceKind() +} + +data class ProtoMapValue(val isPackageFacade: Boolean, val bytes: ByteArray) + +public fun difference(oldData: ProtoMapValue, newData: ProtoMapValue): DifferenceKind { + if (oldData.isPackageFacade != newData.isPackageFacade) return DifferenceKind.CLASS_SIGNATURE + + val differenceObject = + if (oldData.isPackageFacade) DifferenceCalculatorForPackageFacade(oldData, newData) else DifferenceCalculatorForClass(oldData, newData) + + return differenceObject.difference() +} + +private abstract class DifferenceCalculator() { + protected abstract val oldNameResolver: NameResolver + protected abstract val newNameResolver: NameResolver + + protected val compareObject by lazy { ProtoCompareGenerated(oldNameResolver, newNameResolver) } + + abstract fun difference(): DifferenceKind + + protected fun membersOrNone(names: Collection): DifferenceKind = if (names.isEmpty()) DifferenceKind.NONE else DifferenceKind.MEMBERS(names) + + protected fun calcDifferenceForMembers( + oldList: List, + newList: List + ): Collection { + val result = hashSetOf() + + val oldMap = oldList.groupBy { it.hashCode(compareObject.oldStringIndexes, compareObject.oldFqNameIndexes) } + val newMap = newList.groupBy { it.hashCode(compareObject.newStringIndexes, compareObject.newFqNameIndexes) } + + fun List.names(nameResolver: NameResolver): List = + map { nameResolver.getString(it.name) } + + val hashes = oldMap.keySet() + newMap.keySet() + for (hash in hashes) { + val oldMembers = oldMap[hash] + val newMembers = newMap[hash] + + val differentMembers = when { + newMembers == null -> oldMembers!!.names(compareObject.oldNameResolver) + oldMembers == null -> newMembers.names(compareObject.newNameResolver) + else -> calcDifferenceForEqualHashes(oldMembers, newMembers) + } + result.addAll(differentMembers) + } + + return result + } + + private fun calcDifferenceForEqualHashes( + oldList: List, + newList: List + ): Collection { + val result = hashSetOf() + val newSet = HashSet(newList) + + oldList.forEach { oldMember -> + val newMember = newSet.firstOrNull { compareObject.checkEquals(oldMember, it) } + if (newMember != null) { + newSet.remove(newMember) + } + else { + result.add(compareObject.oldNameResolver.getString(oldMember.name)) + } + } + + newSet.forEach { newMember -> + result.add(compareObject.newNameResolver.getString(newMember.name)) + } + + return result + } + + protected fun calcDifferenceForNames( + oldList: List, + newList: List + ): Collection { + val oldNames = oldList.map { compareObject.oldNameResolver.getString(it) }.toSet() + val newNames = newList.map { compareObject.newNameResolver.getString(it) }.toSet() + return HashSetUtil.symmetricDifference(oldNames, newNames) + } +} + +private class DifferenceCalculatorForClass(oldData: ProtoMapValue, newData: ProtoMapValue) : DifferenceCalculator() { + companion object { + private val CONSTRUCTOR = "" + + private val CLASS_SIGNATURE_ENUMS = EnumSet.of( + ProtoCompareGenerated.ProtoBufClassKind.FLAGS, + ProtoCompareGenerated.ProtoBufClassKind.FQ_NAME, + ProtoCompareGenerated.ProtoBufClassKind.TYPE_PARAMETER_LIST, + ProtoCompareGenerated.ProtoBufClassKind.SUPERTYPE_LIST, + ProtoCompareGenerated.ProtoBufClassKind.CLASS_ANNOTATION_LIST + ) + } + + val oldClassData = JvmProtoBufUtil.readClassDataFrom(oldData.bytes) + val newClassData = JvmProtoBufUtil.readClassDataFrom(newData.bytes) + + val oldProto = oldClassData.classProto + val newProto = newClassData.classProto + + override val oldNameResolver = oldClassData.nameResolver + override val newNameResolver = newClassData.nameResolver + + val diff = compareObject.difference(oldProto, newProto) + + override fun difference(): DifferenceKind { + if (diff.isEmpty()) return DifferenceKind.NONE + + CLASS_SIGNATURE_ENUMS.forEach { if (it in diff) return DifferenceKind.CLASS_SIGNATURE } + + return membersOrNone(getChangedMembersNames()) + } + + private fun getChangedMembersNames(): Set { + val names = hashSetOf() + + fun Int.oldToNames() = names.add(oldNameResolver.getString(this)) + fun Int.newToNames() = names.add(newNameResolver.getString(this)) + + for (kind in diff) { + when (kind!!) { + ProtoCompareGenerated.ProtoBufClassKind.COMPANION_OBJECT_NAME -> { + if (oldProto.hasCompanionObjectName()) oldProto.companionObjectName.oldToNames() + if (newProto.hasCompanionObjectName()) newProto.companionObjectName.newToNames() + } + ProtoCompareGenerated.ProtoBufClassKind.NESTED_CLASS_NAME_LIST -> + names.addAll(calcDifferenceForNames(oldProto.nestedClassNameList, newProto.nestedClassNameList)) + ProtoCompareGenerated.ProtoBufClassKind.MEMBER_LIST -> { + val oldMembers = oldProto.memberList.filter { !it.isPrivate } + val newMembers = newProto.memberList.filter { !it.isPrivate } + names.addAll(calcDifferenceForMembers(oldMembers, newMembers)) + } + ProtoCompareGenerated.ProtoBufClassKind.ENUM_ENTRY_LIST -> + names.addAll(calcDifferenceForNames(oldProto.enumEntryList, newProto.enumEntryList)) + ProtoCompareGenerated.ProtoBufClassKind.PRIMARY_CONSTRUCTOR -> + if (areNonPrivatePrimaryConstructorsDifferent()) { + names.add(CONSTRUCTOR) + } + ProtoCompareGenerated.ProtoBufClassKind.SECONDARY_CONSTRUCTOR_LIST -> + if (areNonPrivateSecondaryConstructorsDifferent()) { + names.add(CONSTRUCTOR) + } + ProtoCompareGenerated.ProtoBufClassKind.FLAGS, + ProtoCompareGenerated.ProtoBufClassKind.FQ_NAME, + ProtoCompareGenerated.ProtoBufClassKind.TYPE_PARAMETER_LIST, + ProtoCompareGenerated.ProtoBufClassKind.SUPERTYPE_LIST, + ProtoCompareGenerated.ProtoBufClassKind.CLASS_ANNOTATION_LIST -> + throw IllegalArgumentException("Unexpected kind: $kind") + else -> + throw IllegalArgumentException("Unsupported kind: $kind") + } + } + return names + } + + private fun areNonPrivatePrimaryConstructorsDifferent(): Boolean { + val oldPrimaryConstructor = oldProto.getNonPrivatePrimaryConstructor + val newPrimaryConstructor = newProto.getNonPrivatePrimaryConstructor + if (oldPrimaryConstructor == null && newPrimaryConstructor == null) return false + + if (oldPrimaryConstructor == null || newPrimaryConstructor == null) return true + + return !compareObject.checkEquals(oldPrimaryConstructor, newPrimaryConstructor) + } + + private fun areNonPrivateSecondaryConstructorsDifferent(): Boolean { + val oldSecondaryConstructors = oldProto.secondaryConstructorList.filter { !it.isPrivate } + val newSecondaryConstructors = newProto.secondaryConstructorList.filter { !it.isPrivate } + return (oldSecondaryConstructors.size() != newSecondaryConstructors.size() || + oldSecondaryConstructors.indices.any { !compareObject.checkEquals(oldSecondaryConstructors[it], newSecondaryConstructors[it]) }) + } + + private val ProtoBuf.Class.getNonPrivatePrimaryConstructor: ProtoBuf.Class.PrimaryConstructor? + get() { + if (!hasPrimaryConstructor()) return null + + return if (primaryConstructor?.data?.isPrivate ?: false) null else primaryConstructor + } + + private val ProtoBuf.Callable.isPrivate: Boolean + get() = Visibilities.isPrivate(Deserialization.visibility(Flags.VISIBILITY.get(flags))) +} + +private class DifferenceCalculatorForPackageFacade(oldData: ProtoMapValue, newData: ProtoMapValue) : DifferenceCalculator() { + val oldPackageData = JvmProtoBufUtil.readPackageDataFrom(oldData.bytes) + val newPackageData = JvmProtoBufUtil.readPackageDataFrom(newData.bytes) + + val oldProto = oldPackageData.packageProto + val newProto = newPackageData.packageProto + + override val oldNameResolver = oldPackageData.nameResolver + override val newNameResolver = newPackageData.nameResolver + + val diff = compareObject.difference(oldProto, newProto) + + override fun difference(): DifferenceKind { + if (diff.isEmpty()) return DifferenceKind.NONE + + return membersOrNone(getChangedMembersNames()) + } + + private fun getChangedMembersNames(): Set { + val names = hashSetOf() + + for (kind in diff) { + when (kind!!) { + ProtoCompareGenerated.ProtoBufPackageKind.MEMBER_LIST -> + names.addAll(calcDifferenceForMembers(oldProto.memberList, newProto.memberList)) + else -> + throw IllegalArgumentException("Unsupported kind: $kind") + } + } + return names + } +}