azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add protoDifferenceUtils and implementation for calculation difference between proto data for classes.

Browse Source
This commit is contained in:
Michael Nedzelsky
2015-09-09 15:54:46 +03:00
parent 4a67cde283
commit df283c8f02
2 changed files with 281 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt
+31 -79
View File
@@ -31,7 +31,6 @@ import org.jetbrains.jps.incremental.storage.BuildDataManager
import org.jetbrains.jps.incremental.storage.PathStringDescriptor
import org.jetbrains.jps.incremental.storage.StorageOwner
import org.jetbrains.kotlin.config.IncrementalCompilation
import org.jetbrains.kotlin.descriptors.Visibilities
import org.jetbrains.kotlin.jps.build.GeneratedJvmClass
import org.jetbrains.kotlin.jps.build.KotlinBuilder
import org.jetbrains.kotlin.jps.incremental.storage.BasicMap
@@ -46,11 +45,7 @@ import org.jetbrains.kotlin.load.kotlin.header.isCompatiblePackageFacadeKind
import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
import org.jetbrains.kotlin.resolve.jvm.JvmClassName
import org.jetbrains.kotlin.resolve.jvm.JvmClassName.byInternalName
import org.jetbrains.kotlin.serialization.Flags
import org.jetbrains.kotlin.serialization.ProtoBuf
import org.jetbrains.kotlin.serialization.deserialization.Deserialization
import org.jetbrains.kotlin.serialization.jvm.BitEncoding
import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
import org.jetbrains.org.objectweb.asm.*
import java.io.DataInput
import java.io.DataInputStream
@@ -67,7 +62,7 @@ private val CACHE_DIRECTORY_NAME = "kotlin"
class CacheFormatVersion(targetDataRoot: File) {
companion object {
// Change this when incremental cache format changes
private val INCREMENTAL_CACHE_OWN_VERSION = 4
private val INCREMENTAL_CACHE_OWN_VERSION = 5
private val CACHE_FORMAT_VERSION =
INCREMENTAL_CACHE_OWN_VERSION * 1000000 +
@@ -301,11 +296,11 @@ public class IncrementalCacheImpl(
}
override fun getPackagePartData(fqName: String): ByteArray? {
return protoMap[JvmClassName.byInternalName(fqName)]
return protoMap[JvmClassName.byInternalName(fqName)]?.bytes
}
override fun getModuleMappingData(): ByteArray? {
return protoMap[JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)]
return protoMap[JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)]?.bytes
}
override fun flush(memoryCachesOnly: Boolean) {
@@ -321,7 +316,7 @@ public class IncrementalCacheImpl(
maps.forEach { it.close () }
}
private inner class ProtoMap(storageFile: File) : BasicStringMap<ByteArray>(storageFile, ByteArrayExternalizer) {
private inner class ProtoMap(storageFile: File) : BasicStringMap<ProtoMapValue>(storageFile, ProtoMapValueExternalizer) {
public fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean, checkChangesIsOpenPart: Boolean = true): ChangesInfo {
val header = kotlinClass.classHeader
@@ -333,20 +328,21 @@ public class IncrementalCacheImpl(
return put(className, data, isPackage, checkChangesIsOpenPart)
}
private fun put(className: JvmClassName, data: ByteArray, isPackage: Boolean, checkChangesIsOpenPart: Boolean): ChangesInfo {
private fun put(className: JvmClassName, bytes: ByteArray, isPackage: Boolean, checkChangesIsOpenPart: Boolean): ChangesInfo {
val key = className.internalName
val oldData = storage[key]
val data = ProtoMapValue(isPackage, bytes)
if (!Arrays.equals(data, oldData)) {
if (oldData == null || !Arrays.equals(bytes, oldData.bytes) || isPackage != oldData.isPackageFacade) {
storage.put(key, data)
}
return ChangesInfo(protoChanged = oldData == null ||
!checkChangesIsOpenPart ||
!isOpenPartNotChanged(oldData, data, isPackage))
difference(oldData, data) != DifferenceKind.NONE)
}
public fun get(className: JvmClassName): ByteArray? {
public fun get(className: JvmClassName): ProtoMapValue? {
return storage[className.getInternalName()]
}
@@ -354,55 +350,8 @@ public class IncrementalCacheImpl(
storage.remove(className.getInternalName())
}
override fun dumpValue(value: ByteArray): String {
return java.lang.Long.toHexString(value.md5())
}
private fun isOpenPartNotChanged(oldData: ByteArray, newData: ByteArray, isPackageFacade: Boolean): Boolean {
if (isPackageFacade) {
return isPackageFacadeOpenPartNotChanged(oldData, newData)
}
else {
return isClassOpenPartNotChanged(oldData, newData)
}
}
private fun isPackageFacadeOpenPartNotChanged(oldData: ByteArray, newData: ByteArray): Boolean {
val oldPackageData = JvmProtoBufUtil.readPackageDataFrom(oldData)
val newPackageData = JvmProtoBufUtil.readPackageDataFrom(newData)
val compareObject = ProtoCompareGenerated(oldPackageData.nameResolver, newPackageData.nameResolver)
return compareObject.checkEquals(oldPackageData.packageProto, newPackageData.packageProto)
}
private fun isClassOpenPartNotChanged(oldData: ByteArray, newData: ByteArray): Boolean {
val oldClassData = JvmProtoBufUtil.readClassDataFrom(oldData)
val newClassData = JvmProtoBufUtil.readClassDataFrom(newData)
val compareObject = object : ProtoCompareGenerated(oldClassData.nameResolver, newClassData.nameResolver) {
override fun checkEqualsClassMember(old: ProtoBuf.Class, new: ProtoBuf.Class): Boolean =
checkEquals(old.memberList, new.memberList)
override fun checkEqualsClassSecondaryConstructor(old: ProtoBuf.Class, new: ProtoBuf.Class): Boolean =
checkEquals(old.secondaryConstructorList, new.secondaryConstructorList)
private fun checkEquals(oldList: List<ProtoBuf.Callable>, newList: List<ProtoBuf.Callable>): Boolean {
val oldListFiltered = oldList.filter { !it.isPrivate() }
val newListFiltered = newList.filter { !it.isPrivate() }
if (oldListFiltered.size() != newListFiltered.size()) return false
for (i in oldListFiltered.indices) {
if (!checkEquals(oldListFiltered[i], newListFiltered[i])) return false
}
return true
}
private fun ProtoBuf.Callable.isPrivate(): Boolean = Visibilities.isPrivate(Deserialization.visibility(Flags.VISIBILITY.get(flags)))
}
return compareObject.checkEquals(oldClassData.classProto, newClassData.classProto)
override fun dumpValue(value: ProtoMapValue): String {
return (if (value.isPackageFacade) "1" else "0") + java.lang.Long.toHexString(value.bytes.md5())
}
}
@@ -454,7 +403,7 @@ public class IncrementalCacheImpl(
private object ConstantsMapExternalizer : DataExternalizer<Map<String, Any>> {
override fun save(out: DataOutput, map: Map<String, Any>?) {
out.writeInt(map!!.size())
for (name in map.keySet().toSortedList()) {
for (name in map.keySet().sorted()) {
IOUtil.writeString(name, out)
val value = map[name]!!
when (value) {
@@ -737,20 +686,6 @@ private fun ByteArray.md5(): Long {
)
}
private object ByteArrayExternalizer : DataExternalizer<ByteArray> {
override fun save(out: DataOutput, value: ByteArray) {
out.writeInt(value.size())
out.write(value)
}
override fun read(`in`: DataInput): ByteArray {
val length = `in`.readInt()
val buf = ByteArray(length)
`in`.readFully(buf)
return buf
}
}
private abstract class StringMapExternalizer<T> : DataExternalizer<Map<String, T>> {
override fun save(out: DataOutput, map: Map<String, T>?) {
out.writeInt(map!!.size())
@@ -825,7 +760,7 @@ private val File.normalizedPath: String
private fun <K : Comparable<K>, V> Map<K, V>.dumpMap(dumpValue: (V)->String): String =
StringBuilder {
append("{")
for (key in keySet().sort()) {
for (key in keySet().sorted()) {
if (length() != 1) {
append(", ")
}
@@ -838,7 +773,7 @@ private fun <K : Comparable<K>, V> Map<K, V>.dumpMap(dumpValue: (V)->String): St
@TestOnly
public fun <T : Comparable<T>> Collection<T>.dumpCollection(): String =
"[${sort().map(Any::toString).join(", ")}]"
"[${sorted().map(Any::toString).join(", ")}]"
private class PathFunctionPair(
public val path: String,
@@ -882,3 +817,20 @@ private object PathFunctionPairKeyDescriptor : KeyDescriptor<PathFunctionPair> {
}
}
private object ProtoMapValueExternalizer : DataExternalizer<ProtoMapValue> {
override fun save(out: DataOutput, value: ProtoMapValue) {
out.writeBoolean(value.isPackageFacade)
out.writeInt(value.bytes.size())
out.write(value.bytes)
}
override fun read(`in`: DataInput): ProtoMapValue {
val isPackageFacade = `in`.readBoolean()
val length = `in`.readInt()
val buf = ByteArray(length)
`in`.readFully(buf)
return ProtoMapValue(isPackageFacade, buf)
}
}
jps-plugin/src/org/jetbrains/kotlin/jps/incremental/protoDifferenceUtils.kt
+250
View File
@@ -0,0 +1,250 @@
/*
* Copyright 2010-2015 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.jps.incremental
import org.jetbrains.kotlin.descriptors.Visibilities
import org.jetbrains.kotlin.serialization.Flags
import org.jetbrains.kotlin.serialization.ProtoBuf
import org.jetbrains.kotlin.serialization.deserialization.Deserialization
import org.jetbrains.kotlin.serialization.deserialization.NameResolver
import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
import org.jetbrains.kotlin.utils.HashSetUtil
import java.util.*
public sealed class DifferenceKind() {
public object NONE: DifferenceKind()
public object CLASS_SIGNATURE: DifferenceKind()
public class MEMBERS(val names: Collection<String>): DifferenceKind()
}
data class ProtoMapValue(val isPackageFacade: Boolean, val bytes: ByteArray)
public fun difference(oldData: ProtoMapValue, newData: ProtoMapValue): DifferenceKind {
if (oldData.isPackageFacade != newData.isPackageFacade) return DifferenceKind.CLASS_SIGNATURE
val differenceObject =
if (oldData.isPackageFacade) DifferenceCalculatorForPackageFacade(oldData, newData) else DifferenceCalculatorForClass(oldData, newData)
return differenceObject.difference()
}
private abstract class DifferenceCalculator() {
protected abstract val oldNameResolver: NameResolver
protected abstract val newNameResolver: NameResolver
protected val compareObject by lazy { ProtoCompareGenerated(oldNameResolver, newNameResolver) }
abstract fun difference(): DifferenceKind
protected fun membersOrNone(names: Collection<String>): DifferenceKind = if (names.isEmpty()) DifferenceKind.NONE else DifferenceKind.MEMBERS(names)
protected fun calcDifferenceForMembers(
oldList: List<ProtoBuf.Callable>,
newList: List<ProtoBuf.Callable>
): Collection<String> {
val result = hashSetOf<String>()
val oldMap = oldList.groupBy { it.hashCode(compareObject.oldStringIndexes, compareObject.oldFqNameIndexes) }
val newMap = newList.groupBy { it.hashCode(compareObject.newStringIndexes, compareObject.newFqNameIndexes) }
fun List<ProtoBuf.Callable>.names(nameResolver: NameResolver): List<String> =
map { nameResolver.getString(it.name) }
val hashes = oldMap.keySet() + newMap.keySet()
for (hash in hashes) {
val oldMembers = oldMap[hash]
val newMembers = newMap[hash]
val differentMembers = when {
newMembers == null -> oldMembers!!.names(compareObject.oldNameResolver)
oldMembers == null -> newMembers.names(compareObject.newNameResolver)
else -> calcDifferenceForEqualHashes(oldMembers, newMembers)
}
result.addAll(differentMembers)
}
return result
}
private fun calcDifferenceForEqualHashes(
oldList: List<ProtoBuf.Callable>,
newList: List<ProtoBuf.Callable>
): Collection<String> {
val result = hashSetOf<String>()
val newSet = HashSet(newList)
oldList.forEach { oldMember ->
val newMember = newSet.firstOrNull { compareObject.checkEquals(oldMember, it) }
if (newMember != null) {
newSet.remove(newMember)
}
else {
result.add(compareObject.oldNameResolver.getString(oldMember.name))
}
}
newSet.forEach { newMember ->
result.add(compareObject.newNameResolver.getString(newMember.name))
}
return result
}
protected fun calcDifferenceForNames(
oldList: List<Int>,
newList: List<Int>
): Collection<String> {
val oldNames = oldList.map { compareObject.oldNameResolver.getString(it) }.toSet()
val newNames = newList.map { compareObject.newNameResolver.getString(it) }.toSet()
return HashSetUtil.symmetricDifference(oldNames, newNames)
}
}
private class DifferenceCalculatorForClass(oldData: ProtoMapValue, newData: ProtoMapValue) : DifferenceCalculator() {
companion object {
private val CONSTRUCTOR = "<init>"
private val CLASS_SIGNATURE_ENUMS = EnumSet.of(
ProtoCompareGenerated.ProtoBufClassKind.FLAGS,
ProtoCompareGenerated.ProtoBufClassKind.FQ_NAME,
ProtoCompareGenerated.ProtoBufClassKind.TYPE_PARAMETER_LIST,
ProtoCompareGenerated.ProtoBufClassKind.SUPERTYPE_LIST,
ProtoCompareGenerated.ProtoBufClassKind.CLASS_ANNOTATION_LIST
)
}
val oldClassData = JvmProtoBufUtil.readClassDataFrom(oldData.bytes)
val newClassData = JvmProtoBufUtil.readClassDataFrom(newData.bytes)
val oldProto = oldClassData.classProto
val newProto = newClassData.classProto
override val oldNameResolver = oldClassData.nameResolver
override val newNameResolver = newClassData.nameResolver
val diff = compareObject.difference(oldProto, newProto)
override fun difference(): DifferenceKind {
if (diff.isEmpty()) return DifferenceKind.NONE
CLASS_SIGNATURE_ENUMS.forEach { if (it in diff) return DifferenceKind.CLASS_SIGNATURE }
return membersOrNone(getChangedMembersNames())
}
private fun getChangedMembersNames(): Set<String> {
val names = hashSetOf<String>()
fun Int.oldToNames() = names.add(oldNameResolver.getString(this))
fun Int.newToNames() = names.add(newNameResolver.getString(this))
for (kind in diff) {
when (kind!!) {
ProtoCompareGenerated.ProtoBufClassKind.COMPANION_OBJECT_NAME -> {
if (oldProto.hasCompanionObjectName()) oldProto.companionObjectName.oldToNames()
if (newProto.hasCompanionObjectName()) newProto.companionObjectName.newToNames()
}
ProtoCompareGenerated.ProtoBufClassKind.NESTED_CLASS_NAME_LIST ->
names.addAll(calcDifferenceForNames(oldProto.nestedClassNameList, newProto.nestedClassNameList))
ProtoCompareGenerated.ProtoBufClassKind.MEMBER_LIST -> {
val oldMembers = oldProto.memberList.filter { !it.isPrivate }
val newMembers = newProto.memberList.filter { !it.isPrivate }
names.addAll(calcDifferenceForMembers(oldMembers, newMembers))
}
ProtoCompareGenerated.ProtoBufClassKind.ENUM_ENTRY_LIST ->
names.addAll(calcDifferenceForNames(oldProto.enumEntryList, newProto.enumEntryList))
ProtoCompareGenerated.ProtoBufClassKind.PRIMARY_CONSTRUCTOR ->
if (areNonPrivatePrimaryConstructorsDifferent()) {
names.add(CONSTRUCTOR)
}
ProtoCompareGenerated.ProtoBufClassKind.SECONDARY_CONSTRUCTOR_LIST ->
if (areNonPrivateSecondaryConstructorsDifferent()) {
names.add(CONSTRUCTOR)
}
ProtoCompareGenerated.ProtoBufClassKind.FLAGS,
ProtoCompareGenerated.ProtoBufClassKind.FQ_NAME,
ProtoCompareGenerated.ProtoBufClassKind.TYPE_PARAMETER_LIST,
ProtoCompareGenerated.ProtoBufClassKind.SUPERTYPE_LIST,
ProtoCompareGenerated.ProtoBufClassKind.CLASS_ANNOTATION_LIST ->
throw IllegalArgumentException("Unexpected kind: $kind")
else ->
throw IllegalArgumentException("Unsupported kind: $kind")
}
}
return names
}
private fun areNonPrivatePrimaryConstructorsDifferent(): Boolean {
val oldPrimaryConstructor = oldProto.getNonPrivatePrimaryConstructor
val newPrimaryConstructor = newProto.getNonPrivatePrimaryConstructor
if (oldPrimaryConstructor == null && newPrimaryConstructor == null) return false
if (oldPrimaryConstructor == null || newPrimaryConstructor == null) return true
return !compareObject.checkEquals(oldPrimaryConstructor, newPrimaryConstructor)
}
private fun areNonPrivateSecondaryConstructorsDifferent(): Boolean {
val oldSecondaryConstructors = oldProto.secondaryConstructorList.filter { !it.isPrivate }
val newSecondaryConstructors = newProto.secondaryConstructorList.filter { !it.isPrivate }
return (oldSecondaryConstructors.size() != newSecondaryConstructors.size() ||
oldSecondaryConstructors.indices.any { !compareObject.checkEquals(oldSecondaryConstructors[it], newSecondaryConstructors[it]) })
}
private val ProtoBuf.Class.getNonPrivatePrimaryConstructor: ProtoBuf.Class.PrimaryConstructor?
get() {
if (!hasPrimaryConstructor()) return null
return if (primaryConstructor?.data?.isPrivate ?: false) null else primaryConstructor
}
private val ProtoBuf.Callable.isPrivate: Boolean
get() = Visibilities.isPrivate(Deserialization.visibility(Flags.VISIBILITY.get(flags)))
}
private class DifferenceCalculatorForPackageFacade(oldData: ProtoMapValue, newData: ProtoMapValue) : DifferenceCalculator() {
val oldPackageData = JvmProtoBufUtil.readPackageDataFrom(oldData.bytes)
val newPackageData = JvmProtoBufUtil.readPackageDataFrom(newData.bytes)
val oldProto = oldPackageData.packageProto
val newProto = newPackageData.packageProto
override val oldNameResolver = oldPackageData.nameResolver
override val newNameResolver = newPackageData.nameResolver
val diff = compareObject.difference(oldProto, newProto)
override fun difference(): DifferenceKind {
if (diff.isEmpty()) return DifferenceKind.NONE
return membersOrNone(getChangedMembersNames())
}
private fun getChangedMembersNames(): Set<String> {
val names = hashSetOf<String>()
for (kind in diff) {
when (kind!!) {
ProtoCompareGenerated.ProtoBufPackageKind.MEMBER_LIST ->
names.addAll(calcDifferenceForMembers(oldProto.memberList, newProto.memberList))
else ->
throw IllegalArgumentException("Unsupported kind: $kind")
}
}
return names
}
}