diff --git a/build-common/src/org/jetbrains/kotlin/incremental/ChangesCollector.kt b/build-common/src/org/jetbrains/kotlin/incremental/ChangesCollector.kt new file mode 100644 index 00000000000..9b6a84bd727 --- /dev/null +++ b/build-common/src/org/jetbrains/kotlin/incremental/ChangesCollector.kt @@ -0,0 +1,189 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.incremental + +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.protobuf.MessageLite +import org.jetbrains.kotlin.serialization.Flags +import org.jetbrains.kotlin.serialization.ProtoBuf +import org.jetbrains.kotlin.serialization.deserialization.NameResolver + +class ChangesCollector { + private val removedMembers = hashMapOf>() + private val changedMembers = hashMapOf>() + private val areSubclassesAffected = hashMapOf() + + fun changes(): List { + val changes = arrayListOf() + + for ((fqName, members) in removedMembers) { + if (members.isNotEmpty()) { + changes.add(ChangeInfo.Removed(fqName, members)) + } + } + + for ((fqName, members) in changedMembers) { + if (members.isNotEmpty()) { + changes.add(ChangeInfo.MembersChanged(fqName, members)) + } + } + + for ((fqName, areSubclassesAffected) in areSubclassesAffected) { + changes.add(ChangeInfo.SignatureChanged(fqName, areSubclassesAffected)) + } + + return changes + } + + private fun MutableMap>.getSet(key: T) = + getOrPut(key) { HashSet() } + + private fun collectChangedMember(scope: FqName, name: String) { + changedMembers.getSet(scope).add(name) + } + + private fun collectRemovedMember(scope: FqName, name: String) { + removedMembers.getSet(scope).add(name) + } + + private fun collectChangedMembers(scope: FqName, names: Collection) { + if (names.isNotEmpty()) { + changedMembers.getSet(scope).addAll(names) + } + } + + private fun collectRemovedMembers(scope: FqName, names: Collection) { + if (names.isNotEmpty()) { + removedMembers.getSet(scope).addAll(names) + } + } + + fun collectProtoChanges(oldData: ProtoData?, newData: ProtoData?) { + if (oldData == null && newData == null) { + throw IllegalStateException("Old and new value are null") + } + + if (oldData == null) { + newData!!.collectAll(isRemoved = false) + return + } + + if (newData == null) { + oldData.collectAll(isRemoved = true) + return + } + + when (oldData) { + is ClassProtoData -> { + when (newData) { + is ClassProtoData -> { + val fqName = oldData.nameResolver.getClassId(oldData.proto.fqName).asSingleFqName() + val diff = DifferenceCalculatorForClass(oldData, newData).difference() + if (diff.isClassAffected) { + collectSignature(oldData, diff.areSubclassesAffected) + } + collectChangedMembers(fqName, diff.changedMembersNames) + } + is PackagePartProtoData -> { + collectSignature(oldData, areSubclassesAffected = true) + } + } + } + is PackagePartProtoData -> { + when (newData) { + is ClassProtoData -> { + collectSignature(newData, areSubclassesAffected = false) + } + is PackagePartProtoData -> { + val diff = DifferenceCalculatorForPackageFacade(oldData, newData).difference() + collectChangedMembers(oldData.packageFqName, diff.changedMembersNames) + } + } + } + } + } + + private fun T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List): Set = + members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet() + + private fun ProtoData.collectAll(isRemoved: Boolean) = + when (this) { + is PackagePartProtoData -> collectAllFromPackage(isRemoved) + is ClassProtoData -> collectAllFromClass(isRemoved) + } + + private fun PackagePartProtoData.collectAllFromPackage(isRemoved: Boolean) { + val memberNames = + proto.getNonPrivateNames( + nameResolver, + ProtoBuf.Package::getFunctionList, + ProtoBuf.Package::getPropertyList + ) + + if (isRemoved) { + collectRemovedMembers(packageFqName, memberNames) + } + else { + collectChangedMembers(packageFqName, memberNames) + } + } + + private fun ClassProtoData.collectAllFromClass(isRemoved: Boolean) { + val classFqName = nameResolver.getClassId(proto.fqName).asSingleFqName() + val kind = Flags.CLASS_KIND.get(proto.flags) + + if (kind == ProtoBuf.Class.Kind.COMPANION_OBJECT) { + val memberNames = + proto.getNonPrivateNames( + nameResolver, + ProtoBuf.Class::getConstructorList, + ProtoBuf.Class::getFunctionList, + ProtoBuf.Class::getPropertyList + ) + proto.enumEntryList.map { nameResolver.getString(it.name) } + + val collectMember = if (isRemoved) this@ChangesCollector::collectRemovedMember else this@ChangesCollector::collectChangedMember + collectMember(classFqName.parent(), classFqName.shortName().asString()) + memberNames.forEach { collectMember(classFqName, it) } + } + else { + collectSignature(classFqName, areSubclassesAffected = true) + } + } + + fun collectMemberIfValueWasChanged(scope: FqName, name: String, oldValue: Any?, newValue: Any?) { + if (oldValue == null && newValue == null) { + throw IllegalStateException("Old and new value are null for $scope#$name") + } + + if (oldValue != null && newValue == null) { + collectRemovedMember(scope, name) + } + else if (oldValue != newValue) { + collectChangedMember(scope, name) + } + } + + private fun collectSignature(classData: ClassProtoData, areSubclassesAffected: Boolean) { + val fqName = classData.nameResolver.getClassId(classData.proto.fqName).asSingleFqName() + collectSignature(fqName, areSubclassesAffected) + } + + fun collectSignature(fqName: FqName, areSubclassesAffected: Boolean) { + val prevValue = this.areSubclassesAffected[fqName] ?: false + this.areSubclassesAffected[fqName] = prevValue || areSubclassesAffected + } +} \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt index 3cce58ae0d0..e040662d1ad 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt @@ -18,14 +18,11 @@ package org.jetbrains.kotlin.incremental import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.util.io.FileUtil.toSystemIndependentName -import com.intellij.util.SmartList import com.intellij.util.io.BooleanDataDescriptor import com.intellij.util.io.EnumeratorStringDescriptor import gnu.trove.THashSet import org.jetbrains.annotations.TestOnly import org.jetbrains.kotlin.build.GeneratedJvmClass -import org.jetbrains.kotlin.incremental.ChangeInfo.MembersChanged -import org.jetbrains.kotlin.incremental.ChangeInfo.Removed import org.jetbrains.kotlin.incremental.storage.* import org.jetbrains.kotlin.inline.inlineFunctionsJvmNames import org.jetbrains.kotlin.load.kotlin.ModuleMapping @@ -33,11 +30,7 @@ import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto import org.jetbrains.kotlin.name.FqName -import org.jetbrains.kotlin.protobuf.MessageLite import org.jetbrains.kotlin.resolve.jvm.JvmClassName -import org.jetbrains.kotlin.serialization.Flags -import org.jetbrains.kotlin.serialization.ProtoBuf -import org.jetbrains.kotlin.serialization.deserialization.NameResolver import org.jetbrains.kotlin.serialization.jvm.BitEncoding import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil import org.jetbrains.org.objectweb.asm.* @@ -106,15 +99,14 @@ open class IncrementalCacheImpl( return toSystemIndependentName(File(outputDir, "$internalClassName.class").canonicalPath) } - fun saveModuleMappingToCache(sourceFiles: Collection, file: File): CompilationResult { + fun saveModuleMappingToCache(sourceFiles: Collection, file: File) { val jvmClassName = JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME) protoMap.storeModuleMapping(jvmClassName, file.readBytes()) dirtyOutputClassesMap.notDirty(MODULE_MAPPING_FILE_NAME) sourceFiles.forEach { sourceToClassesMap.add(it, jvmClassName) } - return CompilationResult.NO_CHANGES } - open fun saveFileToCache(generatedClass: GeneratedJvmClass): CompilationResult { + open fun saveFileToCache(generatedClass: GeneratedJvmClass, changesCollector: ChangesCollector) { val sourceFiles: Collection = generatedClass.sourceFiles val kotlinClass: LocalFileKotlinClass = generatedClass.outputClass val className = kotlinClass.className @@ -126,19 +118,17 @@ open class IncrementalCacheImpl( internalNameToSource[className.internalName] = sourceFiles - if (kotlinClass.classId.isLocal) { - return CompilationResult.NO_CHANGES - } + if (kotlinClass.classId.isLocal) return val header = kotlinClass.classHeader - val changesInfo = when (header.kind) { + when (header.kind) { KotlinClassHeader.Kind.FILE_FACADE -> { assert(sourceFiles.size == 1) { "Package part from several source files: $sourceFiles" } packagePartMap.addPackagePart(className) - protoMap.process(kotlinClass, isPackage = true) + - constantsMap.process(kotlinClass, isPackage = true) + - inlineFunctionsMap.process(kotlinClass, isPackage = true) + protoMap.process(kotlinClass, changesCollector) + constantsMap.process(kotlinClass, changesCollector) + inlineFunctionsMap.process(kotlinClass, changesCollector) } KotlinClassHeader.Kind.MULTIFILE_CLASS -> { val partNames = kotlinClass.classHeader.data?.toList() @@ -148,118 +138,43 @@ open class IncrementalCacheImpl( // the class' proto wouldn't ever be deleted, // because we don't write proto for multifile facades. // As a workaround we can remove proto values for multifile facades. - val additionalChangeInfo = if (className in protoMap) { - val info = ChangeInfo.SignatureChanged(className.fqNameForClassNameWithoutDollars, areSubclassesAffected = true) - CompilationResult(changes = sequenceOf(info)) + if (className in protoMap) { + changesCollector.collectSignature(className.fqNameForClassNameWithoutDollars, areSubclassesAffected = true) } - else CompilationResult.NO_CHANGES - protoMap.remove(className) + protoMap.remove(className, changesCollector) classFqNameToSourceMap.remove(className.fqNameForClassNameWithoutDollars) internalNameToSource.remove(className.internalName) // TODO NO_CHANGES? (delegates only) - constantsMap.process(kotlinClass, isPackage = true) + - inlineFunctionsMap.process(kotlinClass, isPackage = true) + - additionalChangeInfo + constantsMap.process(kotlinClass, changesCollector) + inlineFunctionsMap.process(kotlinClass, changesCollector) } KotlinClassHeader.Kind.MULTIFILE_CLASS_PART -> { assert(sourceFiles.size == 1) { "Multifile class part from several source files: $sourceFiles" } packagePartMap.addPackagePart(className) partToMultifileFacade.set(className.internalName, header.multifileClassName!!) - protoMap.process(kotlinClass, isPackage = true) + - constantsMap.process(kotlinClass, isPackage = true) + - inlineFunctionsMap.process(kotlinClass, isPackage = true) + protoMap.process(kotlinClass, changesCollector) + constantsMap.process(kotlinClass, changesCollector) + inlineFunctionsMap.process(kotlinClass, changesCollector) } KotlinClassHeader.Kind.CLASS -> { assert(sourceFiles.size == 1) { "Class is expected to have only one source file: $sourceFiles" } addToClassStorage(kotlinClass, sourceFiles.first()) - protoMap.process(kotlinClass, isPackage = false) + - constantsMap.process(kotlinClass, isPackage = false) + - inlineFunctionsMap.process(kotlinClass, isPackage = false) - } - else -> CompilationResult.NO_CHANGES - } - - changesInfo.logIfSomethingChanged(className) - return changesInfo - } - - private fun CompilationResult.logIfSomethingChanged(className: JvmClassName) { - if (this == CompilationResult.NO_CHANGES) return - - debugLog("$className is changed: $this") - } - - private fun computeChanges(className: JvmClassName, createChangeInfo: (FqName, Collection) -> ChangeInfo): List { - if (className.internalName == MODULE_MAPPING_FILE_NAME) return emptyList() - - val mapValue = protoMap[className] ?: return emptyList() - - return when { - mapValue.isPackageFacade -> { - val (nameResolver, packageProto) = JvmProtoBufUtil.readPackageDataFrom(mapValue.bytes, mapValue.strings) - computePackageChanges(className.packageFqName, packageProto, nameResolver, createChangeInfo) - } - else -> { - val (nameResolver, classProto) = JvmProtoBufUtil.readClassDataFrom(mapValue.bytes, mapValue.strings) - computeClassChanges(nameResolver, classProto, createChangeInfo) + protoMap.process(kotlinClass, changesCollector) + constantsMap.process(kotlinClass, changesCollector) + inlineFunctionsMap.process(kotlinClass, changesCollector) } } } - private fun T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List): Set = - members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet() - - private fun computePackageChanges( - packageFqName: FqName, - protoData: ProtoBuf.Package, - nameResolver: NameResolver, - createChangeInfo: (FqName, Collection) -> ChangeInfo - ): List { - val memberNames = - protoData.getNonPrivateNames( - nameResolver, - ProtoBuf.Package::getFunctionList, - ProtoBuf.Package::getPropertyList - ) - - return listOf(createChangeInfo(packageFqName, memberNames)) - } - - private fun computeClassChanges(nameResolver: NameResolver, classProto: ProtoBuf.Class, createChangeInfo: (FqName, Collection) -> ChangeInfo): List { - val classFqName = nameResolver.getClassId(classProto.fqName).asSingleFqName() - val kind = Flags.CLASS_KIND.get(classProto.flags) - - return if (kind == ProtoBuf.Class.Kind.COMPANION_OBJECT) { - val memberNames = - classProto.getNonPrivateNames( - nameResolver, - ProtoBuf.Class::getConstructorList, - ProtoBuf.Class::getFunctionList, - ProtoBuf.Class::getPropertyList - ) + classProto.enumEntryList.map { nameResolver.getString(it.name) } - - val companionObjectChanged = createChangeInfo(classFqName.parent(), listOfNotNull(classFqName.shortName().asString())) - val companionObjectMembersChanged = createChangeInfo(classFqName, memberNames) - - listOf(companionObjectMembersChanged, companionObjectChanged) - } - else { - listOf(ChangeInfo.SignatureChanged(classFqName, areSubclassesAffected = true)) - } - } - - fun clearCacheForRemovedClasses(): CompilationResult { + fun clearCacheForRemovedClasses(changesCollector: ChangesCollector) { val dirtyClasses = dirtyOutputClassesMap .getDirtyOutputClasses() .map(JvmClassName::byInternalName) .toList() - val changes = dirtyClasses.flatMap { computeChanges(it, ::Removed) }.asSequence() - val changesInfo = CompilationResult(changes = changes) - val facadesWithRemovedParts = hashMapOf>() for (dirtyClass in dirtyClasses) { val facade = partToMultifileFacade.get(dirtyClass.internalName) ?: continue @@ -281,7 +196,7 @@ open class IncrementalCacheImpl( } dirtyClasses.forEach { - protoMap.remove(it) + protoMap.remove(it, changesCollector) packagePartMap.remove(it) multifileFacadeToParts.remove(it) partToMultifileFacade.remove(it) @@ -293,7 +208,6 @@ open class IncrementalCacheImpl( removeAllFromClassStorage(dirtyClasses.map { it.fqNameForClassNameWithoutDollars }) dirtyOutputClassesMap.clean() - return changesInfo } override fun getObsoletePackageParts(): Collection { @@ -335,10 +249,8 @@ open class IncrementalCacheImpl( private inner class ProtoMap(storageFile: File) : BasicStringMap(storageFile, ProtoMapValueExternalizer) { - fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean): CompilationResult { - val header = kotlinClass.classHeader - val bytes = BitEncoding.decodeBytes(header.data!!) - return put(kotlinClass.className, bytes, header.strings!!, isPackage) + fun process(kotlinClass: LocalFileKotlinClass, changesCollector: ChangesCollector) { + return put(kotlinClass, changesCollector) } // A module mapping (.kotlin_module file) is stored in a cache, @@ -348,47 +260,22 @@ open class IncrementalCacheImpl( // from files compiled during last round. // However there is no need to compare old and new data in this case // (also that would fail with exception). - fun storeModuleMapping(className: JvmClassName, bytes: ByteArray): CompilationResult { + fun storeModuleMapping(className: JvmClassName, bytes: ByteArray) { storage[className.internalName] = ProtoMapValue(isPackageFacade = false, bytes = bytes, strings = emptyArray()) - return CompilationResult() } - private fun put( - className: JvmClassName, - bytes: ByteArray, - strings: Array, - isPackage: Boolean - ): CompilationResult { - val key = className.internalName + private fun put(kotlinClass: LocalFileKotlinClass, changesCollector: ChangesCollector) { + val header = kotlinClass.classHeader + + val key = kotlinClass.className.internalName val oldData = storage[key] - val data = ProtoMapValue(isPackage, bytes, strings) + val newData = ProtoMapValue(header.kind != KotlinClassHeader.Kind.CLASS, + BitEncoding.decodeBytes(header.data!!), + header.strings!!) + storage[key] = newData - if (oldData == null || - !Arrays.equals(bytes, oldData.bytes) || - !Arrays.equals(strings, oldData.strings) || - isPackage != oldData.isPackageFacade - ) { - storage[key] = data - } - - if (oldData == null) { - val changes = computeChanges(className, ::MembersChanged).asSequence() - return CompilationResult(changes = changes) - } - - val difference = difference(oldData, data) - val fqName = if (isPackage) className.packageFqName else className.fqNameForClassNameWithoutDollars - val changeList = SmartList() - - if (difference.isClassAffected) { - changeList.add(ChangeInfo.SignatureChanged(fqName, difference.areSubclassesAffected)) - } - - if (difference.changedMembersNames.isNotEmpty()) { - changeList.add(ChangeInfo.MembersChanged(fqName, difference.changedMembersNames)) - } - - return CompilationResult(changes = changeList.asSequence()) + val packageFqName = kotlinClass.className.packageFqName + changesCollector.collectProtoChanges(oldData?.toProtoData(packageFqName), newData.toProtoData(packageFqName)) } operator fun contains(className: JvmClassName): Boolean = @@ -397,8 +284,11 @@ open class IncrementalCacheImpl( operator fun get(className: JvmClassName): ProtoMapValue? = storage[className.internalName] - fun remove(className: JvmClassName) { - storage.remove(className.internalName) + fun remove(className: JvmClassName, changesCollector: ChangesCollector) { + val key = className.internalName + val oldValue = storage[key] ?: return + changesCollector.collectProtoChanges(oldData = oldValue.toProtoData(className.packageFqName), newData = null) + storage.remove(key) } override fun dumpValue(value: ProtoMapValue): String { @@ -406,8 +296,9 @@ open class IncrementalCacheImpl( } } + // todo: reuse code with InlineFunctionsMap? private inner class ConstantsMap(storageFile: File) : BasicStringMap>(storageFile, ConstantsMapExternalizer) { - private fun getConstantsMap(bytes: ByteArray): Map? { + private fun getConstantsMap(bytes: ByteArray): Map { val result = HashMap() ClassReader(bytes).accept(object : ClassVisitor(Opcodes.ASM5) { @@ -420,45 +311,27 @@ open class IncrementalCacheImpl( } }, ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG or ClassReader.SKIP_FRAMES) - return if (result.isEmpty()) null else result + return result } operator fun contains(className: JvmClassName): Boolean = className.internalName in storage - fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean): CompilationResult { - return put(kotlinClass.className, getConstantsMap(kotlinClass.fileContents), isPackage) - } + fun process(kotlinClass: LocalFileKotlinClass, changesCollector: ChangesCollector) { + val key = kotlinClass.className.internalName + val oldMap = storage[key] ?: emptyMap() - private fun put(className: JvmClassName, constantsMap: Map?, isPackage: Boolean): CompilationResult { - val key = className.internalName - - val oldMap = storage[key] - if (oldMap == constantsMap) return CompilationResult.NO_CHANGES - - if (constantsMap != null) { - storage[key] = constantsMap + val newMap = getConstantsMap(kotlinClass.fileContents) + if (newMap.isNotEmpty()) { + storage[key] = newMap } else { - remove(className) + storage.remove(key) } - val changes = - if (constantsMap == null || constantsMap.isEmpty() || - oldMap == null || oldMap.isEmpty() - ) { - emptySequence() - } - else { - // we need only changed constants everything other should be covered by diff - val changedNames = oldMap.filter { constantsMap.containsKey(it.key) && constantsMap[it.key] != it.value }.map { it.key } - - val fqName = if (isPackage) className.packageFqName else className.fqNameForClassNameWithoutDollars - - sequenceOf(ChangeInfo.MembersChanged(fqName, changedNames)) - } - - return CompilationResult(changes = changes) + for (const in oldMap.keys + newMap.keys) { + changesCollector.collectMemberIfValueWasChanged(kotlinClass.scopeFqName(), const, oldMap[const], newMap[const]) + } } fun remove(className: JvmClassName) { @@ -602,39 +475,27 @@ open class IncrementalCacheImpl( return result } - fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean): CompilationResult { - return put(kotlinClass.className, getInlineFunctionsMap(kotlinClass.classHeader, kotlinClass.fileContents), isPackage) - } + fun process(kotlinClass: LocalFileKotlinClass, changesCollector: ChangesCollector) { + val key = kotlinClass.className.internalName + val oldMap = storage[key] ?: emptyMap() - private fun put(className: JvmClassName, newMap: Map, isPackage: Boolean): CompilationResult { - val internalName = className.internalName - val oldMap = storage[internalName] ?: emptyMap() - - val added = hashSetOf() - val changed = hashSetOf() - val allFunctions = oldMap.keys + newMap.keys - - for (fn in allFunctions) { - val oldHash = oldMap[fn] - val newHash = newMap[fn] - - when { - oldHash == null -> added.add(fn) - oldHash != newHash -> changed.add(fn) - } + val newMap = getInlineFunctionsMap(kotlinClass.classHeader, kotlinClass.fileContents) + if (newMap.isNotEmpty()) { + storage[key] = newMap + } + else { + storage.remove(key) } - when { - newMap.isNotEmpty() -> storage[internalName] = newMap - else -> storage.remove(internalName) + for (fn in oldMap.keys + newMap.keys) { + changesCollector.collectMemberIfValueWasChanged(kotlinClass.scopeFqName(), functionNameBySignature(fn), oldMap[fn], newMap[fn]) } - - val fqName = if (isPackage) className.packageFqName else className.fqNameForClassNameWithoutDollars - // TODO get name in better way instead of using substringBefore - val changes = (added.asSequence() + changed.asSequence()).map { ChangeInfo.MembersChanged(fqName, listOf(it.substringBefore("("))) } - return CompilationResult(changes = changes) } + // TODO get name in better way instead of using substringBefore + private fun functionNameBySignature(signature: String): String = + signature.substringBefore("(") + fun remove(className: JvmClassName) { storage.remove(className.internalName) } @@ -663,16 +524,11 @@ sealed class ChangeInfo(val fqName: FqName) { } } -data class CompilationResult( - val changes: Sequence = emptySequence() -) { - companion object { - val NO_CHANGES: CompilationResult = CompilationResult() - } - - operator fun plus(other: CompilationResult): CompilationResult = - CompilationResult(changes + other.changes) -} +private fun LocalFileKotlinClass.scopeFqName() = + when (classHeader.kind) { + KotlinClassHeader.Kind.CLASS -> className.fqNameForClassNameWithoutDollars + else -> className.packageFqName + } fun ByteArray.md5(): Long { val d = MessageDigest.getInstance("MD5").digest(this)!! diff --git a/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt b/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt index 948e39f187b..c28b1a77e7e 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt @@ -78,20 +78,17 @@ fun makeCompileServices( fun updateIncrementalCache( generatedFiles: Iterable, - cache: IncrementalCacheImpl -): CompilationResult { - var changesInfo = CompilationResult.NO_CHANGES + cache: IncrementalCacheImpl, + changesCollector: ChangesCollector +) { for (generatedFile in generatedFiles) { when { - generatedFile is GeneratedJvmClass -> changesInfo += cache.saveFileToCache(generatedFile) - generatedFile.outputFile.isModuleMappingFile() -> changesInfo += cache.saveModuleMappingToCache(generatedFile.sourceFiles, generatedFile.outputFile) + generatedFile is GeneratedJvmClass -> cache.saveFileToCache(generatedFile, changesCollector) + generatedFile.outputFile.isModuleMappingFile() -> cache.saveModuleMappingToCache(generatedFile.sourceFiles, generatedFile.outputFile) } } - val newChangesInfo = cache.clearCacheForRemovedClasses() - changesInfo += newChangesInfo - - return changesInfo + cache.clearCacheForRemovedClasses(changesCollector) } fun LookupStorage.update( @@ -111,14 +108,14 @@ data class DirtyData( val dirtyClassesFqNames: Collection = emptyList() ) -fun CompilationResult.getDirtyData( +fun ChangesCollector.getDirtyData( caches: Iterable, reporter: ICReporter ): DirtyData { val dirtyLookupSymbols = HashSet() val dirtyClassesFqNames = HashSet() - for (change in changes) { + for (change in changes()) { reporter.report { "Process $change" } if (change is ChangeInfo.SignatureChanged) { diff --git a/build-common/src/org/jetbrains/kotlin/incremental/protoDifferenceUtils.kt b/build-common/src/org/jetbrains/kotlin/incremental/protoDifferenceUtils.kt index 1bfa16fc7fb..80f949c1304 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/protoDifferenceUtils.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/protoDifferenceUtils.kt @@ -16,11 +16,11 @@ package org.jetbrains.kotlin.incremental -import org.jetbrains.annotations.TestOnly import org.jetbrains.kotlin.descriptors.Visibilities import org.jetbrains.kotlin.incremental.ProtoCompareGenerated.ProtoBufClassKind import org.jetbrains.kotlin.incremental.ProtoCompareGenerated.ProtoBufPackageKind import org.jetbrains.kotlin.incremental.storage.ProtoMapValue +import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.protobuf.MessageLite import org.jetbrains.kotlin.serialization.Flags import org.jetbrains.kotlin.serialization.ProtoBuf @@ -32,48 +32,23 @@ import java.util.* data class Difference( val isClassAffected: Boolean = false, val areSubclassesAffected: Boolean = false, - val changedMembersNames: Set = emptySet(), - @TestOnly - val rawProtoDifference: EnumSet<*>? = null + val changedMembersNames: Set = emptySet() ) sealed class ProtoData data class ClassProtoData(val proto: ProtoBuf.Class, val nameResolver: NameResolver) : ProtoData() -data class PackagePartProtoData(val proto: ProtoBuf.Package, val nameResolver: NameResolver) : ProtoData() +data class PackagePartProtoData(val proto: ProtoBuf.Package, val nameResolver: NameResolver, val packageFqName: FqName) : ProtoData() -fun ProtoMapValue.toProtoData(): ProtoData = +fun ProtoMapValue.toProtoData(packageFqName: FqName): ProtoData = if (isPackageFacade) { val packageData = JvmProtoBufUtil.readPackageDataFrom(bytes, strings) - PackagePartProtoData(packageData.packageProto, packageData.nameResolver) + PackagePartProtoData(packageData.packageProto, packageData.nameResolver, packageFqName) } else { val classData = JvmProtoBufUtil.readClassDataFrom(bytes, strings) ClassProtoData(classData.classProto, classData.nameResolver) } -fun difference(oldValue: ProtoMapValue, newValue: ProtoMapValue): Difference = - difference(oldValue.toProtoData(), newValue.toProtoData()) - -fun difference(oldData: ProtoData, newData: ProtoData): Difference = - when (oldData) { - is ClassProtoData -> { - when (newData) { - is ClassProtoData -> - DifferenceCalculatorForClass(oldData, newData).difference() - is PackagePartProtoData -> - Difference(isClassAffected = true, areSubclassesAffected = true) - } - } - is PackagePartProtoData -> { - when (newData) { - is ClassProtoData -> - Difference(isClassAffected = true) - is PackagePartProtoData -> - DifferenceCalculatorForPackageFacade(oldData, newData).difference() - } - } - } - internal val MessageLite.isPrivate: Boolean get() = Visibilities.isPrivate(Deserialization.visibility( when (this) { @@ -96,7 +71,7 @@ private fun MessageLite.name(nameResolver: NameResolver): String { internal fun List.names(nameResolver: NameResolver): List = map { it.name(nameResolver) } -private abstract class DifferenceCalculator { +abstract class DifferenceCalculator { protected abstract val compareObject: ProtoCompareGenerated abstract fun difference(): Difference @@ -179,7 +154,7 @@ private abstract class DifferenceCalculator { } } -private class DifferenceCalculatorForClass( +class DifferenceCalculatorForClass( private val oldData: ClassProtoData, private val newData: ClassProtoData ) : DifferenceCalculator() { @@ -272,11 +247,11 @@ private class DifferenceCalculatorForClass( } } - return Difference(isClassAffected, areSubclassesAffected, names, rawProtoDifference = diff) + return Difference(isClassAffected, areSubclassesAffected, names) } } -private class DifferenceCalculatorForPackageFacade( +class DifferenceCalculatorForPackageFacade( private val oldData: PackagePartProtoData, private val newData: PackagePartProtoData ) : DifferenceCalculator() { @@ -317,7 +292,7 @@ private class DifferenceCalculatorForPackageFacade( } } - return Difference(changedMembersNames = names, rawProtoDifference = diff) + return Difference(changedMembersNames = names) } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt index 86d5bb564d1..a1cc1997772 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt @@ -135,7 +135,12 @@ abstract class IncrementalCompilerRunner< caches.platformCache.markDirty(dirtySources) } - protected abstract fun compareAndUpdateCache(services: Services, caches: CacheManager, generatedFiles: List): CompilationResult + protected abstract fun updateCaches( + services: Services, + caches: CacheManager, + generatedFiles: List, + changesCollector: ChangesCollector + ) protected open fun preBuildHook(args: Args, compilationMode: CompilationMode) {} protected open fun postCompilationHook(exitCode: ExitCode) {} @@ -226,11 +231,12 @@ abstract class IncrementalCompilerRunner< allGeneratedFiles.addAll(generatedFiles) caches.inputsCache.registerOutputForSourceFiles(generatedFiles) caches.lookupCache.update(lookupTracker, sourcesToCompile, removedKotlinSources) - val compilationResult = compareAndUpdateCache(services, caches, generatedFiles) + val changesCollector = ChangesCollector() + updateCaches(services, caches, generatedFiles, changesCollector) if (compilationMode is CompilationMode.Rebuild) break - val (dirtyLookupSymbols, dirtyClassFqNames) = compilationResult.getDirtyData(listOf(caches.platformCache), reporter) + val (dirtyLookupSymbols, dirtyClassFqNames) = changesCollector.getDirtyData(listOf(caches.platformCache), reporter) val compiledInThisIterationSet = sourcesToCompile.toHashSet() with (dirtySources) { diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt index 8c6b3025854..7d0da5ec5de 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalJvmCompilerRunner.kt @@ -211,8 +211,14 @@ class IncrementalJvmCompilerRunner( } } - override fun compareAndUpdateCache(services: Services, caches: IncrementalJvmCachesManager, generatedFiles: List): CompilationResult = - updateIncrementalCache(generatedFiles, caches.platformCache) + override fun updateCaches( + services: Services, + caches: IncrementalJvmCachesManager, + generatedFiles: List, + changesCollector: ChangesCollector + ) { + updateIncrementalCache(generatedFiles, caches.platformCache, changesCollector) + } override fun additionalDirtyFiles( caches: IncrementalJvmCachesManager, diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJsProtoComparisonTest.kt b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJsProtoComparisonTest.kt index 547ad15ea69..4359b01d3e0 100644 --- a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJsProtoComparisonTest.kt +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJsProtoComparisonTest.kt @@ -21,11 +21,8 @@ import org.jetbrains.kotlin.cli.common.arguments.K2JsArgumentConstants import org.jetbrains.kotlin.cli.js.K2JSCompiler import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl import org.jetbrains.kotlin.config.Services -import org.jetbrains.kotlin.incremental.ClassProtoData -import org.jetbrains.kotlin.incremental.Difference -import org.jetbrains.kotlin.incremental.difference as differenceImpl -import org.jetbrains.kotlin.incremental.PackagePartProtoData import org.jetbrains.kotlin.incremental.ProtoData +import org.jetbrains.kotlin.incremental.* import org.jetbrains.kotlin.incremental.utils.TestMessageCollector import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl @@ -72,28 +69,33 @@ abstract class AbstractJsProtoComparisonTest : AbstractProtoComparisonTest() for ((sourceFile, protoBytes, _) in incrementalResults.packageParts) { - val proto = ProtoBuf.PackageFragment.parseFrom(protoBytes, JsSerializerProtocol.extensionRegistry) - val nameResolver = NameResolverImpl(proto.strings, proto.qualifiedNames) - - proto.class_List.forEach { - val classId = nameResolver.getClassId(it.fqName) - classes[classId] = ClassProtoData(it, nameResolver) - } - - proto.`package`.apply { - val packageFqName = if (hasExtension(JsProtoBuf.packageFqName)) { - nameResolver.getPackageFqName(getExtension(JsProtoBuf.packageFqName)) - } - else FqName.ROOT - - val packagePartClassId = ClassId(packageFqName, Name.identifier(sourceFile.nameWithoutExtension.capitalize() + "Kt")) - classes[packagePartClassId] = PackagePartProtoData(this, nameResolver) - } + classes.putAll(getProtoData(sourceFile, protoBytes)) } return classes } - override fun difference(oldData: ProtoData, newData: ProtoData): Difference? = - differenceImpl(oldData, newData) + override fun ProtoData.toProtoData(): ProtoData? = this } + +fun getProtoData(sourceFile: File, metadata: ByteArray): Map { + val classes = hashMapOf() + val proto = ProtoBuf.PackageFragment.parseFrom(metadata, JsSerializerProtocol.extensionRegistry) + val nameResolver = NameResolverImpl(proto.strings, proto.qualifiedNames) + + proto.class_List.forEach { + val classId = nameResolver.getClassId(it.fqName) + classes[classId] = ClassProtoData(it, nameResolver) + } + + proto.`package`.apply { + val packageFqName = if (hasExtension(JsProtoBuf.packageFqName)) { + nameResolver.getPackageFqName(getExtension(JsProtoBuf.packageFqName)) + } + else FqName.ROOT + + val packagePartClassId = ClassId(packageFqName, Name.identifier(sourceFile.nameWithoutExtension.capitalize() + "Kt")) + classes[packagePartClassId] = PackagePartProtoData(this, nameResolver, packageFqName) + } + return classes +} \ No newline at end of file diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJvmProtoComparisonTest.kt b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJvmProtoComparisonTest.kt index f14ea22eab1..628fcab36aa 100644 --- a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJvmProtoComparisonTest.kt +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractJvmProtoComparisonTest.kt @@ -16,9 +16,7 @@ package org.jetbrains.kotlin.jps.incremental -import org.jetbrains.kotlin.incremental.Difference -import org.jetbrains.kotlin.incremental.LocalFileKotlinClass -import org.jetbrains.kotlin.incremental.difference +import org.jetbrains.kotlin.incremental.* import org.jetbrains.kotlin.incremental.storage.ProtoMapValue import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinaryClass import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader @@ -36,25 +34,20 @@ abstract class AbstractJvmProtoComparisonTest : AbstractProtoComparisonTest { - ProtoMapValue(false, bytes, strings) + ProtoMapValue(false, bytes, strings).toProtoData(packageFqName) } KotlinClassHeader.Kind.FILE_FACADE, KotlinClassHeader.Kind.MULTIFILE_CLASS_PART -> { - ProtoMapValue(true, bytes, strings) + ProtoMapValue(true, bytes, strings).toProtoData(packageFqName) } else -> { null diff --git a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractProtoComparisonTest.kt b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractProtoComparisonTest.kt index a74fba2f39f..9e1c15e1268 100644 --- a/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractProtoComparisonTest.kt +++ b/jps-plugin/jps-tests/test/org/jetbrains/kotlin/jps/incremental/AbstractProtoComparisonTest.kt @@ -17,7 +17,7 @@ package org.jetbrains.kotlin.jps.incremental import org.jetbrains.kotlin.TestWithWorkingDir -import org.jetbrains.kotlin.incremental.Difference +import org.jetbrains.kotlin.incremental.* import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.test.KotlinTestUtils import org.jetbrains.kotlin.utils.Printer @@ -25,7 +25,7 @@ import java.io.File abstract class AbstractProtoComparisonTest : TestWithWorkingDir() { protected abstract fun compileAndGetClasses(sourceDir: File, outputDir: File): Map - protected abstract fun difference(oldData: PROTO_DATA, newData: PROTO_DATA): Difference? + protected abstract fun PROTO_DATA.toProtoData(): ProtoData? protected open fun expectedOutputFile(testDir: File): File = File(testDir, "result.out") @@ -48,32 +48,41 @@ abstract class AbstractProtoComparisonTest : TestWithWorkingDir() { } (oldClassMap.keys.intersect(newClassMap.keys)).sortedBy { it.toString() }.forEach { classId -> - val oldData = oldClassMap[classId]!! - val newData = newClassMap[classId]!! - val diff = difference(oldData, newData) + val oldData = oldClassMap[classId]!!.toProtoData() + val newData = newClassMap[classId]!!.toProtoData() - if (diff == null) { + if (oldData == null || newData == null) { p.println("SKIPPED $classId") return@forEach } - diff.rawProtoDifference?.let { + val rawProtoDifference = when { + oldData is ClassProtoData && newData is ClassProtoData -> { + ProtoCompareGenerated(oldData.nameResolver, newData.nameResolver).difference(oldData.proto, newData.proto) + } + oldData is PackagePartProtoData && newData is PackagePartProtoData -> { + ProtoCompareGenerated(oldData.nameResolver, newData.nameResolver).difference(oldData.proto, newData.proto) + } + else -> null + } + rawProtoDifference?.let { if (it.isNotEmpty()) { p.println("PROTO DIFFERENCE in $classId: ${it.sortedBy { it.name }.joinToString()}") } } - val changes = arrayListOf() - if (diff.isClassAffected) { - changes.add("CLASS_SIGNATURE") - } - if (diff.changedMembersNames.isNotEmpty()) { - changes.add("MEMBERS\n ${diff.changedMembersNames.sorted()}") - } - if (changes.isEmpty()) { + val changesInfo = ChangesCollector().apply { collectProtoChanges(oldData, newData) }.changes() + if (changesInfo.isEmpty()) { return@forEach } + val changes = changesInfo.map { + when (it) { + is ChangeInfo.SignatureChanged -> "CLASS_SIGNATURE" + is ChangeInfo.MembersChanged -> "MEMBERS\n ${it.names.sorted()}" + } + }.sorted() + p.println("CHANGES in $classId: ${changes.joinToString()}") } diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index da1f8c92fa8..1f131ee18c6 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -314,8 +314,9 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { context.checkCanceled() - val changesInfo = generatedFiles.entries.fold(CompilationResult.NO_CHANGES) { acc, (target, files) -> - acc + updateIncrementalCache(files, incrementalCaches[target]!!) + val changesCollector = ChangesCollector() + for ((target, files) in generatedFiles) { + updateIncrementalCache(files, incrementalCaches[target]!!, changesCollector) } updateLookupStorage(chunk, lookupTracker, dataManager, dirtyFilesHolder, filesToCompile) @@ -323,7 +324,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { return OK } - changesInfo.processChangesUsingLookups(filesToCompile.values().toSet(), dataManager, fsOperations, incrementalCaches.values) + changesCollector.processChangesUsingLookups(filesToCompile.values().toSet(), dataManager, fsOperations, incrementalCaches.values) return OK } @@ -432,7 +433,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) { for (target in chunk.targets) { val cache = incrementalCaches[target]!! val removedAndDirtyFiles = filesToCompile[target] + dirtyFilesHolder.getRemovedFiles(target).map(::File) - cache.markOutputClassesDirty(removedAndDirtyFiles) + cache.markDirty(removedAndDirtyFiles) } } @@ -774,7 +775,7 @@ private class JpsICReporter : ICReporter { } } -private fun CompilationResult.processChangesUsingLookups( +private fun ChangesCollector.processChangesUsingLookups( compiledFiles: Set, dataManager: BuildDataManager, fsOperations: FSOperationsHelper,