diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt index 8586614bbdd..d8bf653bbab 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/build/KotlinBuilder.kt @@ -205,14 +205,17 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR updateLookupStorage(chunk, lookupTracker, dataManager, dirtyFilesHolder, filesToCompile) val caches = filesToCompile.keySet().map { incrementalCaches[it]!! } - processChanges(context, chunk, allCompiledFiles, caches, changesInfo) + processChanges(context, chunk, filesToCompile.values(), allCompiledFiles, dataManager, caches, changesInfo) + return ADDITIONAL_PASS_REQUIRED } - fun processChanges( + private fun processChanges( context: CompileContext, chunk: ModuleChunk, + compiledFiles: Collection, allCompiledFiles: MutableSet, + dataManager: BuildDataManager, caches: List, changesInfo: ChangesInfo ) { @@ -250,7 +253,34 @@ public class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR } } - changesInfo.doProcessChanges() + fun ChangesInfo.doProcessChangesUsingLookups() { + val lookupStorage = dataManager.getStorage(KotlinDataContainerTarget, LookupStorageProvider) + + // TODO group by fqName? + for (change in changes) { + + if (change !is ChangeInfo.MembersChanged) continue + + val files = change.names + .flatMap { lookupStorage.get(LookupSymbol(it, change.fqName.asString())) } + .asSequence() + .map { File(it) } + .filter { it !in compiledFiles && it.exists() } + + files.forEach { + FSOperations.markDirty(context, CompilationRound.NEXT, it) + } + } + + caches.forEach { it.cleanDirtyInlineFunctions() } + } + + if (IncrementalCompilation.isExperimental()) { + changesInfo.doProcessChangesUsingLookups() + } + else { + changesInfo.doProcessChanges() + } } private fun doCompileModuleChunk( diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt index a16b0012439..2692a562e7a 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt @@ -16,6 +16,7 @@ package org.jetbrains.kotlin.jps.incremental +import com.google.protobuf.MessageLite import com.intellij.openapi.util.io.FileUtil import com.intellij.util.io.BooleanDataDescriptor import com.intellij.util.io.EnumeratorStringDescriptor @@ -28,6 +29,7 @@ import org.jetbrains.jps.builders.storage.StorageProvider import org.jetbrains.jps.incremental.ModuleBuildTarget import org.jetbrains.jps.incremental.storage.BuildDataManager import org.jetbrains.jps.incremental.storage.PathStringDescriptor +import org.jetbrains.kotlin.config.IncrementalCompilation import org.jetbrains.kotlin.inline.inlineFunctionsJvmNames import org.jetbrains.kotlin.jps.build.GeneratedJvmClass import org.jetbrains.kotlin.jps.build.KotlinBuilder @@ -36,8 +38,12 @@ import org.jetbrains.kotlin.load.kotlin.ModuleMapping import org.jetbrains.kotlin.load.kotlin.header.* import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache import org.jetbrains.kotlin.load.kotlin.incremental.components.JvmPackagePartProto +import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.resolve.jvm.JvmClassName +import org.jetbrains.kotlin.serialization.ProtoBuf +import org.jetbrains.kotlin.serialization.deserialization.NameResolver import org.jetbrains.kotlin.serialization.jvm.BitEncoding +import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil import org.jetbrains.org.objectweb.asm.* import java.io.File import java.security.MessageDigest @@ -122,10 +128,14 @@ public class IncrementalCacheImpl( dependents.forEach(::addFilesAffectedByChangedInlineFuns) } - dirtyInlineFunctionsMap.clean() + cleanDirtyInlineFunctions() return result.map { File(it) } } + public fun cleanDirtyInlineFunctions() { + dirtyInlineFunctionsMap.clean() + } + override fun getClassFilePath(internalClassName: String): String { return File(outputDir, "$internalClassName.class").canonicalPath } @@ -160,9 +170,11 @@ public class IncrementalCacheImpl( assert(sourceFiles.size() == 1) { "Package part from several source files: $sourceFiles" } packagePartMap.addPackagePart(className) - protoMap.process(kotlinClass, isPackage = true) + + val isPackage = true + + protoMap.process(kotlinClass, isPackage) + constantsMap.process(kotlinClass) + - inlineFunctionsMap.process(kotlinClass) + inlineFunctionsMap.process(kotlinClass, isPackage) } header.isCompatibleMultifileClassKind() -> { val partNames = kotlinClass.classHeader.filePartClassNames?.toList() @@ -171,21 +183,25 @@ public class IncrementalCacheImpl( // TODO NO_CHANGES? (delegates only, see package facade) constantsMap.process(kotlinClass) + - inlineFunctionsMap.process(kotlinClass) + inlineFunctionsMap.process(kotlinClass, isPackage = true) } header.isCompatibleMultifileClassPartKind() -> { assert(sourceFiles.size() == 1) { "Multifile class part from several source files: $sourceFiles" } packagePartMap.addPackagePart(className) multifileClassPartMap.add(className.internalName, header.multifileClassName!!) - protoMap.process(kotlinClass, isPackage = true) + + val isPackage = true + + protoMap.process(kotlinClass, isPackage) + constantsMap.process(kotlinClass) + - inlineFunctionsMap.process(kotlinClass) + inlineFunctionsMap.process(kotlinClass, isPackage) } header.isCompatibleClassKind() && !header.isLocalClass -> { - protoMap.process(kotlinClass, isPackage = false) + + val isPackage = false + + protoMap.process(kotlinClass, isPackage) + constantsMap.process(kotlinClass) + - inlineFunctionsMap.process(kotlinClass) + inlineFunctionsMap.process(kotlinClass, isPackage) } else -> ChangesInfo.NO_CHANGES } @@ -201,12 +217,57 @@ public class IncrementalCacheImpl( } public fun clearCacheForRemovedClasses(): ChangesInfo { + + fun T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List) = + members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet() + + fun createChangeInfo(className: JvmClassName): ChangeInfo? { + if (className.internalName == MODULE_MAPPING_FILE_NAME) return null + + val mapValue = protoMap.get(className) ?: return null + + return when { + mapValue.isPackageFacade -> { + val packageData = JvmProtoBufUtil.readPackageDataFrom(mapValue.bytes, mapValue.strings) + + val memberNames = + packageData.packageProto.getNonPrivateNames( + packageData.nameResolver, + ProtoBuf.Package::getFunctionList, + ProtoBuf.Package::getPropertyList + ) + + ChangeInfo.Removed(className.packageFqName, memberNames) + } + else -> { + val classData = JvmProtoBufUtil.readClassDataFrom(mapValue.bytes, mapValue.strings) + + val memberNames = + classData.classProto.getNonPrivateNames( + classData.nameResolver, + ProtoBuf.Class::getConstructorList, + ProtoBuf.Class::getFunctionList, + ProtoBuf.Class::getPropertyList + ) + + classData.classProto.enumEntryList.map { classData.nameResolver.getString(it) }.toSet() + + ChangeInfo.Removed(className.fqNameForClassNameWithoutDollars, memberNames) + } + } + } + val dirtyClasses = dirtyOutputClassesMap .getDirtyOutputClasses() .map(JvmClassName::byInternalName) .toList() - val changesInfo = dirtyClasses.fold(ChangesInfo.NO_CHANGES) { info, className -> + val changes = + if (IncrementalCompilation.isExperimental()) + dirtyClasses.map { createChangeInfo(it) }.asSequence().filterNotNull() + else + emptySequence() + + val changesInfo = dirtyClasses.fold(ChangesInfo(changes = changes)) { info, className -> val newInfo = ChangesInfo(protoChanged = className in protoMap, constantsChanged = className in constantsMap) newInfo.logIfSomethingChanged(className) @@ -268,10 +329,10 @@ public class IncrementalCacheImpl( private inner class ProtoMap(storageFile: File) : BasicStringMap(storageFile, ProtoMapValueExternalizer) { - public fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean, checkChangesIsOpenPart: Boolean = true): ChangesInfo { + public fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean): ChangesInfo { val header = kotlinClass.classHeader val bytes = BitEncoding.decodeBytes(header.annotationData!!) - return put(kotlinClass.className, bytes, header.strings!!, isPackage, checkChangesIsOpenPart) + return put(kotlinClass.className, bytes, header.strings!!, isPackage, checkChangesIsOpenPart = true) } public fun process(className: JvmClassName, data: ByteArray, strings: Array, isPackage: Boolean, checkChangesIsOpenPart: Boolean): ChangesInfo { @@ -288,13 +349,27 @@ public class IncrementalCacheImpl( if (oldData == null || !Arrays.equals(bytes, oldData.bytes) || !Arrays.equals(strings, oldData.strings) || - isPackage != oldData.isPackageFacade) { + isPackage != oldData.isPackageFacade + ) { storage[key] = data } - return ChangesInfo(protoChanged = oldData == null || - !checkChangesIsOpenPart || - difference(oldData, data) != DifferenceKind.NONE) + if (oldData == null || !checkChangesIsOpenPart) return ChangesInfo(protoChanged = true) + + val diff = difference(oldData, data) + + if (!IncrementalCompilation.isExperimental()) return ChangesInfo(protoChanged = diff != DifferenceKind.NONE) + + val fqName = if (isPackage) className.packageFqName else className.fqNameForClassNameWithoutDollars + + val changes = + when (diff) { + is DifferenceKind.NONE -> emptySequence() + is DifferenceKind.CLASS_SIGNATURE -> sequenceOf(ChangeInfo.SignatureChanged(fqName)) + is DifferenceKind.MEMBERS -> sequenceOf(ChangeInfo.MembersChanged(fqName, diff.names)) + } + + return ChangesInfo(protoChanged = diff != DifferenceKind.NONE, changes = changes) } public fun contains(className: JvmClassName): Boolean = @@ -388,11 +463,11 @@ public class IncrementalCacheImpl( return result } - public fun process(kotlinClass: LocalFileKotlinClass): ChangesInfo { - return put(kotlinClass.className, getInlineFunctionsMap(kotlinClass.fileContents)) + public fun process(kotlinClass: LocalFileKotlinClass, isPackage: Boolean): ChangesInfo { + return put(kotlinClass.className, getInlineFunctionsMap(kotlinClass.fileContents), isPackage) } - private fun put(className: JvmClassName, newMap: Map): ChangesInfo { + private fun put(className: JvmClassName, newMap: Map, isPackage: Boolean): ChangesInfo { val internalName = className.internalName val oldMap = storage[internalName] ?: emptyMap() @@ -419,8 +494,19 @@ public class IncrementalCacheImpl( dirtyInlineFunctionsMap.put(className, changed.toList()) } + val changes = + if (IncrementalCompilation.isExperimental()) { + val fqName = if (isPackage) className.packageFqName else className.fqNameForClassNameWithoutDollars + // TODO get name in better way instead of using substringBefore + (added.asSequence() + changed.asSequence()).map { ChangeInfo.MembersChanged(fqName, listOf(it.substringBefore("("))) } + } + else { + emptySequence() + } + return ChangesInfo(inlineChanged = changed.isNotEmpty(), - inlineAdded = added.isNotEmpty()) + inlineAdded = added.isNotEmpty(), + changes = changes) } public fun remove(className: JvmClassName) { @@ -559,11 +645,18 @@ public class IncrementalCacheImpl( } } +sealed class ChangeInfo(val fqName: FqName) { + open class MembersChanged(fqName: FqName, val names: Collection) : ChangeInfo(fqName) + class Removed(fqName: FqName, names: Collection) : MembersChanged(fqName, names) + class SignatureChanged(fqName: FqName) : ChangeInfo(fqName) +} + data class ChangesInfo( - public val protoChanged: Boolean = false, - public val constantsChanged: Boolean = false, - public val inlineChanged: Boolean = false, - public val inlineAdded: Boolean = false + val protoChanged: Boolean = false, + val constantsChanged: Boolean = false, + val inlineChanged: Boolean = false, + val inlineAdded: Boolean = false, + val changes: Sequence = emptySequence() ) { companion object { public val NO_CHANGES: ChangesInfo = ChangesInfo() @@ -573,7 +666,8 @@ data class ChangesInfo( ChangesInfo(protoChanged || other.protoChanged, constantsChanged || other.constantsChanged, inlineChanged || other.inlineChanged, - inlineAdded || other.inlineAdded) + inlineAdded || other.inlineAdded, + changes + other.changes) } @@ -607,9 +701,6 @@ private fun ByteArray.md5(): Long { ) } -private val File.normalizedPath: String - get() = FileUtil.toSystemIndependentName(canonicalPath) - @TestOnly private fun , V> Map.dumpMap(dumpValue: (V)->String): String = StringBuilder { diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupStorage.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupStorage.kt index 349981a99e9..cf32e331995 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupStorage.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/LookupStorage.kt @@ -55,6 +55,16 @@ class LookupStorage(private val targetDataDir: File) : BasicMapsOwner() { } } + fun get(lookupSymbol: LookupSymbol): Collection { + val key = LookupSymbolKey(lookupSymbol.name, lookupSymbol.scope) + val fileIds = lookupMap[key] ?: return emptySet() + + return fileIds.map { + // null means it's outdated + idToFile[it]?.path + }.filterNotNull() + } + public fun add(lookupSymbol: LookupSymbol, containingPaths: Collection) { val key = LookupSymbolKey(lookupSymbol.name, lookupSymbol.scope) val fileIds = containingPaths.map { addFileIfNeeded(File(it)) }.toHashSet() diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/protoDifferenceUtils.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/protoDifferenceUtils.kt index 09191759457..5124aa85eda 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/protoDifferenceUtils.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/protoDifferenceUtils.kt @@ -44,6 +44,26 @@ public fun difference(oldData: ProtoMapValue, newData: ProtoMapValue): Differenc return differenceObject.difference() } +internal val MessageLite.isPrivate: Boolean + get() = Visibilities.isPrivate(Deserialization.visibility( + when (this) { + is ProtoBuf.Constructor -> Flags.VISIBILITY.get(flags) + is ProtoBuf.Function -> Flags.VISIBILITY.get(flags) + is ProtoBuf.Property -> Flags.VISIBILITY.get(flags) + else -> error("Unknown message: $this") + })) + +private fun MessageLite.name(nameResolver: NameResolver): String { + return when (this) { + is ProtoBuf.Constructor -> "" + is ProtoBuf.Function -> nameResolver.getString(name) + is ProtoBuf.Property -> nameResolver.getString(name) + else -> error("Unknown message: $this") + } +} + +internal fun List.names(nameResolver: NameResolver): List = map { it.name(nameResolver) } + private abstract class DifferenceCalculator() { protected abstract val oldNameResolver: NameResolver protected abstract val newNameResolver: NameResolver @@ -57,9 +77,6 @@ private abstract class DifferenceCalculator() { protected fun calcDifferenceForMembers(oldList: List, newList: List): Collection { val result = hashSetOf() - fun List.names(nameResolver: NameResolver): List = - map { it.name(nameResolver) } - val oldMap = oldList.groupBy { it.getHashCode({ compareObject.oldGetIndexOfString(it) }, { compareObject.oldGetIndexOfClassId(it) }) } val newMap = @@ -114,15 +131,6 @@ private abstract class DifferenceCalculator() { return HashSetUtil.symmetricDifference(oldNames, newNames) } - protected val MessageLite.isPrivate: Boolean - get() = Visibilities.isPrivate(Deserialization.visibility( - when (this) { - is ProtoBuf.Constructor -> Flags.VISIBILITY.get(flags) - is ProtoBuf.Function -> Flags.VISIBILITY.get(flags) - is ProtoBuf.Property -> Flags.VISIBILITY.get(flags) - else -> error("Unknown message: $this") - })) - private fun MessageLite.getHashCode(stringIndexes: (Int) -> Int, fqNameIndexes: (Int) -> Int): Int { return when (this) { is ProtoBuf.Constructor -> hashCode(stringIndexes, fqNameIndexes) @@ -132,15 +140,6 @@ private abstract class DifferenceCalculator() { } } - private fun MessageLite.name(nameResolver: NameResolver): String { - return when (this) { - is ProtoBuf.Constructor -> "" - is ProtoBuf.Function -> nameResolver.getString(name) - is ProtoBuf.Property -> nameResolver.getString(name) - else -> error("Unknown message: $this") - } - } - private fun ProtoCompareGenerated.checkEquals(old: MessageLite, new: MessageLite): Boolean { return when { old is ProtoBuf.Constructor && new is ProtoBuf.Constructor -> checkEquals(old, new)