IC of sealed classes

Supported case then children of sealed classes could be declared anywhere in a module. If list of classes implementing sealing class changes the sealed class and all its inheritors should be recompiled (now sealed class should be compiled together with children in order to calculate all possible inheritors at compile time) and and invalidated (as they could have when operators).
2020-12-09 17:33:09 +03:00
parent a0651cdba7
commit 36f99156fd
11 changed files with 183 additions and 20 deletions
@@ -19,6 +19,7 @@ package org.jetbrains.kotlin.incremental
 import com.intellij.util.io.EnumeratorStringDescriptor
 import org.jetbrains.kotlin.incremental.storage.*
 import org.jetbrains.kotlin.metadata.ProtoBuf
+import org.jetbrains.kotlin.metadata.deserialization.Flags
 import org.jetbrains.kotlin.metadata.deserialization.NameResolver
 import org.jetbrains.kotlin.metadata.deserialization.TypeTable
 import org.jetbrains.kotlin.metadata.deserialization.supertypes
@@ -34,12 +35,15 @@ interface IncrementalCacheCommon {
    val thisWithDependentCaches: Iterable<AbstractIncrementalCache<*>>
    fun classesFqNamesBySources(files: Iterable<File>): Collection<FqName>
    fun getSubtypesOf(className: FqName): Sequence<FqName>
+    fun getSupertypesOf(className: FqName): Sequence<FqName>
    fun getSourceFileIfClass(fqName: FqName): File?
    fun markDirty(removedAndCompiledSources: Collection<File>)
    fun clearCacheForRemovedClasses(changesCollector: ChangesCollector)
    fun getComplementaryFilesRecursive(dirtyFiles: Collection<File>): Collection<File>
    fun updateComplementaryFiles(dirtyFiles: Collection<File>, expectActualTracker: ExpectActualTrackerImpl)
    fun dump(): String
+
+    fun isSealed(className: FqName): Boolean?
 }

 /**
@@ -50,6 +54,7 @@ abstract class AbstractIncrementalCache<ClassName>(
    protected val pathConverter: FileToPathConverter
 ) : BasicMapsOwner(workingDir), IncrementalCacheCommon {
    companion object {
+        private val CLASS_ATTRIBUTES = "class-attributes"
        private val SUBTYPES = "subtypes"
        private val SUPERTYPES = "supertypes"
        private val CLASS_FQ_NAME_TO_SOURCE = "class-fq-name-to-source"
@@ -71,6 +76,7 @@ abstract class AbstractIncrementalCache<ClassName>(
        result
    }

+    internal val classAttributesMap = registerMap(ClassAttributesMap(CLASS_ATTRIBUTES.storageFile))
    private val subtypesMap = registerMap(SubtypesMap(SUBTYPES.storageFile))
    private val supertypesMap = registerMap(SupertypesMap(SUPERTYPES.storageFile))
    protected val classFqNameToSourceMap = registerMap(ClassFqNameToSourceMap(CLASS_FQ_NAME_TO_SOURCE.storageFile, pathConverter))
@@ -90,6 +96,14 @@ abstract class AbstractIncrementalCache<ClassName>(
    override fun getSubtypesOf(className: FqName): Sequence<FqName> =
        subtypesMap[className].asSequence()

+    override fun getSupertypesOf(className: FqName): Sequence<FqName> {
+        return supertypesMap[className].asSequence()
+    }
+
+    override fun isSealed(className: FqName): Boolean? {
+        return classAttributesMap[className]?.isSealed
+    }
+
    override fun getSourceFileIfClass(fqName: FqName): File? =
        classFqNameToSourceMap[fqName]

@@ -118,6 +132,7 @@ abstract class AbstractIncrementalCache<ClassName>(

        supertypesMap[child] = parents
        classFqNameToSourceMap[child] = srcFile
+        classAttributesMap[child] = ICClassesAttributes(ProtoBuf.Modality.SEALED == Flags.MODALITY.get(proto.flags))
    }

    protected fun removeAllFromClassStorage(removedClasses: Collection<FqName>, changesCollector: ChangesCollector) {
@@ -152,7 +167,10 @@ abstract class AbstractIncrementalCache<ClassName>(
            }
        }

-        removedFqNames.forEach { classFqNameToSourceMap.remove(it) }
+        removedFqNames.forEach {
+            classFqNameToSourceMap.remove(it)
+            classAttributesMap.remove(it)
+        }
    }

    protected class ClassFqNameToSourceMap(
@@ -19,12 +19,14 @@ package org.jetbrains.kotlin.incremental
 import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.deserialization.Flags
 import org.jetbrains.kotlin.metadata.deserialization.NameResolver
+import org.jetbrains.kotlin.metadata.deserialization.supertypes
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.protobuf.MessageLite
 import org.jetbrains.kotlin.serialization.deserialization.getClassId

 class ChangesCollector {
    private val removedMembers = hashMapOf<FqName, MutableSet<String>>()
+    private val changedParents = hashMapOf<FqName, MutableSet<FqName>>()
    private val changedMembers = hashMapOf<FqName, MutableSet<String>>()
    private val areSubclassesAffected = hashMapOf<FqName, Boolean>()

@@ -47,6 +49,10 @@ class ChangesCollector {
            changes.add(ChangeInfo.SignatureChanged(fqName, areSubclassesAffected))
        }

+        for ((fqName, changedParents) in changedParents) {
+            changes.add(ChangeInfo.ParentsChanged(fqName, changedParents))
+        }
+
        return changes
    }

@@ -79,12 +85,12 @@ class ChangesCollector {
        }

        if (oldData == null) {
-            newData!!.collectAll(isRemoved = false, collectAllMembersForNewClass = collectAllMembersForNewClass)
+            newData!!.collectAll(isRemoved = false, isAdded = true, collectAllMembersForNewClass = collectAllMembersForNewClass)
            return
        }

        if (newData == null) {
-            oldData.collectAll(isRemoved = true)
+            oldData.collectAll(isRemoved = true, isAdded = false)
            return
        }

@@ -98,6 +104,7 @@ class ChangesCollector {
                            collectSignature(oldData, diff.areSubclassesAffected)
                        }
                        collectChangedMembers(fqName, diff.changedMembersNames)
+                        addChangedParents(fqName, diff.changedSupertypes)
                    }
                    is PackagePartProtoData -> {
                        collectSignature(oldData, areSubclassesAffected = true)
@@ -121,10 +128,11 @@ class ChangesCollector {
    private fun <T> T.getNonPrivateNames(nameResolver: NameResolver, vararg members: T.() -> List<MessageLite>): Set<String> =
            members.flatMap { this.it().filterNot { it.isPrivate }.names(nameResolver) }.toSet()

-    private fun ProtoData.collectAll(isRemoved: Boolean, collectAllMembersForNewClass: Boolean = false) =
+    //TODO remember all sealed parent classes
+    private fun ProtoData.collectAll(isRemoved: Boolean, isAdded: Boolean, collectAllMembersForNewClass: Boolean = false) =
        when (this) {
            is PackagePartProtoData -> collectAllFromPackage(isRemoved)
-            is ClassProtoData -> collectAllFromClass(isRemoved, collectAllMembersForNewClass)
+            is ClassProtoData -> collectAllFromClass(isRemoved, isAdded, collectAllMembersForNewClass)
        }

    private fun PackagePartProtoData.collectAllFromPackage(isRemoved: Boolean) {
@@ -143,7 +151,7 @@ class ChangesCollector {
        }
    }

-    private fun ClassProtoData.collectAllFromClass(isRemoved: Boolean, collectAllMembersForNewClass: Boolean = false) {
+    private fun ClassProtoData.collectAllFromClass(isRemoved: Boolean, isAdded: Boolean, collectAllMembersForNewClass: Boolean = false) {
        val classFqName = nameResolver.getClassId(proto.fqName).asSingleFqName()
        val kind = Flags.CLASS_KIND.get(proto.flags)

@@ -162,6 +170,23 @@ class ChangesCollector {

            collectSignature(classFqName, areSubclassesAffected = true)
        }
+
+        if (isRemoved || isAdded) {
+            collectChangedParents(classFqName, proto.supertypeList)
+        }
+    }
+
+    private fun addChangedParents(fqName: FqName, parents: Collection<FqName>) {
+        if (parents.isNotEmpty()) {
+            changedParents.getOrPut(fqName) { HashSet() }.addAll(parents)
+        }
+    }
+
+    private fun ClassProtoData.collectChangedParents(fqName: FqName, parents: Collection<ProtoBuf.Type>) {
+        val changedParentsFqNames = parents.map { type ->
+            nameResolver.getClassId(type.className).asSingleFqName()
+        }
+        addChangedParents(fqName, changedParentsFqNames)
    }

    private fun ClassProtoData.getNonPrivateMemberNames(): Set<String> {
@@ -145,6 +145,7 @@ open class IncrementalJvmCache(
                }
                protoMap.remove(className, changesCollector)
                classFqNameToSourceMap.remove(className.fqNameForClassNameWithoutDollars)
+                classAttributesMap.remove(className.fqNameForClassNameWithoutDollars)
                internalNameToSource.remove(className.internalName)

                // TODO NO_CHANGES? (delegates only)
@@ -568,6 +569,7 @@ sealed class ChangeInfo(val fqName: FqName) {

    class SignatureChanged(fqName: FqName, val areSubclassesAffected: Boolean) : ChangeInfo(fqName)

+    class ParentsChanged(fqName: FqName, val parentsChanged: Collection<FqName>) : ChangeInfo(fqName)

    protected open fun toStringProperties(): String = "fqName = $fqName"

@@ -136,7 +136,8 @@ fun LookupStorage.update(

 data class DirtyData(
    val dirtyLookupSymbols: Collection<LookupSymbol> = emptyList(),
-    val dirtyClassesFqNames: Collection<FqName> = emptyList()
+    val dirtyClassesFqNames: Collection<FqName> = emptyList(),
+    val dirtyClassesFqNamesForceRecompile: Collection<FqName> = emptyList()
 )

 fun ChangesCollector.getDirtyData(
@@ -146,6 +147,9 @@ fun ChangesCollector.getDirtyData(
    val dirtyLookupSymbols = HashSet<LookupSymbol>()
    val dirtyClassesFqNames = HashSet<FqName>()

+    val sealedParents = HashMap<FqName, MutableSet<FqName>>()
+    val notSealedParents = HashSet<FqName>()
+
    for (change in changes()) {
        reporter.reportVerbose { "Process $change" }

@@ -170,10 +174,35 @@ fun ChangesCollector.getDirtyData(
            }

            fqNames.mapTo(dirtyLookupSymbols) { LookupSymbol(SAM_LOOKUP_NAME.asString(), it.asString()) }
+        } else if (change is ChangeInfo.ParentsChanged) {
+            fun FqName.isSealed(): Boolean {
+                if (notSealedParents.contains(this)) return false
+                if (sealedParents.containsKey(this)) return true
+                return isSealed(this, caches).also { sealed ->
+                    if (sealed) {
+                        sealedParents[this] = HashSet()
+                    } else {
+                        notSealedParents.add(this)
+                    }
+                }
+            }
+            change.parentsChanged.forEach { parent ->
+                if (parent.isSealed()) {
+                    sealedParents.getOrPut(parent) { HashSet() }.add(change.fqName)
+                }
+            }
        }
    }

-    return DirtyData(dirtyLookupSymbols, dirtyClassesFqNames)
+    val forceRecompile = HashSet<FqName>().apply {
+        addAll(sealedParents.keys)
+        //we should recompile all inheritors with parent sealed class: add known subtypes
+        addAll(sealedParents.keys.flatMap { withSubtypes(it, caches) })
+        //we should recompile all inheritors with parent sealed class: add new subtypes
+        addAll(sealedParents.values.flatten())
+    }
+
+    return DirtyData(dirtyLookupSymbols, dirtyClassesFqNames, forceRecompile)
 }

 fun mapLookupSymbolsToFiles(
@@ -217,6 +246,11 @@ fun mapClassesFqNamesToFiles(
    return fqNameToAffectedFiles.values.flattenTo(HashSet())
 }

+fun isSealed(
+    fqName: FqName,
+    caches: Iterable<IncrementalCacheCommon>
+): Boolean = caches.any { it.isSealed(fqName) ?: false }
+
 fun withSubtypes(
    typeFqName: FqName,
    caches: Iterable<IncrementalCacheCommon>
@@ -28,12 +28,14 @@ import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.protobuf.MessageLite
 import org.jetbrains.kotlin.serialization.deserialization.ProtoEnumFlags
 import org.jetbrains.kotlin.serialization.deserialization.descriptorVisibility
+import org.jetbrains.kotlin.serialization.deserialization.getClassId
 import java.util.*

 data class Difference(
    val isClassAffected: Boolean = false,
    val areSubclassesAffected: Boolean = false,
-    val changedMembersNames: Set<String> = emptySet()
+    val changedMembersNames: Set<String> = emptySet(),
+    val changedSupertypes: Set<FqName> = emptySet()
 )

 sealed class ProtoData
@@ -187,6 +189,7 @@ class DifferenceCalculatorForClass(

        var isClassAffected = false
        var areSubclassesAffected = false
+        val changedSupertypes = HashSet<FqName>()
        val names = hashSetOf<String>()
        val classIsSealed = newProto.isSealed && oldProto.isSealed

@@ -247,12 +250,21 @@ class DifferenceCalculatorForClass(
                ProtoBufClassKind.FLAGS,
                ProtoBufClassKind.FQ_NAME,
                ProtoBufClassKind.TYPE_PARAMETER_LIST,
-                ProtoBufClassKind.SUPERTYPE_LIST,
-                ProtoBufClassKind.SUPERTYPE_ID_LIST,
                ProtoBufClassKind.JS_EXT_CLASS_ANNOTATION_LIST -> {
                    isClassAffected = true
                    areSubclassesAffected = true
                }
+
+                ProtoBufClassKind.SUPERTYPE_LIST,
+                ProtoBufClassKind.SUPERTYPE_ID_LIST -> {
+                    isClassAffected = true
+                    areSubclassesAffected = true
+
+                    val oldSupertypes = oldProto.supertypeList.map { oldNameResolver.getClassId(it.className).asSingleFqName() }
+                    val newSupertypes = newProto.supertypeList.map { newNameResolver.getClassId(it.className).asSingleFqName() }
+                    val changed = (oldSupertypes union newSupertypes) subtract (oldSupertypes intersect newSupertypes)
+                    changedSupertypes.addAll(changed)
+                }
                ProtoBufClassKind.JVM_EXT_CLASS_MODULE_NAME,
                ProtoBufClassKind.JS_EXT_CLASS_CONTAINING_FILE_ID -> {
                    // TODO
@@ -281,7 +293,7 @@ class DifferenceCalculatorForClass(
            }
        }

-        return Difference(isClassAffected, areSubclassesAffected, names)
+        return Difference(isClassAffected, areSubclassesAffected, names, changedSupertypes)
    }
 }

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental.storage
+
+import com.intellij.util.io.DataExternalizer
+import org.jetbrains.kotlin.name.FqName
+import java.io.DataInput
+import java.io.DataOutput
+import java.io.File
+
+internal data class ICClassesAttributes(val isSealed: Boolean)
+
+internal object ICClassesAttributesExternalizer : DataExternalizer<ICClassesAttributes> {
+    override fun read(input: DataInput): ICClassesAttributes {
+        return ICClassesAttributes(input.readBoolean())
+    }
+
+    override fun save(output: DataOutput, value: ICClassesAttributes) {
+        output.writeBoolean(value.isSealed)
+    }
+}
+
+internal open class ClassAttributesMap(
+    storageFile: File
+) : BasicStringMap<ICClassesAttributes>(storageFile, ICClassesAttributesExternalizer) {
+    override fun dumpValue(value: ICClassesAttributes): String = value.toString()
+
+    operator fun set(key: FqName, value: ICClassesAttributes) {
+        storage[key.asString()] = value
+    }
+
+    operator fun get(key: FqName): ICClassesAttributes? = storage[key.asString()]
+
+    fun remove(key: FqName) {
+        storage.remove(key.asString())
+    }
+}
@@ -310,9 +310,10 @@ abstract class IncrementalCompilerRunner<
            }
            if (compilationMode is CompilationMode.Rebuild) break

-            val (dirtyLookupSymbols, dirtyClassFqNames) = changesCollector.getDirtyData(listOf(caches.platformCache), reporter)
+            val (dirtyLookupSymbols, dirtyClassFqNames, forceRecompile) = changesCollector.getDirtyData(listOf(caches.platformCache), reporter)
            val compiledInThisIterationSet = sourcesToCompile.toHashSet()

+            val forceToRecompileFiles = mapClassesFqNamesToFiles(listOf(caches.platformCache), forceRecompile, reporter)
            with(dirtySources) {
                clear()
                addAll(mapLookupSymbolsToFiles(caches.lookupCache, dirtyLookupSymbols, reporter, excludes = compiledInThisIterationSet))
@@ -324,6 +325,9 @@ abstract class IncrementalCompilerRunner<
                        excludes = compiledInThisIterationSet
                    )
                )
+                if (!compiledInThisIterationSet.containsAll(forceToRecompileFiles)) {
+                    addAll(forceToRecompileFiles)
+                }
            }

            buildDirtyLookupSymbols.addAll(dirtyLookupSymbols)
@@ -125,7 +125,7 @@ class IncrementalJsCompilerRunner(
        val removedClassesChanges = getRemovedClassesChanges(caches, changedFiles)
        dirtyFiles.addByDirtySymbols(removedClassesChanges.dirtyLookupSymbols)
        dirtyFiles.addByDirtyClasses(removedClassesChanges.dirtyClassesFqNames)
-
+        dirtyFiles.addByDirtyClasses(removedClassesChanges.dirtyClassesFqNamesForceRecompile)
        return CompilationMode.Incremental(dirtyFiles)
    }

@@ -227,7 +227,7 @@ class IncrementalJvmCompilerRunner(
        dirtyFiles.addByDirtySymbols(androidLayoutChanges)
        dirtyFiles.addByDirtySymbols(removedClassesChanges.dirtyLookupSymbols)
        dirtyFiles.addByDirtyClasses(removedClassesChanges.dirtyClassesFqNames)
-
+        dirtyFiles.addByDirtyClasses(removedClassesChanges.dirtyClassesFqNamesForceRecompile)
        return CompilationMode.Incremental(dirtyFiles)
    }

@@ -90,6 +90,8 @@ abstract class AbstractProtoComparisonTest<PROTO_DATA> : TestWithWorkingDir() {
                when (it) {
                    is ChangeInfo.SignatureChanged -> "CLASS_SIGNATURE"
                    is ChangeInfo.MembersChanged -> "MEMBERS\n    ${it.names.sorted()}"
+                    is ChangeInfo.ParentsChanged -> "PARENTS\n    ${it.parentsChanged.map { it.asString()}.sorted()}"
+
                }
            }.sorted()

@@ -250,7 +250,7 @@ class KotlinBuilder : ModuleLevelBuilder(BuilderCategory.SOURCE_PROCESSOR) {
        removedClasses.forEach { changesCollector.collectSignature(FqName(it), areSubclassesAffected = true) }
        val affectedByRemovedClasses = changesCollector.getDirtyFiles(incrementalCaches.values, kotlinContext.lookupStorageManager)

-        fsOperations.markFilesForCurrentRound(affectedByRemovedClasses)
+        fsOperations.markFilesForCurrentRound(affectedByRemovedClasses.dirtyFiles + affectedByRemovedClasses.forceRecompileTogether)
    }

    override fun chunkBuildFinished(context: CompileContext, chunk: ModuleChunk) {
@@ -695,21 +695,36 @@ private fun ChangesCollector.processChangesUsingLookups(
    reporter.reportVerbose { "Start processing changes" }

    val dirtyFiles = getDirtyFiles(allCaches, lookupStorageManager)
-    fsOperations.markInChunkOrDependents(dirtyFiles.asIterable(), excludeFiles = compiledFiles)
+    // if list of inheritors of sealed class has changed it should be recompiled with all the inheritors
+    // Here we have a small optimization. Do not recompile the bunch if ALL these files were recompiled during the previous round.
+    val excludeFiles = if (compiledFiles.containsAll(dirtyFiles.forceRecompileTogether))
+        compiledFiles
+    else
+        compiledFiles.minus(dirtyFiles.forceRecompileTogether)
+    fsOperations.markInChunkOrDependents(
+        (dirtyFiles.dirtyFiles + dirtyFiles.forceRecompileTogether).asIterable(),
+        excludeFiles = excludeFiles
+    )

    reporter.reportVerbose { "End of processing changes" }
 }

+data class FilesToRecompile(val dirtyFiles: Set<File>, val forceRecompileTogether: Set<File>)
+
 private fun ChangesCollector.getDirtyFiles(
    caches: Iterable<IncrementalCacheCommon>,
    lookupStorageManager: JpsLookupStorageManager
-): Set<File> {
+): FilesToRecompile {
    val reporter = JpsICReporter()
-    val (dirtyLookupSymbols, dirtyClassFqNames) = getDirtyData(caches, reporter)
+    val (dirtyLookupSymbols, dirtyClassFqNames, forceRecompile) = getDirtyData(caches, reporter)
    val dirtyFilesFromLookups = lookupStorageManager.withLookupStorage {
        mapLookupSymbolsToFiles(it, dirtyLookupSymbols, reporter)
    }
-    return dirtyFilesFromLookups + mapClassesFqNamesToFiles(caches, dirtyClassFqNames, reporter)
+    return FilesToRecompile(
+        dirtyFilesFromLookups + mapClassesFqNamesToFiles(caches, dirtyClassFqNames, reporter),
+        mapClassesFqNamesToFiles(caches, forceRecompile, reporter)
+    )
+
 }

 private fun getLookupTracker(project: JpsProject, representativeTarget: KotlinModuleBuildTarget<*>): LookupTracker {