Save class hierarchy to incremental caches

Author:    Alexey Tsvetkov <Alexey.Tsvetkov@jetbrains.com>

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/CacheVersion.kt

@@ -26,7 +26,7 @@ import org.jetbrains.kotlin.load.java.JvmAbi
 import java.io.File
 
 private val NORMAL_VERSION = 7
-private val EXPERIMENTAL_VERSION = 1
+private val EXPERIMENTAL_VERSION = 2
 private val DATA_CONTAINER_VERSION = 1
 
 private val NORMAL_VERSION_FILE_NAME = "format-version.txt"

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt

@@ -45,6 +45,8 @@ import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.deserialization.NameResolver
+import org.jetbrains.kotlin.serialization.deserialization.TypeTable
+import org.jetbrains.kotlin.serialization.deserialization.supertypes
 import org.jetbrains.kotlin.serialization.jvm.BitEncoding
 import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
 import org.jetbrains.org.objectweb.asm.*
@@ -69,12 +71,20 @@ public class IncrementalCacheImpl(
         val DIRTY_OUTPUT_CLASSES = "dirty-output-classes"
         val DIRTY_INLINE_FUNCTIONS = "dirty-inline-functions"
         val INLINED_TO = "inlined-to"
+        val SUBTYPES = "subtypes"
+        val SUPERTYPES = "supertypes"
 
         private val MODULE_MAPPING_FILE_NAME = "." + ModuleMapping.MAPPING_FILE_EXT
     }
 
     private val baseDir = File(paths.getTargetDataRoot(target), KOTLIN_CACHE_DIRECTORY_NAME)
     private val cacheVersionProvider = CacheVersionProvider(paths)
+    private val experimentalMaps = arrayListOf<BasicMap<*, *>>()
+
+    private fun <K, V, M : BasicMap<K, V>> registerExperimentalMap(map: M): M {
+        experimentalMaps.add(map)
+        return registerMap(map)
+    }
 
     private val String.storageFile: File
         get() = File(baseDir, this + "." + CACHE_EXTENSION)
@@ -89,10 +99,15 @@ public class IncrementalCacheImpl(
     private val dirtyOutputClassesMap = registerMap(DirtyOutputClassesMap(DIRTY_OUTPUT_CLASSES.storageFile))
     private val dirtyInlineFunctionsMap = registerMap(DirtyInlineFunctionsMap(DIRTY_INLINE_FUNCTIONS.storageFile))
     private val inlinedTo = registerMap(InlineFunctionsFilesMap(INLINED_TO.storageFile))
+    private val subtypesMap = registerExperimentalMap(SubtypesMap(SUBTYPES.storageFile))
+    private val supertypesMap = registerExperimentalMap(SupertypesMap(SUPERTYPES.storageFile))
 
     private val dependents = arrayListOf<IncrementalCacheImpl>()
     private val outputDir = requireNotNull(target.outputDir) { "Target is expected to have output directory: $target" }
 
+    private val dependentsWithThis: Iterable<IncrementalCacheImpl>
+            get() = dependents + this
+
     override fun registerInline(fromPath: String, jvmSignature: String, toPath: String) {
         inlinedTo.add(fromPath, jvmSignature, toPath)
     }
@@ -118,13 +133,10 @@ public class IncrementalCacheImpl(
         for ((className, functions) in dirtyInlineFunctionsMap.getEntries()) {
             val classFilePath = getClassFilePath(className.internalName)
 
-            fun addFilesAffectedByChangedInlineFuns(cache: IncrementalCacheImpl) {
+            for (cache in dependentsWithThis) {
                 val targetFiles = functions.flatMap { cache.inlinedTo[classFilePath, it] }
                 result.addAll(targetFiles)
             }
-
-            addFilesAffectedByChangedInlineFuns(this)
-            dependents.forEach(::addFilesAffectedByChangedInlineFuns)
         }
 
         return result.map { File(it) }
@@ -149,7 +161,7 @@ public class IncrementalCacheImpl(
     public fun saveFileToCache(generatedClass: GeneratedJvmClass): CompilationResult {
         val sourceFiles: Collection<File> = generatedClass.sourceFiles
         val kotlinClass: LocalFileKotlinClass = generatedClass.outputClass
-        val className = JvmClassName.byClassId(kotlinClass.classId)
+        val className = kotlinClass.className
 
         dirtyOutputClassesMap.notDirty(className.internalName)
         sourceFiles.forEach {
@@ -185,6 +197,8 @@ public class IncrementalCacheImpl(
                 inlineFunctionsMap.process(kotlinClass, isPackage = true)
             }
             header.isCompatibleClassKind() && !header.isLocalClass -> {
+                addToClassStorage(kotlinClass)
+
                 protoMap.process(kotlinClass, isPackage = false) +
                 constantsMap.process(kotlinClass) +
                 inlineFunctionsMap.process(kotlinClass, isPackage = false)
@@ -268,6 +282,9 @@ public class IncrementalCacheImpl(
             constantsMap.remove(it)
             inlineFunctionsMap.remove(it)
         }
+
+        removeAllFromClassStorage(dirtyClasses)
+
         dirtyOutputClassesMap.clean()
         return changesInfo
     }
@@ -316,6 +333,7 @@ public class IncrementalCacheImpl(
 
     public fun cleanExperimental() {
         cacheVersionProvider.experimentalVersion(target).clean()
+        experimentalMaps.forEach { it.clean() }
     }
 
     private inner class ProtoMap(storageFile: File) : BasicStringMap<ProtoMapValue>(storageFile, ProtoMapValueExternalizer) {
@@ -576,6 +594,46 @@ public class IncrementalCacheImpl(
         }
     }
 
+    private fun addToClassStorage(kotlinClass: LocalFileKotlinClass) {
+        if (!IncrementalCompilation.isExperimental()) return
+
+        val classData = JvmProtoBufUtil.readClassDataFrom(kotlinClass.classHeader.annotationData!!, kotlinClass.classHeader.strings!!)
+        val supertypes = classData.classProto.supertypes(TypeTable(classData.classProto.typeTable))
+        val parents = supertypes.map { classData.nameResolver.getClassId(it.className).asSingleFqName() }
+                                .filter { it.asString() != "kotlin.Any" }
+        val child = kotlinClass.classId.asSingleFqName()
+
+        parents.forEach { subtypesMap.add(it, child) }
+        supertypesMap[child] = parents
+    }
+
+    private fun removeAllFromClassStorage(removedClasses: Collection<JvmClassName>) {
+        if (!IncrementalCompilation.isExperimental() || removedClasses.isEmpty()) return
+
+        val removedFqNames = removedClasses.map { it.fqNameForClassNameWithoutDollars }.toSet()
+
+        for (cache in dependentsWithThis) {
+            val parentsFqNames = hashSetOf<FqName>()
+            val childrenFqNames = hashSetOf<FqName>()
+
+            for (removedFqName in removedFqNames) {
+                parentsFqNames.addAll(cache.supertypesMap[removedFqName])
+                childrenFqNames.addAll(cache.subtypesMap[removedFqName])
+
+                cache.supertypesMap.remove(removedFqName)
+                cache.subtypesMap.remove(removedFqName)
+            }
+
+            for (child in childrenFqNames) {
+                cache.supertypesMap.removeValues(child, removedFqNames)
+            }
+
+            for (parent in parentsFqNames) {
+                cache.subtypesMap.removeValues(parent, removedFqNames)
+            }
+        }
+    }
+
     private inner class DirtyOutputClassesMap(storageFile: File) : BasicStringMap<Boolean>(storageFile, BooleanDataDescriptor.INSTANCE) {
         public fun markDirty(className: String) {
             storage[className] = true

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/BasicMap.kt

@@ -60,7 +60,10 @@ internal abstract class BasicMap<K : Comparable<K>, V>(
         }.toString()
     }
 
+    @TestOnly
     protected abstract fun dumpKey(key: K): String
+
+    @TestOnly
     protected abstract fun dumpValue(value: V): String
 }
 

jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/ClassOneToManyMap.kt

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.jps.incremental.storage
+
+import org.jetbrains.kotlin.jps.incremental.dumpCollection
+import org.jetbrains.kotlin.name.FqName
+import java.io.File
+
+internal open class ClassOneToManyMap(
+        storageFile: File
+) : BasicStringMap<Collection<String>>(storageFile, StringCollectionExternalizer) {
+    override fun dumpValue(value: Collection<String>): String = value.dumpCollection()
+
+    fun add(key: FqName, value: FqName) {
+        storage.append(key.asString()) { out -> out.writeUTF(value.asString()) }
+    }
+
+    operator fun get(key: FqName): Collection<FqName> =
+            storage[key.asString()]?.map(::FqName) ?: setOf()
+
+    operator fun set(key: FqName, values: Collection<FqName>) {
+        if (values.isEmpty()) {
+            remove(key)
+            return
+        }
+
+        storage[key.asString()] = values.map(FqName::asString)
+    }
+
+    fun remove(key: FqName) {
+        storage.remove(key.asString())
+    }
+
+    fun removeValues(key: FqName, removed: Set<FqName>) {
+        val notRemoved = this[key].filter { it !in removed }
+        this[key] = notRemoved
+    }
+}
+
+internal class SubtypesMap(storageFile: File) : ClassOneToManyMap(storageFile)
+internal class SupertypesMap(storageFile: File) : ClassOneToManyMap(storageFile)

jps-plugin/test/org/jetbrains/kotlin/jps/build/ExperimentalIncrementalLazyCachesTestGenerated.java

@@ -41,6 +41,12 @@ public class ExperimentalIncrementalLazyCachesTestGenerated extends AbstractExpe
         doTest(fileName);
     }
 
+    @TestMetadata("classInheritance")
+    public void testClassInheritance() throws Exception {
+        String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/");
+        doTest(fileName);
+    }
+
     @TestMetadata("constant")
     public void testConstant() throws Exception {
         String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/lazyKotlinCaches/constant/");

jps-plugin/test/org/jetbrains/kotlin/jps/build/IncrementalLazyCachesTestGenerated.java

@@ -41,6 +41,12 @@ public class IncrementalLazyCachesTestGenerated extends AbstractIncrementalLazyC
         doTest(fileName);
     }
 
+    @TestMetadata("classInheritance")
+    public void testClassInheritance() throws Exception {
+        String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/");
+        doTest(fileName);
+    }
+
     @TestMetadata("constant")
     public void testConstant() throws Exception {
         String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/lazyKotlinCaches/constant/");

jps-plugin/testData/incremental/changeIncrementalOption/experimentalOn/expected-kotlin-caches.txt

@@ -16,6 +16,8 @@ Module 'module2' production
     package-parts.tab
     proto.tab
     source-to-classes.tab
+    subtypes.tab
+    supertypes.tab
 Module 'module2' tests
 Module 'module3' production
     experimental-format-version.txt

jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/build.log

@@ -0,0 +1,8 @@
+Cleaning output files:
+out/production/module/A.class
+out/production/module/B.class
+out/production/module/C.class
+End of files
+Compiling files:
+src/main.kt
+End of files

jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/expected-kotlin-caches.txt

@@ -0,0 +1,6 @@
+kotlin-data-container
+Module 'module' production
+    format-version.txt
+    proto.tab
+    source-to-classes.tab
+Module 'module' tests

jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/experimental-expected-kotlin-caches.txt

@@ -0,0 +1,14 @@
+kotlin-data-container
+    data-container-format-version.txt
+    counters.tab
+    file-to-id.tab
+    id-to-file.tab
+    lookups.tab
+Module 'module' production
+    experimental-format-version.txt
+    format-version.txt
+    proto.tab
+    source-to-classes.tab
+    subtypes.tab
+    supertypes.tab
+Module 'module' tests

jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/main.kt

@@ -0,0 +1,5 @@
+open class A
+
+open class B : A()
+
+class C : B()