Save class hierarchy to incremental caches

Author: Alexey Tsvetkov <Alexey.Tsvetkov@jetbrains.com> Original commit: c567376e35
2015-11-06 14:47:14 +03:00
parent 7d8d421155
commit 6e08ea4524
12 changed files with 169 additions and 6 deletions
@@ -26,7 +26,7 @@ import org.jetbrains.kotlin.load.java.JvmAbi
 import java.io.File

 private val NORMAL_VERSION = 7
-private val EXPERIMENTAL_VERSION = 1
+private val EXPERIMENTAL_VERSION = 2
 private val DATA_CONTAINER_VERSION = 1

 private val NORMAL_VERSION_FILE_NAME = "format-version.txt"
@@ -45,6 +45,8 @@ import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 import org.jetbrains.kotlin.serialization.ProtoBuf
 import org.jetbrains.kotlin.serialization.deserialization.NameResolver
+import org.jetbrains.kotlin.serialization.deserialization.TypeTable
+import org.jetbrains.kotlin.serialization.deserialization.supertypes
 import org.jetbrains.kotlin.serialization.jvm.BitEncoding
 import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
 import org.jetbrains.org.objectweb.asm.*
@@ -69,12 +71,20 @@ public class IncrementalCacheImpl(
        val DIRTY_OUTPUT_CLASSES = "dirty-output-classes"
        val DIRTY_INLINE_FUNCTIONS = "dirty-inline-functions"
        val INLINED_TO = "inlined-to"
+        val SUBTYPES = "subtypes"
+        val SUPERTYPES = "supertypes"

        private val MODULE_MAPPING_FILE_NAME = "." + ModuleMapping.MAPPING_FILE_EXT
    }

    private val baseDir = File(paths.getTargetDataRoot(target), KOTLIN_CACHE_DIRECTORY_NAME)
    private val cacheVersionProvider = CacheVersionProvider(paths)
+    private val experimentalMaps = arrayListOf<BasicMap<*, *>>()
+
+    private fun <K, V, M : BasicMap<K, V>> registerExperimentalMap(map: M): M {
+        experimentalMaps.add(map)
+        return registerMap(map)
+    }

    private val String.storageFile: File
        get() = File(baseDir, this + "." + CACHE_EXTENSION)
@@ -89,10 +99,15 @@ public class IncrementalCacheImpl(
    private val dirtyOutputClassesMap = registerMap(DirtyOutputClassesMap(DIRTY_OUTPUT_CLASSES.storageFile))
    private val dirtyInlineFunctionsMap = registerMap(DirtyInlineFunctionsMap(DIRTY_INLINE_FUNCTIONS.storageFile))
    private val inlinedTo = registerMap(InlineFunctionsFilesMap(INLINED_TO.storageFile))
+    private val subtypesMap = registerExperimentalMap(SubtypesMap(SUBTYPES.storageFile))
+    private val supertypesMap = registerExperimentalMap(SupertypesMap(SUPERTYPES.storageFile))

    private val dependents = arrayListOf<IncrementalCacheImpl>()
    private val outputDir = requireNotNull(target.outputDir) { "Target is expected to have output directory: $target" }

+    private val dependentsWithThis: Iterable<IncrementalCacheImpl>
+            get() = dependents + this
+
    override fun registerInline(fromPath: String, jvmSignature: String, toPath: String) {
        inlinedTo.add(fromPath, jvmSignature, toPath)
    }
@@ -118,13 +133,10 @@ public class IncrementalCacheImpl(
        for ((className, functions) in dirtyInlineFunctionsMap.getEntries()) {
            val classFilePath = getClassFilePath(className.internalName)

-            fun addFilesAffectedByChangedInlineFuns(cache: IncrementalCacheImpl) {
+            for (cache in dependentsWithThis) {
                val targetFiles = functions.flatMap { cache.inlinedTo[classFilePath, it] }
                result.addAll(targetFiles)
            }
-
-            addFilesAffectedByChangedInlineFuns(this)
-            dependents.forEach(::addFilesAffectedByChangedInlineFuns)
        }

        return result.map { File(it) }
@@ -149,7 +161,7 @@ public class IncrementalCacheImpl(
    public fun saveFileToCache(generatedClass: GeneratedJvmClass): CompilationResult {
        val sourceFiles: Collection<File> = generatedClass.sourceFiles
        val kotlinClass: LocalFileKotlinClass = generatedClass.outputClass
-        val className = JvmClassName.byClassId(kotlinClass.classId)
+        val className = kotlinClass.className

        dirtyOutputClassesMap.notDirty(className.internalName)
        sourceFiles.forEach {
@@ -185,6 +197,8 @@ public class IncrementalCacheImpl(
                inlineFunctionsMap.process(kotlinClass, isPackage = true)
            }
            header.isCompatibleClassKind() && !header.isLocalClass -> {
+                addToClassStorage(kotlinClass)
+
                protoMap.process(kotlinClass, isPackage = false) +
                constantsMap.process(kotlinClass) +
                inlineFunctionsMap.process(kotlinClass, isPackage = false)
@@ -268,6 +282,9 @@ public class IncrementalCacheImpl(
            constantsMap.remove(it)
            inlineFunctionsMap.remove(it)
        }
+
+        removeAllFromClassStorage(dirtyClasses)
+
        dirtyOutputClassesMap.clean()
        return changesInfo
    }
@@ -316,6 +333,7 @@ public class IncrementalCacheImpl(

    public fun cleanExperimental() {
        cacheVersionProvider.experimentalVersion(target).clean()
+        experimentalMaps.forEach { it.clean() }
    }

    private inner class ProtoMap(storageFile: File) : BasicStringMap<ProtoMapValue>(storageFile, ProtoMapValueExternalizer) {
@@ -576,6 +594,46 @@ public class IncrementalCacheImpl(
        }
    }

+    private fun addToClassStorage(kotlinClass: LocalFileKotlinClass) {
+        if (!IncrementalCompilation.isExperimental()) return
+
+        val classData = JvmProtoBufUtil.readClassDataFrom(kotlinClass.classHeader.annotationData!!, kotlinClass.classHeader.strings!!)
+        val supertypes = classData.classProto.supertypes(TypeTable(classData.classProto.typeTable))
+        val parents = supertypes.map { classData.nameResolver.getClassId(it.className).asSingleFqName() }
+                                .filter { it.asString() != "kotlin.Any" }
+        val child = kotlinClass.classId.asSingleFqName()
+
+        parents.forEach { subtypesMap.add(it, child) }
+        supertypesMap[child] = parents
+    }
+
+    private fun removeAllFromClassStorage(removedClasses: Collection<JvmClassName>) {
+        if (!IncrementalCompilation.isExperimental() || removedClasses.isEmpty()) return
+
+        val removedFqNames = removedClasses.map { it.fqNameForClassNameWithoutDollars }.toSet()
+
+        for (cache in dependentsWithThis) {
+            val parentsFqNames = hashSetOf<FqName>()
+            val childrenFqNames = hashSetOf<FqName>()
+
+            for (removedFqName in removedFqNames) {
+                parentsFqNames.addAll(cache.supertypesMap[removedFqName])
+                childrenFqNames.addAll(cache.subtypesMap[removedFqName])
+
+                cache.supertypesMap.remove(removedFqName)
+                cache.subtypesMap.remove(removedFqName)
+            }
+
+            for (child in childrenFqNames) {
+                cache.supertypesMap.removeValues(child, removedFqNames)
+            }
+
+            for (parent in parentsFqNames) {
+                cache.subtypesMap.removeValues(parent, removedFqNames)
+            }
+        }
+    }
+
    private inner class DirtyOutputClassesMap(storageFile: File) : BasicStringMap<Boolean>(storageFile, BooleanDataDescriptor.INSTANCE) {
        public fun markDirty(className: String) {
            storage[className] = true
@@ -60,7 +60,10 @@ internal abstract class BasicMap<K : Comparable<K>, V>(
        }.toString()
    }

+    @TestOnly
    protected abstract fun dumpKey(key: K): String
+
+    @TestOnly
    protected abstract fun dumpValue(value: V): String
 }

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.jps.incremental.storage
+
+import org.jetbrains.kotlin.jps.incremental.dumpCollection
+import org.jetbrains.kotlin.name.FqName
+import java.io.File
+
+internal open class ClassOneToManyMap(
+        storageFile: File
+) : BasicStringMap<Collection<String>>(storageFile, StringCollectionExternalizer) {
+    override fun dumpValue(value: Collection<String>): String = value.dumpCollection()
+
+    fun add(key: FqName, value: FqName) {
+        storage.append(key.asString()) { out -> out.writeUTF(value.asString()) }
+    }
+
+    operator fun get(key: FqName): Collection<FqName> =
+            storage[key.asString()]?.map(::FqName) ?: setOf()
+
+    operator fun set(key: FqName, values: Collection<FqName>) {
+        if (values.isEmpty()) {
+            remove(key)
+            return
+        }
+
+        storage[key.asString()] = values.map(FqName::asString)
+    }
+
+    fun remove(key: FqName) {
+        storage.remove(key.asString())
+    }
+
+    fun removeValues(key: FqName, removed: Set<FqName>) {
+        val notRemoved = this[key].filter { it !in removed }
+        this[key] = notRemoved
+    }
+}
+
+internal class SubtypesMap(storageFile: File) : ClassOneToManyMap(storageFile)
+internal class SupertypesMap(storageFile: File) : ClassOneToManyMap(storageFile)
@@ -41,6 +41,12 @@ public class ExperimentalIncrementalLazyCachesTestGenerated extends AbstractExpe
        doTest(fileName);
    }

+    @TestMetadata("classInheritance")
+    public void testClassInheritance() throws Exception {
+        String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/");
+        doTest(fileName);
+    }
+
    @TestMetadata("constant")
    public void testConstant() throws Exception {
        String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/lazyKotlinCaches/constant/");
@@ -41,6 +41,12 @@ public class IncrementalLazyCachesTestGenerated extends AbstractIncrementalLazyC
        doTest(fileName);
    }

+    @TestMetadata("classInheritance")
+    public void testClassInheritance() throws Exception {
+        String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/");
+        doTest(fileName);
+    }
+
    @TestMetadata("constant")
    public void testConstant() throws Exception {
        String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/lazyKotlinCaches/constant/");
@@ -16,6 +16,8 @@ Module 'module2' production
    package-parts.tab
    proto.tab
    source-to-classes.tab
+    subtypes.tab
+    supertypes.tab
 Module 'module2' tests
 Module 'module3' production
    experimental-format-version.txt
@@ -0,0 +1,8 @@
+Cleaning output files:
+out/production/module/A.class
+out/production/module/B.class
+out/production/module/C.class
+End of files
+Compiling files:
+src/main.kt
+End of files
@@ -0,0 +1,6 @@
+kotlin-data-container
+Module 'module' production
+    format-version.txt
+    proto.tab
+    source-to-classes.tab
+Module 'module' tests
@@ -0,0 +1,14 @@
+kotlin-data-container
+    data-container-format-version.txt
+    counters.tab
+    file-to-id.tab
+    id-to-file.tab
+    lookups.tab
+Module 'module' production
+    experimental-format-version.txt
+    format-version.txt
+    proto.tab
+    source-to-classes.tab
+    subtypes.tab
+    supertypes.tab
+Module 'module' tests
@@ -0,0 +1,5 @@
+open class A
+
+open class B : A()
+
+class C : B()