From c567376e355b4a0da038d065482f481a415f01dd Mon Sep 17 00:00:00 2001 From: Alexey Tsvetkov Date: Fri, 6 Nov 2015 14:47:14 +0300 Subject: [PATCH] Save class hierarchy to incremental caches Author: Alexey Tsvetkov --- .../kotlin/jps/incremental/CacheVersion.kt | 2 +- .../jps/incremental/IncrementalCacheImpl.kt | 68 +++++++++++++++++-- .../jps/incremental/storage/BasicMap.kt | 3 + .../incremental/storage/ClassOneToManyMap.kt | 55 +++++++++++++++ ...talIncrementalLazyCachesTestGenerated.java | 6 ++ .../IncrementalLazyCachesTestGenerated.java | 6 ++ .../experimentalOn/expected-kotlin-caches.txt | 2 + .../classInheritance/build.log | 8 +++ .../expected-kotlin-caches.txt | 6 ++ .../experimental-expected-kotlin-caches.txt | 14 ++++ .../lazyKotlinCaches/classInheritance/main.kt | 5 ++ .../classInheritance/main.kt.touch | 0 12 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/ClassOneToManyMap.kt create mode 100644 jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/build.log create mode 100644 jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/expected-kotlin-caches.txt create mode 100644 jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/experimental-expected-kotlin-caches.txt create mode 100644 jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/main.kt create mode 100644 jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/main.kt.touch diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/CacheVersion.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/CacheVersion.kt index e4a7107146f..8759a94a67a 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/CacheVersion.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/CacheVersion.kt @@ -26,7 +26,7 @@ import org.jetbrains.kotlin.load.java.JvmAbi import java.io.File private val NORMAL_VERSION = 7 -private val EXPERIMENTAL_VERSION = 1 +private val EXPERIMENTAL_VERSION = 2 private val DATA_CONTAINER_VERSION = 1 private val NORMAL_VERSION_FILE_NAME = "format-version.txt" diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt index 24a32ced6dd..7f7493ff775 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/IncrementalCacheImpl.kt @@ -45,6 +45,8 @@ import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.resolve.jvm.JvmClassName import org.jetbrains.kotlin.serialization.ProtoBuf import org.jetbrains.kotlin.serialization.deserialization.NameResolver +import org.jetbrains.kotlin.serialization.deserialization.TypeTable +import org.jetbrains.kotlin.serialization.deserialization.supertypes import org.jetbrains.kotlin.serialization.jvm.BitEncoding import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil import org.jetbrains.org.objectweb.asm.* @@ -69,12 +71,20 @@ public class IncrementalCacheImpl( val DIRTY_OUTPUT_CLASSES = "dirty-output-classes" val DIRTY_INLINE_FUNCTIONS = "dirty-inline-functions" val INLINED_TO = "inlined-to" + val SUBTYPES = "subtypes" + val SUPERTYPES = "supertypes" private val MODULE_MAPPING_FILE_NAME = "." + ModuleMapping.MAPPING_FILE_EXT } private val baseDir = File(paths.getTargetDataRoot(target), KOTLIN_CACHE_DIRECTORY_NAME) private val cacheVersionProvider = CacheVersionProvider(paths) + private val experimentalMaps = arrayListOf>() + + private fun > registerExperimentalMap(map: M): M { + experimentalMaps.add(map) + return registerMap(map) + } private val String.storageFile: File get() = File(baseDir, this + "." + CACHE_EXTENSION) @@ -89,10 +99,15 @@ public class IncrementalCacheImpl( private val dirtyOutputClassesMap = registerMap(DirtyOutputClassesMap(DIRTY_OUTPUT_CLASSES.storageFile)) private val dirtyInlineFunctionsMap = registerMap(DirtyInlineFunctionsMap(DIRTY_INLINE_FUNCTIONS.storageFile)) private val inlinedTo = registerMap(InlineFunctionsFilesMap(INLINED_TO.storageFile)) + private val subtypesMap = registerExperimentalMap(SubtypesMap(SUBTYPES.storageFile)) + private val supertypesMap = registerExperimentalMap(SupertypesMap(SUPERTYPES.storageFile)) private val dependents = arrayListOf() private val outputDir = requireNotNull(target.outputDir) { "Target is expected to have output directory: $target" } + private val dependentsWithThis: Iterable + get() = dependents + this + override fun registerInline(fromPath: String, jvmSignature: String, toPath: String) { inlinedTo.add(fromPath, jvmSignature, toPath) } @@ -118,13 +133,10 @@ public class IncrementalCacheImpl( for ((className, functions) in dirtyInlineFunctionsMap.getEntries()) { val classFilePath = getClassFilePath(className.internalName) - fun addFilesAffectedByChangedInlineFuns(cache: IncrementalCacheImpl) { + for (cache in dependentsWithThis) { val targetFiles = functions.flatMap { cache.inlinedTo[classFilePath, it] } result.addAll(targetFiles) } - - addFilesAffectedByChangedInlineFuns(this) - dependents.forEach(::addFilesAffectedByChangedInlineFuns) } return result.map { File(it) } @@ -149,7 +161,7 @@ public class IncrementalCacheImpl( public fun saveFileToCache(generatedClass: GeneratedJvmClass): CompilationResult { val sourceFiles: Collection = generatedClass.sourceFiles val kotlinClass: LocalFileKotlinClass = generatedClass.outputClass - val className = JvmClassName.byClassId(kotlinClass.classId) + val className = kotlinClass.className dirtyOutputClassesMap.notDirty(className.internalName) sourceFiles.forEach { @@ -185,6 +197,8 @@ public class IncrementalCacheImpl( inlineFunctionsMap.process(kotlinClass, isPackage = true) } header.isCompatibleClassKind() && !header.isLocalClass -> { + addToClassStorage(kotlinClass) + protoMap.process(kotlinClass, isPackage = false) + constantsMap.process(kotlinClass) + inlineFunctionsMap.process(kotlinClass, isPackage = false) @@ -268,6 +282,9 @@ public class IncrementalCacheImpl( constantsMap.remove(it) inlineFunctionsMap.remove(it) } + + removeAllFromClassStorage(dirtyClasses) + dirtyOutputClassesMap.clean() return changesInfo } @@ -316,6 +333,7 @@ public class IncrementalCacheImpl( public fun cleanExperimental() { cacheVersionProvider.experimentalVersion(target).clean() + experimentalMaps.forEach { it.clean() } } private inner class ProtoMap(storageFile: File) : BasicStringMap(storageFile, ProtoMapValueExternalizer) { @@ -576,6 +594,46 @@ public class IncrementalCacheImpl( } } + private fun addToClassStorage(kotlinClass: LocalFileKotlinClass) { + if (!IncrementalCompilation.isExperimental()) return + + val classData = JvmProtoBufUtil.readClassDataFrom(kotlinClass.classHeader.annotationData!!, kotlinClass.classHeader.strings!!) + val supertypes = classData.classProto.supertypes(TypeTable(classData.classProto.typeTable)) + val parents = supertypes.map { classData.nameResolver.getClassId(it.className).asSingleFqName() } + .filter { it.asString() != "kotlin.Any" } + val child = kotlinClass.classId.asSingleFqName() + + parents.forEach { subtypesMap.add(it, child) } + supertypesMap[child] = parents + } + + private fun removeAllFromClassStorage(removedClasses: Collection) { + if (!IncrementalCompilation.isExperimental() || removedClasses.isEmpty()) return + + val removedFqNames = removedClasses.map { it.fqNameForClassNameWithoutDollars }.toSet() + + for (cache in dependentsWithThis) { + val parentsFqNames = hashSetOf() + val childrenFqNames = hashSetOf() + + for (removedFqName in removedFqNames) { + parentsFqNames.addAll(cache.supertypesMap[removedFqName]) + childrenFqNames.addAll(cache.subtypesMap[removedFqName]) + + cache.supertypesMap.remove(removedFqName) + cache.subtypesMap.remove(removedFqName) + } + + for (child in childrenFqNames) { + cache.supertypesMap.removeValues(child, removedFqNames) + } + + for (parent in parentsFqNames) { + cache.subtypesMap.removeValues(parent, removedFqNames) + } + } + } + private inner class DirtyOutputClassesMap(storageFile: File) : BasicStringMap(storageFile, BooleanDataDescriptor.INSTANCE) { public fun markDirty(className: String) { storage[className] = true diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/BasicMap.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/BasicMap.kt index 839e645b726..64bf6b88f91 100644 --- a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/BasicMap.kt +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/BasicMap.kt @@ -60,7 +60,10 @@ internal abstract class BasicMap, V>( }.toString() } + @TestOnly protected abstract fun dumpKey(key: K): String + + @TestOnly protected abstract fun dumpValue(value: V): String } diff --git a/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/ClassOneToManyMap.kt b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/ClassOneToManyMap.kt new file mode 100644 index 00000000000..249a96e8d02 --- /dev/null +++ b/jps-plugin/src/org/jetbrains/kotlin/jps/incremental/storage/ClassOneToManyMap.kt @@ -0,0 +1,55 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.jps.incremental.storage + +import org.jetbrains.kotlin.jps.incremental.dumpCollection +import org.jetbrains.kotlin.name.FqName +import java.io.File + +internal open class ClassOneToManyMap( + storageFile: File +) : BasicStringMap>(storageFile, StringCollectionExternalizer) { + override fun dumpValue(value: Collection): String = value.dumpCollection() + + fun add(key: FqName, value: FqName) { + storage.append(key.asString()) { out -> out.writeUTF(value.asString()) } + } + + operator fun get(key: FqName): Collection = + storage[key.asString()]?.map(::FqName) ?: setOf() + + operator fun set(key: FqName, values: Collection) { + if (values.isEmpty()) { + remove(key) + return + } + + storage[key.asString()] = values.map(FqName::asString) + } + + fun remove(key: FqName) { + storage.remove(key.asString()) + } + + fun removeValues(key: FqName, removed: Set) { + val notRemoved = this[key].filter { it !in removed } + this[key] = notRemoved + } +} + +internal class SubtypesMap(storageFile: File) : ClassOneToManyMap(storageFile) +internal class SupertypesMap(storageFile: File) : ClassOneToManyMap(storageFile) diff --git a/jps-plugin/test/org/jetbrains/kotlin/jps/build/ExperimentalIncrementalLazyCachesTestGenerated.java b/jps-plugin/test/org/jetbrains/kotlin/jps/build/ExperimentalIncrementalLazyCachesTestGenerated.java index 1cacaaa5088..0a61111e4b0 100644 --- a/jps-plugin/test/org/jetbrains/kotlin/jps/build/ExperimentalIncrementalLazyCachesTestGenerated.java +++ b/jps-plugin/test/org/jetbrains/kotlin/jps/build/ExperimentalIncrementalLazyCachesTestGenerated.java @@ -41,6 +41,12 @@ public class ExperimentalIncrementalLazyCachesTestGenerated extends AbstractExpe doTest(fileName); } + @TestMetadata("classInheritance") + public void testClassInheritance() throws Exception { + String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/"); + doTest(fileName); + } + @TestMetadata("constant") public void testConstant() throws Exception { String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/lazyKotlinCaches/constant/"); diff --git a/jps-plugin/test/org/jetbrains/kotlin/jps/build/IncrementalLazyCachesTestGenerated.java b/jps-plugin/test/org/jetbrains/kotlin/jps/build/IncrementalLazyCachesTestGenerated.java index 865585c9c52..a84128bbfb3 100644 --- a/jps-plugin/test/org/jetbrains/kotlin/jps/build/IncrementalLazyCachesTestGenerated.java +++ b/jps-plugin/test/org/jetbrains/kotlin/jps/build/IncrementalLazyCachesTestGenerated.java @@ -41,6 +41,12 @@ public class IncrementalLazyCachesTestGenerated extends AbstractIncrementalLazyC doTest(fileName); } + @TestMetadata("classInheritance") + public void testClassInheritance() throws Exception { + String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/"); + doTest(fileName); + } + @TestMetadata("constant") public void testConstant() throws Exception { String fileName = KotlinTestUtils.navigationMetadata("jps-plugin/testData/incremental/lazyKotlinCaches/constant/"); diff --git a/jps-plugin/testData/incremental/changeIncrementalOption/experimentalOn/expected-kotlin-caches.txt b/jps-plugin/testData/incremental/changeIncrementalOption/experimentalOn/expected-kotlin-caches.txt index 3df952e931c..c892b694310 100644 --- a/jps-plugin/testData/incremental/changeIncrementalOption/experimentalOn/expected-kotlin-caches.txt +++ b/jps-plugin/testData/incremental/changeIncrementalOption/experimentalOn/expected-kotlin-caches.txt @@ -16,6 +16,8 @@ Module 'module2' production package-parts.tab proto.tab source-to-classes.tab + subtypes.tab + supertypes.tab Module 'module2' tests Module 'module3' production experimental-format-version.txt diff --git a/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/build.log b/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/build.log new file mode 100644 index 00000000000..0eb0c755f29 --- /dev/null +++ b/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/build.log @@ -0,0 +1,8 @@ +Cleaning output files: +out/production/module/A.class +out/production/module/B.class +out/production/module/C.class +End of files +Compiling files: +src/main.kt +End of files diff --git a/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/expected-kotlin-caches.txt b/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/expected-kotlin-caches.txt new file mode 100644 index 00000000000..cf4dcf7ea18 --- /dev/null +++ b/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/expected-kotlin-caches.txt @@ -0,0 +1,6 @@ +kotlin-data-container +Module 'module' production + format-version.txt + proto.tab + source-to-classes.tab +Module 'module' tests \ No newline at end of file diff --git a/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/experimental-expected-kotlin-caches.txt b/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/experimental-expected-kotlin-caches.txt new file mode 100644 index 00000000000..61e7c351e3b --- /dev/null +++ b/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/experimental-expected-kotlin-caches.txt @@ -0,0 +1,14 @@ +kotlin-data-container + data-container-format-version.txt + counters.tab + file-to-id.tab + id-to-file.tab + lookups.tab +Module 'module' production + experimental-format-version.txt + format-version.txt + proto.tab + source-to-classes.tab + subtypes.tab + supertypes.tab +Module 'module' tests diff --git a/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/main.kt b/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/main.kt new file mode 100644 index 00000000000..e425830313d --- /dev/null +++ b/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/main.kt @@ -0,0 +1,5 @@ +open class A + +open class B : A() + +class C : B() \ No newline at end of file diff --git a/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/main.kt.touch b/jps-plugin/testData/incremental/lazyKotlinCaches/classInheritance/main.kt.touch new file mode 100644 index 00000000000..e69de29bb2d