azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract common caches

Browse Source
This commit is contained in:
Alexey Tsvetkov
2017-06-26 20:35:41 +03:00
parent 367b9f6664
commit 3ad5e78d77
10 changed files with 229 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheCommon.kt
+117
View File
@@ -0,0 +1,117 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.incremental
import com.intellij.util.io.EnumeratorStringDescriptor
import org.jetbrains.kotlin.incremental.storage.*
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.serialization.ProtoBuf
import org.jetbrains.kotlin.serialization.deserialization.NameResolver
import org.jetbrains.kotlin.serialization.deserialization.TypeTable
import org.jetbrains.kotlin.serialization.deserialization.supertypes
import java.io.File
/**
* Incremental cache common for JVM and JS
*/
open class IncrementalCacheCommon(workingDir: File) : BasicMapsOwner(workingDir) {
companion object {
private val SUBTYPES = "subtypes"
private val SUPERTYPES = "supertypes"
private val CLASS_FQ_NAME_TO_SOURCE = "class-fq-name-to-source"
}
private val dependents = arrayListOf<IncrementalCacheCommon>()
fun addDependentCache(cache: IncrementalCacheCommon) {
dependents.add(cache)
}
val thisWithDependentCaches: Iterable<IncrementalCacheCommon> by lazy {
val result = arrayListOf(this)
result.addAll(dependents)
result
}
private val subtypesMap = registerMap(SubtypesMap(SUBTYPES.storageFile))
private val supertypesMap = registerMap(SupertypesMap(SUPERTYPES.storageFile))
protected val classFqNameToSourceMap = registerMap(ClassFqNameToSourceMap(CLASS_FQ_NAME_TO_SOURCE.storageFile))
fun getSubtypesOf(className: FqName): Sequence<FqName> =
subtypesMap[className].asSequence()
fun getSourceFileIfClass(fqName: FqName): File? =
classFqNameToSourceMap[fqName]
protected fun addToClassStorage(proto: ProtoBuf.Class, nameResolver: NameResolver, srcFile: File) {
val supertypes = proto.supertypes(TypeTable(proto.typeTable))
val parents = supertypes.map { nameResolver.getClassId(it.className).asSingleFqName() }
.filter { it.asString() != "kotlin.Any" }
.toSet()
val child = nameResolver.getClassId(proto.fqName).asSingleFqName()
parents.forEach { subtypesMap.add(it, child) }
val removedSupertypes = supertypesMap[child].filter { it !in parents }
removedSupertypes.forEach { subtypesMap.removeValues(it, setOf(child)) }
supertypesMap[child] = parents
classFqNameToSourceMap[child] = srcFile
}
protected fun removeAllFromClassStorage(removedClasses: Collection<FqName>) {
if (removedClasses.isEmpty()) return
val removedFqNames = removedClasses.toSet()
for (cache in thisWithDependentCaches) {
val parentsFqNames = hashSetOf<FqName>()
val childrenFqNames = hashSetOf<FqName>()
for (removedFqName in removedFqNames) {
parentsFqNames.addAll(cache.supertypesMap[removedFqName])
childrenFqNames.addAll(cache.subtypesMap[removedFqName])
cache.supertypesMap.remove(removedFqName)
cache.subtypesMap.remove(removedFqName)
}
for (child in childrenFqNames) {
cache.supertypesMap.removeValues(child, removedFqNames)
}
for (parent in parentsFqNames) {
cache.subtypesMap.removeValues(parent, removedFqNames)
}
}
removedFqNames.forEach { classFqNameToSourceMap.remove(it) }
}
protected class ClassFqNameToSourceMap(storageFile: File) : BasicStringMap<String>(storageFile, EnumeratorStringDescriptor(), PathStringDescriptor) {
operator fun set(fqName: FqName, sourceFile: File) {
storage[fqName.asString()] = sourceFile.canonicalPath
}
operator fun get(fqName: FqName): File? =
storage[fqName.asString()]?.let(::File)
fun remove(fqName: FqName) {
storage.remove(fqName.asString())
}
override fun dumpValue(value: String) = value
}
}
build-common/src/org/jetbrains/kotlin/incremental/IncrementalCacheImpl.kt
+4 -91
View File
@@ -38,8 +38,6 @@ import org.jetbrains.kotlin.resolve.jvm.JvmClassName
import org.jetbrains.kotlin.serialization.Flags
import org.jetbrains.kotlin.serialization.ProtoBuf
import org.jetbrains.kotlin.serialization.deserialization.NameResolver
import org.jetbrains.kotlin.serialization.deserialization.TypeTable
import org.jetbrains.kotlin.serialization.deserialization.supertypes
import org.jetbrains.kotlin.serialization.jvm.BitEncoding
import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
import org.jetbrains.org.objectweb.asm.*
@@ -53,7 +51,7 @@ open class IncrementalCacheImpl<Target>(
private val targetDataRoot: File,
targetOutputDir: File?,
target: Target
) : BasicMapsOwner(), IncrementalCache {
) : IncrementalCacheCommon(File(targetDataRoot, KOTLIN_CACHE_DIRECTORY_NAME)), IncrementalCache {
companion object {
private val PROTO_MAP = "proto"
private val CONSTANTS_MAP = "constants"
@@ -63,19 +61,11 @@ open class IncrementalCacheImpl<Target>(
private val SOURCE_TO_CLASSES = "source-to-classes"
private val DIRTY_OUTPUT_CLASSES = "dirty-output-classes"
private val INLINE_FUNCTIONS = "inline-functions"
private val SUBTYPES = "subtypes"
private val SUPERTYPES = "supertypes"
private val CLASS_FQ_NAME_TO_SOURCE = "class-fq-name-to-source"
private val INTERNAL_NAME_TO_SOURCE = "internal-name-to-source"
private val MODULE_MAPPING_FILE_NAME = "." + ModuleMapping.MAPPING_FILE_EXT
}
private val baseDir = File(targetDataRoot, KOTLIN_CACHE_DIRECTORY_NAME)
protected val String.storageFile: File
get() = File(baseDir, this + "." + CACHE_EXTENSION)
private val protoMap = registerMap(ProtoMap(PROTO_MAP.storageFile))
private val constantsMap = registerMap(ConstantsMap(CONSTANTS_MAP.storageFile))
private val packagePartMap = registerMap(PackagePartMap(PACKAGE_PARTS.storageFile))
@@ -84,27 +74,13 @@ open class IncrementalCacheImpl<Target>(
private val sourceToClassesMap = registerMap(SourceToClassesMap(SOURCE_TO_CLASSES.storageFile))
private val dirtyOutputClassesMap = registerMap(DirtyOutputClassesMap(DIRTY_OUTPUT_CLASSES.storageFile))
private val inlineFunctionsMap = registerMap(InlineFunctionsMap(INLINE_FUNCTIONS.storageFile))
private val subtypesMap = registerMap(SubtypesMap(SUBTYPES.storageFile))
private val supertypesMap = registerMap(SupertypesMap(SUPERTYPES.storageFile))
private val classFqNameToSourceMap = registerMap(ClassFqNameToSourceMap(CLASS_FQ_NAME_TO_SOURCE.storageFile))
// todo: try to use internal names only?
private val internalNameToSource = registerMap(InternalNameToSourcesMap(INTERNAL_NAME_TO_SOURCE.storageFile))
private val dependents = arrayListOf<IncrementalCacheImpl<Target>>()
private val outputDir by lazy(LazyThreadSafetyMode.NONE) { requireNotNull(targetOutputDir) { "Target is expected to have output directory: $target" } }
val thisWithDependentCaches: Iterable<IncrementalCacheImpl<Target>> by lazy {
val result = arrayListOf(this)
result.addAll(dependents)
result
}
protected open fun debugLog(message: String) {}
fun addDependentCache(cache: IncrementalCacheImpl<Target>) {
dependents.add(cache)
}
fun markOutputClassesDirty(removedAndCompiledSources: List<File>) {
for (sourceFile in removedAndCompiledSources) {
val classes = sourceToClassesMap[sourceFile]
@@ -121,12 +97,6 @@ open class IncrementalCacheImpl<Target>(
fun classesBySources(sources: Iterable<File>): Iterable<JvmClassName> =
sources.flatMap { sourceToClassesMap[it] }
fun getSubtypesOf(className: FqName): Sequence<FqName> =
subtypesMap[className].asSequence()
fun getSourceFileIfClass(fqName: FqName): File? =
classFqNameToSourceMap[fqName]
fun sourcesByInternalName(internalName: String): Collection<File> =
internalNameToSource[internalName]
@@ -310,7 +280,7 @@ open class IncrementalCacheImpl<Target>(
internalNameToSource.remove(it.internalName)
}
removeAllFromClassStorage(dirtyClasses)
removeAllFromClassStorage(dirtyClasses.map { it.fqNameForClassNameWithoutDollars })
dirtyOutputClassesMap.clean()
return changesInfo
@@ -555,22 +525,6 @@ open class IncrementalCacheImpl<Target>(
}
}
inner class ClassFqNameToSourceMap(storageFile: File) : BasicStringMap<String>(storageFile, EnumeratorStringDescriptor(), PathStringDescriptor) {
operator fun set(fqName: FqName, sourceFile: File) {
storage[fqName.asString()] = sourceFile.canonicalPath
}
operator fun get(fqName: FqName): File? =
storage[fqName.asString()]?.let(::File)
fun remove(fqName: FqName) {
storage.remove(fqName.asString())
}
override fun dumpValue(value: String) = value
}
inner class InternalNameToSourcesMap(storageFile: File) : BasicStringMap<Collection<String>>(storageFile, EnumeratorStringDescriptor(), PathCollectionExternalizer) {
operator fun set(internalName: String, sourceFiles: Iterable<File>) {
storage[internalName] = sourceFiles.map { it.canonicalPath }
@@ -588,49 +542,8 @@ open class IncrementalCacheImpl<Target>(
}
private fun addToClassStorage(kotlinClass: LocalFileKotlinClass, srcFile: File) {
val classData = JvmProtoBufUtil.readClassDataFrom(kotlinClass.classHeader.data!!, kotlinClass.classHeader.strings!!)
val supertypes = classData.classProto.supertypes(TypeTable(classData.classProto.typeTable))
val parents = supertypes.map { classData.nameResolver.getClassId(it.className).asSingleFqName() }
.filter { it.asString() != "kotlin.Any" }
.toSet()
val child = kotlinClass.classId.asSingleFqName()
parents.forEach { subtypesMap.add(it, child) }
val removedSupertypes = supertypesMap[child].filter { it !in parents }
removedSupertypes.forEach { subtypesMap.removeValues(it, setOf(child)) }
supertypesMap[child] = parents
classFqNameToSourceMap[kotlinClass.className.fqNameForClassNameWithoutDollars] = srcFile
}
private fun removeAllFromClassStorage(removedClasses: Collection<JvmClassName>) {
if (removedClasses.isEmpty()) return
val removedFqNames = removedClasses.map { it.fqNameForClassNameWithoutDollars }.toSet()
for (cache in thisWithDependentCaches) {
val parentsFqNames = hashSetOf<FqName>()
val childrenFqNames = hashSetOf<FqName>()
for (removedFqName in removedFqNames) {
parentsFqNames.addAll(cache.supertypesMap[removedFqName])
childrenFqNames.addAll(cache.subtypesMap[removedFqName])
cache.supertypesMap.remove(removedFqName)
cache.subtypesMap.remove(removedFqName)
}
for (child in childrenFqNames) {
cache.supertypesMap.removeValues(child, removedFqNames)
}
for (parent in parentsFqNames) {
cache.subtypesMap.removeValues(parent, removedFqNames)
}
}
removedFqNames.forEach { classFqNameToSourceMap.remove(it) }
val (nameResolver, proto) = JvmProtoBufUtil.readClassDataFrom(kotlinClass.classHeader.data!!, kotlinClass.classHeader.strings!!)
addToClassStorage(proto, nameResolver, srcFile)
}
private inner class DirtyOutputClassesMap(storageFile: File) : BasicStringMap<Boolean>(storageFile, BooleanDataDescriptor.INSTANCE) {
build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt
+1 -4
View File
@@ -29,15 +29,12 @@ import java.io.File
import java.util.*
open class LookupStorage(private val targetDataDir: File) : BasicMapsOwner() {
open class LookupStorage(targetDataDir: File) : BasicMapsOwner(targetDataDir) {
companion object {
private val DELETED_TO_SIZE_TRESHOLD = 0.5
private val MINIMUM_GARBAGE_COLLECTIBLE_SIZE = 10000
}
private val String.storageFile: File
get() = File(targetDataDir, this + "." + CACHE_EXTENSION)
private val countersFile = "counters".storageFile
private val idToFile = registerMap(IdToFileMap("id-to-file".storageFile))
private val fileToId = registerMap(FileToIdMap("file-to-id".storageFile))
build-common/src/org/jetbrains/kotlin/incremental/buildUtil.kt
+6 -6
View File
@@ -187,8 +187,8 @@ data class DirtyData(
val dirtyClassesFqNames: Collection<FqName> = emptyList()
)
fun <Target> CompilationResult.getDirtyData(
caches: Iterable<IncrementalCacheImpl<Target>>,
fun CompilationResult.getDirtyData(
caches: Iterable<IncrementalCacheCommon>,
reporter: ICReporter
): DirtyData {
val dirtyLookupSymbols = HashSet<LookupSymbol>()
@@ -241,8 +241,8 @@ fun mapLookupSymbolsToFiles(
return dirtyFiles
}
fun <Target> mapClassesFqNamesToFiles(
caches: Iterable<IncrementalCacheImpl<Target>>,
fun mapClassesFqNamesToFiles(
caches: Iterable<IncrementalCacheCommon>,
classesFqNames: Iterable<FqName>,
reporter: ICReporter,
excludes: Set<File> = emptySet()
@@ -265,9 +265,9 @@ fun <Target> mapClassesFqNamesToFiles(
private fun findSrcDirRoot(file: File, roots: Iterable<File>): File? =
roots.firstOrNull { FileUtil.isAncestor(it, file, false) }
fun <Target> withSubtypes(
fun withSubtypes(
typeFqName: FqName,
caches: Iterable<IncrementalCacheImpl<Target>>
caches: Iterable<IncrementalCacheCommon>
): Set<FqName> {
val types = LinkedList(listOf(typeFqName))
val subtypes = hashSetOf<FqName>()
build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMapsOwner.kt
+5 -1
View File
@@ -17,14 +17,18 @@
package org.jetbrains.kotlin.incremental.storage
import org.jetbrains.annotations.TestOnly
import java.io.File
open class BasicMapsOwner {
open class BasicMapsOwner(private val baseDir: File) {
private val maps = arrayListOf<BasicMap<*, *>>()
companion object {
val CACHE_EXTENSION = "tab"
}
protected val String.storageFile: File
get() = File(baseDir, this + "." + CACHE_EXTENSION)
protected fun <K, V, M : BasicMap<K, V>> registerMap(map: M): M {
maps.add(map)
return map
build-common/src/org/jetbrains/kotlin/incremental/storage/LazyStorage.kt
+1 -1
View File
@@ -87,7 +87,7 @@ class LazyStorage<K, V>(
try {
storage?.close()
}
catch (ignored: IOException) {
catch (ignored: Throwable) {
}
PersistentHashMap.deleteFilesStartingWith(storageFile)